diff --git a/chain/src/main/java/org/ethereum/beacon/chain/BeaconTupleDetails.java b/chain/src/main/java/org/ethereum/beacon/chain/BeaconTupleDetails.java index 1a7e57796..c77473f04 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/BeaconTupleDetails.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/BeaconTupleDetails.java @@ -38,4 +38,8 @@ public BeaconStateEx getFinalState() { return getState(); } + @Override + public String toString() { + return getFinalState().toString(); + } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java b/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java index aa3fca282..3ada9d6ec 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java @@ -98,17 +98,17 @@ private void initializeStorage() { } @Override - public synchronized boolean insert(BeaconBlock block) { + public synchronized ImportResult insert(BeaconBlock block) { if (rejectedByTime(block)) { - return false; + return ImportResult.ExpiredBlock; } if (exist(block)) { - return false; + return ImportResult.ExistingBlock; } if (!hasParent(block)) { - return false; + return ImportResult.NoParent; } long s = System.nanoTime(); @@ -121,7 +121,7 @@ public synchronized boolean insert(BeaconBlock block) { if (!blockVerification.isPassed()) { logger.warn("Block verification failed: " + blockVerification + ": " + block.toString(spec.getConstants(), parentState.getGenesisTime(), spec::signing_root)); - return false; + return ImportResult.InvalidBlock; } BeaconStateEx postBlockState = blockTransition.apply(preBlockState, block); @@ -130,7 +130,7 @@ public synchronized boolean insert(BeaconBlock block) { stateVerifier.verify(postBlockState, block); if (!stateVerification.isPassed()) { logger.warn("State verification failed: " + stateVerification); - return false; + return ImportResult.StateMismatch; } BeaconTuple newTuple = BeaconTuple.of(block, postBlockState); @@ -154,7 +154,7 @@ public synchronized boolean insert(BeaconBlock block) { spec::signing_root), String.format("%.3f", ((double) total) / 1_000_000_000d)); - return true; + return ImportResult.OK; } @Override diff --git a/chain/src/main/java/org/ethereum/beacon/chain/MutableBeaconChain.java b/chain/src/main/java/org/ethereum/beacon/chain/MutableBeaconChain.java index ebd575f89..23d4f2a04 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/MutableBeaconChain.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/MutableBeaconChain.java @@ -4,11 +4,21 @@ public interface MutableBeaconChain extends BeaconChain { + enum ImportResult { + OK, + ExistingBlock, + NoParent, + ExpiredBlock, + InvalidBlock, + StateMismatch, + UnexpectedError + } + /** * Inserts new block into a chain. * * @param block a block. * @return whether a block was inserted or not. */ - boolean insert(BeaconBlock block); + ImportResult insert(BeaconBlock block); } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/ProposedBlockProcessorImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/ProposedBlockProcessorImpl.java index 1a81bacfa..0ff909478 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/ProposedBlockProcessorImpl.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/ProposedBlockProcessorImpl.java @@ -1,5 +1,6 @@ package org.ethereum.beacon.chain; +import org.ethereum.beacon.chain.MutableBeaconChain.ImportResult; import org.ethereum.beacon.core.BeaconBlock; import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.beacon.stream.SimpleProcessor; @@ -19,10 +20,10 @@ public ProposedBlockProcessorImpl(MutableBeaconChain beaconChain, Schedulers sch } @Override - public void newBlockProposed(BeaconBlock newBlcok) { - boolean result = beaconChain.insert(newBlcok); - if (result) { - blocksStream.onNext(newBlcok); + public void newBlockProposed(BeaconBlock newBlock) { + ImportResult result = beaconChain.insert(newBlock); + if (result == ImportResult.OK) { + blocksStream.onNext(newBlock); } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorImpl.java index f45862aa3..d7d7b56c5 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorImpl.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/observer/ObservableStateProcessorImpl.java @@ -11,6 +11,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.stream.Collectors; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.ethereum.beacon.chain.BeaconChainHead; import org.ethereum.beacon.chain.BeaconTuple; import org.ethereum.beacon.chain.BeaconTupleDetails; @@ -39,7 +41,12 @@ import reactor.core.publisher.Flux; public class ObservableStateProcessorImpl implements ObservableStateProcessor { + private static final Logger logger = LogManager.getLogger(ObservableStateProcessorImpl.class); + private static final int MAX_TUPLE_CACHE_SIZE = 32; + + private final int maxEmptySlotTransitions = 256; + private final BeaconTupleStorage tupleStorage; private final HeadFunction headFunction; @@ -218,22 +225,40 @@ private void newHead(BeaconTupleDetails head) { private void newSlot(SlotNumber newSlot) { if (head.getBlock().getSlot().greater(newSlot)) { + logger.info("Ignore new slot " + newSlot + " below head block: " + head.getBlock()); + return; + } + if (newSlot.greater(head.getBlock().getSlot().plus(maxEmptySlotTransitions))) { + logger.debug("Ignore new slot " + newSlot + " far above head block: " + head.getBlock()); return; } + updateCurrentObservableState(head, newSlot); } private void updateCurrentObservableState(BeaconTupleDetails head, SlotNumber slot) { assert slot.greaterEqual(head.getBlock().getSlot()); - PendingOperations pendingOperations = - getPendingOperations(head.getFinalState(), copyAttestationCache()); if (slot.greater(head.getBlock().getSlot())) { - BeaconStateEx stateUponASlot = emptySlotTransition.apply(head.getFinalState(), slot); + BeaconStateEx stateUponASlot; + if (latestState.getSlot().greater(spec.getConstants().getGenesisSlot()) + && spec.getObjectHasher() + .getHashTruncateLast(head.getBlock()) + .equals( + spec.get_block_root_at_slot(latestState, latestState.getSlot().decrement()))) { + + // latestState is actual with respect to current head + stateUponASlot = emptySlotTransition.apply(latestState, slot); + } else { + // recalculate all empty slots starting from the head + stateUponASlot = emptySlotTransition.apply(head.getFinalState(), slot); + } latestState = stateUponASlot; + PendingOperations pendingOperations = getPendingOperations(stateUponASlot, copyAttestationCache()); observableStateStream.onNext( new ObservableBeaconState(head.getBlock(), stateUponASlot, pendingOperations)); } else { + PendingOperations pendingOperations = getPendingOperations(head.getFinalState(), copyAttestationCache()); if (head.getPostSlotState().isPresent()) { latestState = head.getPostSlotState().get(); observableStateStream.onNext(new ObservableBeaconState( @@ -255,16 +280,9 @@ private PendingOperations getPendingOperations( BeaconState state, Map> attestationMap) { List attestations = attestationMap.values().stream() .flatMap(Collection::stream) - .filter(attestation -> { - /* attestation_slot = get_attestation_slot(state, attestation) - assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH */ - SlotNumber attestationSlot = spec.get_attestation_slot(state, attestation.getData()); - SlotNumber lowerBoundary = - attestationSlot.plus(spec.getConstants().getMinAttestationInclusionDelay()); - SlotNumber upperBoundary = attestationSlot.plus(spec.getConstants().getSlotsPerEpoch()); - return lowerBoundary.lessEqual(state.getSlot()) - && state.getSlot().lessEqual(upperBoundary); - }) + .filter(attestation -> + attestation.getData().getTargetEpoch().lessEqual(spec.get_current_epoch(state))) + .filter(attestation -> spec.verify_attestation(state, attestation)) .sorted(Comparator.comparing(attestation -> attestation.getData().getTargetEpoch())) .collect(Collectors.toList()); diff --git a/chain/src/main/java/org/ethereum/beacon/chain/observer/PendingOperationsState.java b/chain/src/main/java/org/ethereum/beacon/chain/observer/PendingOperationsState.java index 49516316c..6d7ffec5c 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/observer/PendingOperationsState.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/observer/PendingOperationsState.java @@ -53,7 +53,7 @@ public List peekAggregateAttestations(int maxCount) { private Attestation aggregateAttestations(List attestations) { assert !attestations.isEmpty(); - assert attestations.stream().skip(1).allMatch(a -> a.equals(attestations.get(0))); + assert attestations.stream().skip(1).allMatch(a -> a.getData().equals(attestations.get(0).getData())); Bitfield participants = attestations.stream() diff --git a/chain/src/main/java/org/ethereum/beacon/chain/storage/BeaconChainStorage.java b/chain/src/main/java/org/ethereum/beacon/chain/storage/BeaconChainStorage.java index d6f88c956..1fe9e21f9 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/storage/BeaconChainStorage.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/storage/BeaconChainStorage.java @@ -1,5 +1,7 @@ package org.ethereum.beacon.chain.storage; +import org.ethereum.beacon.core.BeaconBlockHeader; +import org.ethereum.beacon.db.source.DataSource; import org.ethereum.beacon.db.source.SingleValueSource; import tech.pegasys.artemis.ethereum.core.Hash32; @@ -7,6 +9,8 @@ public interface BeaconChainStorage { BeaconBlockStorage getBlockStorage(); + DataSource getBlockHeaderStorage(); + BeaconStateStorage getStateStorage(); BeaconTupleStorage getTupleStorage(); diff --git a/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/BeaconChainStorageImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/BeaconChainStorageImpl.java index 49fb87c81..748ceb2c2 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/BeaconChainStorageImpl.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/BeaconChainStorageImpl.java @@ -4,6 +4,8 @@ import org.ethereum.beacon.chain.storage.BeaconChainStorage; import org.ethereum.beacon.chain.storage.BeaconStateStorage; import org.ethereum.beacon.chain.storage.BeaconTupleStorage; +import org.ethereum.beacon.core.BeaconBlockHeader; +import org.ethereum.beacon.db.source.DataSource; import org.ethereum.beacon.db.source.SingleValueSource; import tech.pegasys.artemis.ethereum.core.Hash32; @@ -11,6 +13,7 @@ public class BeaconChainStorageImpl implements BeaconChainStorage { private final BeaconBlockStorage blockStorage; + private final DataSource blockHeaderStorage; private final BeaconStateStorage stateStorage; private final BeaconTupleStorage tupleStorage; private final SingleValueSource justifiedStorage; @@ -18,11 +21,13 @@ public class BeaconChainStorageImpl implements BeaconChainStorage { public BeaconChainStorageImpl( BeaconBlockStorage blockStorage, + DataSource blockHeaderStorage, BeaconStateStorage stateStorage, BeaconTupleStorage tupleStorage, SingleValueSource justifiedStorage, SingleValueSource finalizedStorage) { this.blockStorage = blockStorage; + this.blockHeaderStorage = blockHeaderStorage; this.stateStorage = stateStorage; this.tupleStorage = tupleStorage; this.justifiedStorage = justifiedStorage; @@ -34,6 +39,11 @@ public BeaconBlockStorage getBlockStorage() { return blockStorage; } + @Override + public DataSource getBlockHeaderStorage() { + return blockHeaderStorage; + } + @Override public BeaconStateStorage getStateStorage() { return stateStorage; diff --git a/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/DelegateBlockHeaderStorageImpl.java b/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/DelegateBlockHeaderStorageImpl.java new file mode 100644 index 000000000..767f7f8d0 --- /dev/null +++ b/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/DelegateBlockHeaderStorageImpl.java @@ -0,0 +1,54 @@ +package org.ethereum.beacon.chain.storage.impl; + +import java.util.Optional; +import javax.annotation.Nonnull; +import org.ethereum.beacon.chain.storage.BeaconBlockStorage; +import org.ethereum.beacon.consensus.hasher.ObjectHasher; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.BeaconBlockHeader; +import org.ethereum.beacon.db.source.DataSource; +import tech.pegasys.artemis.ethereum.core.Hash32; + +public class DelegateBlockHeaderStorageImpl implements DataSource { + + private final BeaconBlockStorage delegateBlockStorage; + private final ObjectHasher objectHasher; + + public DelegateBlockHeaderStorageImpl( + BeaconBlockStorage delegateBlockStorage, + ObjectHasher objectHasher) { + this.delegateBlockStorage = delegateBlockStorage; + this.objectHasher = objectHasher; + } + + @Override + public Optional get(@Nonnull Hash32 key) { + return delegateBlockStorage + .get(key) + .map(this::createHeader); + } + + private BeaconBlockHeader createHeader(BeaconBlock block) { + return new BeaconBlockHeader( + block.getSlot(), + block.getPreviousBlockRoot(), + block.getStateRoot(), + objectHasher.getHash(block.getBody()), + block.getSignature()); + } + + @Override + public void put(@Nonnull Hash32 key, @Nonnull BeaconBlockHeader value) { + throw new UnsupportedOperationException(); + } + + @Override + public void remove(@Nonnull Hash32 key) { + throw new UnsupportedOperationException(); + } + + @Override + public void flush() { + throw new UnsupportedOperationException(); + } +} diff --git a/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/MemBeaconChainStorageFactory.java b/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/MemBeaconChainStorageFactory.java index ab0d8f7cd..04d716a74 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/MemBeaconChainStorageFactory.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/MemBeaconChainStorageFactory.java @@ -27,7 +27,12 @@ public BeaconChainStorage create(Database database) { new BeaconStateStorageImpl(new HashMapDataSource<>(), objectHasher); BeaconTupleStorage tupleStorage = new BeaconTupleStorageImpl(blockStorage, stateStorage); - return new BeaconChainStorageImpl(blockStorage, stateStorage, tupleStorage, - SingleValueSource.memSource(), SingleValueSource.memSource()); + return new BeaconChainStorageImpl( + blockStorage, + new DelegateBlockHeaderStorageImpl(blockStorage, objectHasher), + stateStorage, + tupleStorage, + SingleValueSource.memSource(), + SingleValueSource.memSource()); } } diff --git a/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/SSZBeaconChainStorageFactory.java b/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/SSZBeaconChainStorageFactory.java index 8f7b8d494..57e9f390e 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/SSZBeaconChainStorageFactory.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/SSZBeaconChainStorageFactory.java @@ -41,7 +41,12 @@ public BeaconChainStorage create(Database database) { SingleValueSource finalizedStorage = createHash32Storage(database, "finalized-hash"); return new BeaconChainStorageImpl( - blockStorage, stateStorage, tupleStorage, justifiedStorage, finalizedStorage); + blockStorage, + new DelegateBlockHeaderStorageImpl(blockStorage, objectHasher), + stateStorage, + tupleStorage, + justifiedStorage, + finalizedStorage); } private SingleValueSource createHash32Storage(Database database, String name) { diff --git a/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/SerializerFactory.java b/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/SerializerFactory.java index bd237d46d..de4c23857 100644 --- a/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/SerializerFactory.java +++ b/chain/src/main/java/org/ethereum/beacon/chain/storage/impl/SerializerFactory.java @@ -13,8 +13,7 @@ public interface SerializerFactory { Function getSerializer(Class objectClass); static SerializerFactory createSSZ(SpecConstants specConstants) { - return new SSZSerializerFactory( - new SSZBuilder() + return new SSZSerializerFactory(new SSZBuilder() .withExternalVarResolver(new SpecConstantsResolver(specConstants)) .buildSerializer()); } diff --git a/chain/src/test/java/org/ethereum/beacon/chain/DefaultBeaconChainTest.java b/chain/src/test/java/org/ethereum/beacon/chain/DefaultBeaconChainTest.java index ab16df449..e044aa91d 100644 --- a/chain/src/test/java/org/ethereum/beacon/chain/DefaultBeaconChainTest.java +++ b/chain/src/test/java/org/ethereum/beacon/chain/DefaultBeaconChainTest.java @@ -2,6 +2,7 @@ import java.util.Collections; import java.util.stream.IntStream; +import org.ethereum.beacon.chain.MutableBeaconChain.ImportResult; import org.ethereum.beacon.chain.storage.BeaconChainStorage; import org.ethereum.beacon.chain.storage.impl.SSZBeaconChainStorageFactory; import org.ethereum.beacon.chain.storage.impl.SerializerFactory; @@ -54,7 +55,7 @@ public void insertAChain() { BeaconTuple recentlyProcessed = beaconChain.getRecentlyProcessed(); BeaconBlock aBlock = createBlock(recentlyProcessed, spec, schedulers.getCurrentTime(), perSlotTransition); - Assert.assertTrue(beaconChain.insert(aBlock)); + Assert.assertEquals(ImportResult.OK, beaconChain.insert(aBlock)); Assert.assertEquals(aBlock, beaconChain.getRecentlyProcessed().getBlock()); System.out.println("Inserted block: " + (idx + 1)); diff --git a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/BlockProcessing.java b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/BlockProcessing.java index cd900f5fa..a020a74f0 100644 --- a/consensus/src/main/java/org/ethereum/beacon/consensus/spec/BlockProcessing.java +++ b/consensus/src/main/java/org/ethereum/beacon/consensus/spec/BlockProcessing.java @@ -207,16 +207,17 @@ default void process_attester_slashing(MutableBeaconState state, AttesterSlashin assertTrue(slashed_any); } - default void verify_attestation(BeaconState state, Attestation attestation) { + default boolean verify_attestation(BeaconState state, Attestation attestation) { AttestationData data = attestation.getData(); /* attestation_slot = get_attestation_slot(state, attestation) assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH */ SlotNumber attestation_slot = get_attestation_slot(state, data); - assertTrue( + if (! attestation_slot.plus(getConstants().getMinAttestationInclusionDelay()).lessEqual(state.getSlot()) - && state.getSlot().lessEqual(attestation_slot.plus(getConstants().getSlotsPerEpoch())) - ); + && state.getSlot().lessEqual(attestation_slot.plus(getConstants().getSlotsPerEpoch()))) { + return false; + } /* Check target epoch, source epoch, source root, and source crosslink data = attestation.data @@ -234,15 +235,19 @@ default void verify_attestation(BeaconState state, Attestation attestation) { && data.getSourceEpoch().equals(state.getPreviousJustifiedEpoch()) && data.getSourceRoot().equals(state.getPreviousJustifiedRoot()) && data.getPreviousCrosslinkRoot().equals(hash_tree_root(state.getPreviousCrosslinks().get(data.getShard()))); - assertTrue(current_epoch_attestation || previous_epoch_attestation); + if (!(current_epoch_attestation || previous_epoch_attestation)) { + return false; + } /* Check crosslink data root assert data.crosslink_data_root == ZERO_HASH # [to be removed in phase 1] */ - assertTrue(data.getCrosslinkDataRoot().equals(Hash32.ZERO)); + if (!(data.getCrosslinkDataRoot().equals(Hash32.ZERO))) { + return false; + } /* Check signature and bitfields assert verify_indexed_attestation(state, convert_to_indexed(state, attestation)) */ - assertTrue(verify_indexed_attestation(state, convert_to_indexed(state, attestation))); + return verify_indexed_attestation(state, convert_to_indexed(state, attestation)); } /* diff --git a/consensus/src/main/java/org/ethereum/beacon/consensus/transition/StateCachingTransition.java b/consensus/src/main/java/org/ethereum/beacon/consensus/transition/StateCachingTransition.java index 0537bb153..34938f810 100644 --- a/consensus/src/main/java/org/ethereum/beacon/consensus/transition/StateCachingTransition.java +++ b/consensus/src/main/java/org/ethereum/beacon/consensus/transition/StateCachingTransition.java @@ -26,7 +26,7 @@ public StateCachingTransition(BeaconChainSpec spec) { @Override public BeaconStateEx apply(BeaconStateEx source) { - logger.debug(() -> "Applying state caching to state: (" + + logger.trace(() -> "Applying state caching to state: (" + spec.hash_tree_root(source).toStringShort() + ") " + source.toString(spec.getConstants(), spec::signing_root)); @@ -38,7 +38,7 @@ public BeaconStateEx apply(BeaconStateEx source) { BeaconStateEx ret = new BeaconStateExImpl(state.createImmutable(), TransitionType.CACHING); - logger.debug(() -> "State caching result state: (" + + logger.trace(() -> "State caching result state: (" + spec.hash_tree_root(ret).toStringShort() + ") " + ret.toString(spec.getConstants(), spec::signing_root)); diff --git a/core/src/main/java/org/ethereum/beacon/core/BeaconBlock.java b/core/src/main/java/org/ethereum/beacon/core/BeaconBlock.java index f21f2df8f..a7c248ad3 100644 --- a/core/src/main/java/org/ethereum/beacon/core/BeaconBlock.java +++ b/core/src/main/java/org/ethereum/beacon/core/BeaconBlock.java @@ -59,6 +59,15 @@ public BeaconBlock( this.signature = signature; } + public BeaconBlock(BeaconBlockHeader header, BeaconBlockBody body) { + this( + header.getSlot(), + header.getPreviousBlockRoot(), + header.getStateRoot(), + body, + header.getSignature()); + } + @Override public Optional getHash() { return Optional.ofNullable(hashCache); diff --git a/core/src/main/java/org/ethereum/beacon/core/operations/Deposit.java b/core/src/main/java/org/ethereum/beacon/core/operations/Deposit.java index 47ff99db1..0c9f69801 100644 --- a/core/src/main/java/org/ethereum/beacon/core/operations/Deposit.java +++ b/core/src/main/java/org/ethereum/beacon/core/operations/Deposit.java @@ -57,6 +57,14 @@ public boolean equals(Object o) { && Objects.equal(data, deposit.data); } + @Override + public int hashCode() { + int result = proof.hashCode(); + result = 31 * result + index.hashCode(); + result = 31 * result + data.hashCode(); + return result; + } + @Override public String toString() { return "Deposit[" diff --git a/core/src/main/java/org/ethereum/beacon/core/operations/ProposerSlashing.java b/core/src/main/java/org/ethereum/beacon/core/operations/ProposerSlashing.java index e3d04a8e5..bb9d20efd 100644 --- a/core/src/main/java/org/ethereum/beacon/core/operations/ProposerSlashing.java +++ b/core/src/main/java/org/ethereum/beacon/core/operations/ProposerSlashing.java @@ -46,6 +46,14 @@ public boolean equals(Object o) { && Objects.equal(header2, that.header2); } + @Override + public int hashCode() { + int result = proposerIndex.hashCode(); + result = 31 * result + header1.hashCode(); + result = 31 * result + header2.hashCode(); + return result; + } + @Override public String toString() { return toString(null, null); diff --git a/core/src/main/java/org/ethereum/beacon/core/operations/VoluntaryExit.java b/core/src/main/java/org/ethereum/beacon/core/operations/VoluntaryExit.java index 481bce595..d8c75c163 100644 --- a/core/src/main/java/org/ethereum/beacon/core/operations/VoluntaryExit.java +++ b/core/src/main/java/org/ethereum/beacon/core/operations/VoluntaryExit.java @@ -67,6 +67,14 @@ public boolean equals(Object o) { && Objects.equal(signature, voluntaryExit.signature); } + @Override + public int hashCode() { + int result = epoch.hashCode(); + result = 31 * result + validatorIndex.hashCode(); + result = 31 * result + signature.hashCode(); + return result; + } + @Override public String toString() { return toString(null); diff --git a/core/src/main/java/org/ethereum/beacon/core/operations/deposit/DepositData.java b/core/src/main/java/org/ethereum/beacon/core/operations/deposit/DepositData.java index bf9291e19..f0b171d13 100644 --- a/core/src/main/java/org/ethereum/beacon/core/operations/deposit/DepositData.java +++ b/core/src/main/java/org/ethereum/beacon/core/operations/deposit/DepositData.java @@ -64,4 +64,13 @@ public boolean equals(Object o) { && Objects.equal(amount, that.amount) && Objects.equal(signature, that.signature); } + + @Override + public int hashCode() { + int result = pubKey.hashCode(); + result = 31 * result + withdrawalCredentials.hashCode(); + result = 31 * result + amount.hashCode(); + result = 31 * result + signature.hashCode(); + return result; + } } diff --git a/core/src/main/java/org/ethereum/beacon/core/operations/slashing/AttesterSlashing.java b/core/src/main/java/org/ethereum/beacon/core/operations/slashing/AttesterSlashing.java index 1fd835638..552251a5d 100644 --- a/core/src/main/java/org/ethereum/beacon/core/operations/slashing/AttesterSlashing.java +++ b/core/src/main/java/org/ethereum/beacon/core/operations/slashing/AttesterSlashing.java @@ -35,6 +35,13 @@ public boolean equals(Object o) { return attestation2.equals(that.attestation2); } + @Override + public int hashCode() { + int result = attestation1.hashCode(); + result = 31 * result + attestation2.hashCode(); + return result; + } + @Override public String toString() { return toString(null, null); diff --git a/core/src/main/java/org/ethereum/beacon/core/operations/slashing/IndexedAttestation.java b/core/src/main/java/org/ethereum/beacon/core/operations/slashing/IndexedAttestation.java index def78711b..91588490f 100644 --- a/core/src/main/java/org/ethereum/beacon/core/operations/slashing/IndexedAttestation.java +++ b/core/src/main/java/org/ethereum/beacon/core/operations/slashing/IndexedAttestation.java @@ -86,6 +86,15 @@ public boolean equals(Object o) { && Objects.equal(signature, that.signature); } + @Override + public int hashCode() { + int result = custodyBit0Indices.hashCode(); + result = 31 * result + custodyBit1Indices.hashCode(); + result = 31 * result + data.hashCode(); + result = 31 * result + signature.hashCode(); + return result; + } + @Override public String toString() { return toString(null, null); diff --git a/core/src/test/java/org/ethereum/beacon/core/ModelsSerializeTest.java b/core/src/test/java/org/ethereum/beacon/core/ModelsSerializeTest.java index 40dd2975c..ec5b70498 100644 --- a/core/src/test/java/org/ethereum/beacon/core/ModelsSerializeTest.java +++ b/core/src/test/java/org/ethereum/beacon/core/ModelsSerializeTest.java @@ -37,6 +37,7 @@ import org.ethereum.beacon.core.types.SlotNumber; import org.ethereum.beacon.core.types.ValidatorIndex; import org.ethereum.beacon.core.util.BeaconBlockTestUtil; +import org.ethereum.beacon.core.util.TestDataFactory; import org.ethereum.beacon.crypto.Hashes; import org.ethereum.beacon.ssz.SSZBuilder; import org.ethereum.beacon.ssz.SSZSerializer; @@ -53,6 +54,7 @@ public class ModelsSerializeTest { private SSZSerializer sszSerializer; private SpecConstants specConstants; + private TestDataFactory dataFactory; @Before public void setup() { @@ -60,93 +62,37 @@ public void setup() { sszSerializer = new SSZBuilder() .withExternalVarResolver(new SpecConstantsResolver(specConstants)) .buildSerializer(); - } - - private AttestationData createAttestationData() { - AttestationData expected = - new AttestationData( - Hashes.sha256(BytesValue.fromHexString("aa")), - EpochNumber.ZERO, - Hashes.sha256(BytesValue.fromHexString("bb")), - EpochNumber.of(123), - Hashes.sha256(BytesValue.fromHexString("cc")), - ShardNumber.of(345), - Hashes.sha256(BytesValue.fromHexString("dd")), - Hash32.ZERO); - - return expected; + dataFactory = new TestDataFactory(specConstants); } @Test public void attestationDataTest() { - AttestationData expected = createAttestationData(); + AttestationData expected = dataFactory.createAttestationData(); BytesValue encoded = sszSerializer.encode2(expected); AttestationData reconstructed = sszSerializer.decode(encoded, AttestationData.class); assertEquals(expected, reconstructed); } - private Attestation createAttestation() { - AttestationData attestationData = createAttestationData(); - Attestation attestation = - new Attestation( - Bitfield.of(BytesValue.fromHexString("aa")), - attestationData, - Bitfield.of(BytesValue.fromHexString("bb")), - BLSSignature.wrap(Bytes96.fromHexString("cc"))); - - return attestation; - } - @Test public void attestationTest() { - Attestation expected = createAttestation(); + Attestation expected = dataFactory.createAttestation(); BytesValue encoded = sszSerializer.encode2(expected); Attestation reconstructed = sszSerializer.decode(encoded, Attestation.class); assertEquals(expected, reconstructed); } - private DepositData createDepositData() { - DepositData depositData = - new DepositData( - BLSPubkey.wrap(Bytes48.TRUE), - Hashes.sha256(BytesValue.fromHexString("aa")), - Gwei.ZERO, BLSSignature.wrap(Bytes96.ZERO)); - - return depositData; - } - @Test public void depositDataTest() { - DepositData expected = createDepositData(); + DepositData expected = dataFactory.createDepositData(); BytesValue encoded = sszSerializer.encode2(expected); DepositData reconstructed = sszSerializer.decode(encoded, DepositData.class); assertEquals(expected, reconstructed); } - private Deposit createDeposit1() { - Deposit deposit = new Deposit( - ReadVector.wrap( - Collections.nCopies(specConstants.getDepositContractTreeDepth().getIntValue(), Hash32.ZERO), Integer::new), - UInt64.ZERO, createDepositData()); - - return deposit; - } - - private Deposit createDeposit2() { - ArrayList hashes = new ArrayList<>(); - hashes.add(Hashes.sha256(BytesValue.fromHexString("aa"))); - hashes.add(Hashes.sha256(BytesValue.fromHexString("bb"))); - hashes.addAll(Collections.nCopies(specConstants.getDepositContractTreeDepth().getIntValue() - hashes.size(), Hash32.ZERO)); - ReadVector proof = ReadVector.wrap(hashes, Integer::new); - Deposit deposit = new Deposit(proof, UInt64.ZERO, createDepositData()); - - return deposit; - } - @Test public void depositTest() { - Deposit expected1 = createDeposit1(); - Deposit expected2 = createDeposit2(); + Deposit expected1 = dataFactory.createDeposit1(); + Deposit expected2 = dataFactory.createDeposit2(); BytesValue encoded1 = sszSerializer.encode2(expected1); BytesValue encoded2 = sszSerializer.encode2(expected2); Deposit reconstructed1 = sszSerializer.decode(encoded1, Deposit.class); @@ -155,15 +101,9 @@ public void depositTest() { assertEquals(expected2, reconstructed2); } - private VoluntaryExit createExit() { - VoluntaryExit voluntaryExit = new VoluntaryExit(EpochNumber.of(123), ValidatorIndex.MAX, BLSSignature.wrap(Bytes96.fromHexString("aa"))); - - return voluntaryExit; - } - @Test public void exitTest() { - VoluntaryExit expected = createExit(); + VoluntaryExit expected = dataFactory.createExit(); BytesValue encoded = sszSerializer.encode2(expected); VoluntaryExit reconstructed = sszSerializer.decode(encoded, VoluntaryExit.class); assertEquals(expected, reconstructed); @@ -178,73 +118,18 @@ public void beaconBlockHeaderTest() { assertEquals(expected, reconstructed); } - private ProposerSlashing createProposerSlashing(Random random) { - ProposerSlashing proposerSlashing = - new ProposerSlashing( - ValidatorIndex.MAX, - BeaconBlockTestUtil.createRandomHeader(random), - BeaconBlockTestUtil.createRandomHeader(random)); - - return proposerSlashing; - } - @Test public void proposerSlashingTest() { - Random random = new Random(); - ProposerSlashing expected = createProposerSlashing(random); + Random random = new Random(1); + ProposerSlashing expected = dataFactory.createProposerSlashing(random); BytesValue encoded = sszSerializer.encode2(expected); ProposerSlashing reconstructed = sszSerializer.decode(encoded, ProposerSlashing.class); assertEquals(expected, reconstructed); } - private BeaconBlockBody createBeaconBlockBody() { - Random random = new Random(); - List proposerSlashings = new ArrayList<>(); - proposerSlashings.add(createProposerSlashing(random)); - List attesterSlashings = new ArrayList<>(); - attesterSlashings.add(createAttesterSlashings()); - attesterSlashings.add(createAttesterSlashings()); - List attestations = new ArrayList<>(); - attestations.add(createAttestation()); - List deposits = new ArrayList<>(); - deposits.add(createDeposit1()); - deposits.add(createDeposit2()); - List voluntaryExits = new ArrayList<>(); - voluntaryExits.add(createExit()); - List transfers = new ArrayList<>(); - BeaconBlockBody beaconBlockBody = - BeaconBlockBody.create( - BLSSignature.ZERO, - new Eth1Data(Hash32.ZERO, UInt64.ZERO, Hash32.ZERO), - Bytes32.ZERO, - proposerSlashings, - attesterSlashings, - attestations, - deposits, - voluntaryExits, - transfers - ); - - return beaconBlockBody; - } - - private AttesterSlashing createAttesterSlashings() { - return new AttesterSlashing( - createSlashableAttestation(), - createSlashableAttestation()); - } - - private IndexedAttestation createSlashableAttestation() { - return new IndexedAttestation( - Arrays.asList(ValidatorIndex.of(234), ValidatorIndex.of(235)), - Arrays.asList(ValidatorIndex.of(678), ValidatorIndex.of(679)), - createAttestationData(), - BLSSignature.wrap(Bytes96.fromHexString("aa"))); - } - @Test public void slashableAttestationTest() { - IndexedAttestation expected = createSlashableAttestation(); + IndexedAttestation expected = dataFactory.createSlashableAttestation(); BytesValue encoded = sszSerializer.encode2(expected); IndexedAttestation reconstructed = sszSerializer.decode(encoded, IndexedAttestation.class); assertEquals(expected, reconstructed); @@ -252,7 +137,7 @@ public void slashableAttestationTest() { @Test public void attesterSlashingTest() { - AttesterSlashing expected = createAttesterSlashings(); + AttesterSlashing expected = dataFactory.createAttesterSlashings(); BytesValue encoded = sszSerializer.encode2(expected); AttesterSlashing reconstructed = sszSerializer.decode(encoded, AttesterSlashing.class); assertEquals(expected, reconstructed); @@ -260,41 +145,23 @@ public void attesterSlashingTest() { @Test public void beaconBlockBodyTest() { - BeaconBlockBody expected = createBeaconBlockBody(); + BeaconBlockBody expected = dataFactory.createBeaconBlockBody(); BytesValue encoded = sszSerializer.encode2(expected); BeaconBlockBody reconstructed = sszSerializer.decode(encoded, BeaconBlockBody.class); assertEquals(expected, reconstructed); } - private BeaconBlock createBeaconBlock() { - BeaconBlock beaconBlock = - new BeaconBlock( - SlotNumber.castFrom(UInt64.MAX_VALUE), - Hashes.sha256(BytesValue.fromHexString("aa")), - Hashes.sha256(BytesValue.fromHexString("bb")), - createBeaconBlockBody(), - BLSSignature.wrap(Bytes96.fromHexString("aa"))); - - return beaconBlock; - } - @Test public void beaconBlockTest() { - BeaconBlock expected = createBeaconBlock(); + BeaconBlock expected = dataFactory.createBeaconBlock(); BytesValue encoded = sszSerializer.encode2(expected); BeaconBlock reconstructed = sszSerializer.decode(encoded, BeaconBlock.class); assertEquals(expected, reconstructed); } - private BeaconState createBeaconState() { - BeaconState beaconState = BeaconState.getEmpty(); - - return beaconState; - } - @Test public void beaconStateTest() { - BeaconState expected = createBeaconState(); + BeaconState expected = dataFactory.createBeaconState(); BytesValue encoded = sszSerializer.encode2(expected); BeaconState reconstructed = sszSerializer.decode(encoded, BeaconStateImpl.class); assertEquals(expected, reconstructed); @@ -302,94 +169,49 @@ public void beaconStateTest() { @Test public void beaconStateExTest() { - BeaconState expected = createBeaconState(); + BeaconState expected = dataFactory.createBeaconState(); BeaconStateEx stateEx = new BeaconStateExImpl(expected); BytesValue encoded = sszSerializer.encode2(stateEx); BeaconState reconstructed = sszSerializer.decode(encoded, BeaconStateImpl.class); assertEquals(expected, reconstructed); } - private Crosslink createCrosslink() { - Crosslink crosslink = Crosslink.EMPTY; - - return crosslink; - } - @Test public void crosslinkTest() { - Crosslink expected = createCrosslink(); + Crosslink expected = dataFactory.createCrosslink(); BytesValue encoded = sszSerializer.encode2(expected); Crosslink reconstructed = sszSerializer.decode(encoded, Crosslink.class); assertEquals(expected, reconstructed); } - private Eth1DataVote createEth1DataVote() { - Eth1DataVote eth1DataVote = new Eth1DataVote(Eth1Data.EMPTY, UInt64.MAX_VALUE); - - return eth1DataVote; - } - @Test public void eth1DataVoteTest() { - Eth1DataVote expected = createEth1DataVote(); + Eth1DataVote expected = dataFactory.createEth1DataVote(); BytesValue encoded = sszSerializer.encode2(expected); Eth1DataVote reconstructed = sszSerializer.decode(encoded, Eth1DataVote.class); assertEquals(expected, reconstructed); } - private Fork createFork() { - Fork fork = Fork.EMPTY; - - return fork; - } - @Test public void forkTest() { - Fork expected = createFork(); + Fork expected = dataFactory.createFork(); BytesValue encoded = sszSerializer.encode2(expected); Fork reconstructed = sszSerializer.decode(encoded, Fork.class); assertEquals(expected, reconstructed); } - private PendingAttestation createPendingAttestation() { - PendingAttestation pendingAttestation = - new PendingAttestation( - Bitfield.of(BytesValue.fromHexString("aa")), - createAttestationData(), - SlotNumber.ZERO, - ValidatorIndex.ZERO); - - return pendingAttestation; - } - @Test public void pendingAttestationTest() { - PendingAttestation expected = createPendingAttestation(); + PendingAttestation expected = dataFactory.createPendingAttestation(); BytesValue encoded = sszSerializer.encode2(expected); PendingAttestation reconstructed = sszSerializer.decode(encoded, PendingAttestation.class); assertEquals(expected, reconstructed); } - private ValidatorRecord createValidatorRecord() { - ValidatorRecord validatorRecord = - ValidatorRecord.Builder.fromDepositData(createDepositData()) - .withPubKey(BLSPubkey.ZERO) - .withWithdrawalCredentials(Hash32.ZERO) - .withActivationEligibilityEpoch(EpochNumber.ZERO) - .withActivationEpoch(EpochNumber.ZERO) - .withExitEpoch(EpochNumber.ZERO) - .withWithdrawableEpoch(EpochNumber.ZERO) - .withSlashed(Boolean.FALSE) - .withEffectiveBalance(Gwei.ZERO) - .build(); - - return validatorRecord; - } - @Test public void validatorRecordTest() { - ValidatorRecord expected = createValidatorRecord(); + ValidatorRecord expected = dataFactory.createValidatorRecord(); BytesValue encoded = sszSerializer.encode2(expected); ValidatorRecord reconstructed = sszSerializer.decode(encoded, ValidatorRecord.class); assertEquals(expected, reconstructed); diff --git a/core/src/test/java/org/ethereum/beacon/core/util/TestDataFactory.java b/core/src/test/java/org/ethereum/beacon/core/util/TestDataFactory.java new file mode 100644 index 000000000..20675c5b2 --- /dev/null +++ b/core/src/test/java/org/ethereum/beacon/core/util/TestDataFactory.java @@ -0,0 +1,245 @@ +package org.ethereum.beacon.core.util; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.BeaconBlockBody; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.operations.Deposit; +import org.ethereum.beacon.core.operations.ProposerSlashing; +import org.ethereum.beacon.core.operations.Transfer; +import org.ethereum.beacon.core.operations.VoluntaryExit; +import org.ethereum.beacon.core.operations.attestation.AttestationData; +import org.ethereum.beacon.core.operations.attestation.Crosslink; +import org.ethereum.beacon.core.operations.deposit.DepositData; +import org.ethereum.beacon.core.operations.slashing.AttesterSlashing; +import org.ethereum.beacon.core.operations.slashing.IndexedAttestation; +import org.ethereum.beacon.core.spec.SpecConstants; +import org.ethereum.beacon.core.state.Eth1Data; +import org.ethereum.beacon.core.state.Eth1DataVote; +import org.ethereum.beacon.core.state.Fork; +import org.ethereum.beacon.core.state.PendingAttestation; +import org.ethereum.beacon.core.state.ValidatorRecord; +import org.ethereum.beacon.core.types.BLSPubkey; +import org.ethereum.beacon.core.types.BLSSignature; +import org.ethereum.beacon.core.types.Bitfield; +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.Gwei; +import org.ethereum.beacon.core.types.ShardNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.core.types.ValidatorIndex; +import org.ethereum.beacon.crypto.Hashes; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes48; +import tech.pegasys.artemis.util.bytes.Bytes96; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.collections.ReadVector; +import tech.pegasys.artemis.util.uint.UInt64; + + +public class TestDataFactory { + private SpecConstants specConstants; + + public TestDataFactory() { + this(BeaconChainSpec.DEFAULT_CONSTANTS); + } + + public TestDataFactory(SpecConstants specConstants) { + this.specConstants = specConstants; + } + + public AttestationData createAttestationData() { + AttestationData expected = + new AttestationData( + Hashes.sha256(BytesValue.fromHexString("aa")), + EpochNumber.ZERO, + Hashes.sha256(BytesValue.fromHexString("bb")), + EpochNumber.of(123), + Hashes.sha256(BytesValue.fromHexString("cc")), + ShardNumber.of(345), + Hashes.sha256(BytesValue.fromHexString("dd")), + Hash32.ZERO); + + return expected; + } + + public Attestation createAttestation() { + return createAttestation(BytesValue.fromHexString("aa")); + } + + public Attestation createAttestation(BytesValue someValue) { + AttestationData attestationData = createAttestationData(); + Attestation attestation = + new Attestation( + Bitfield.of(someValue), + attestationData, + Bitfield.of(BytesValue.fromHexString("bb")), + BLSSignature.wrap(Bytes96.fromHexString("cc"))); + + return attestation; + } + + public DepositData createDepositData() { + DepositData depositData = + new DepositData( + BLSPubkey.wrap(Bytes48.TRUE), + Hashes.sha256(BytesValue.fromHexString("aa")), + Gwei.ZERO, BLSSignature.wrap(Bytes96.ZERO)); + + return depositData; + } + + public Deposit createDeposit1() { + Deposit deposit = new Deposit( + ReadVector.wrap( + Collections.nCopies(specConstants.getDepositContractTreeDepth().getIntValue(), Hash32.ZERO), Integer::new), + UInt64.ZERO, createDepositData()); + + return deposit; + } + + public Deposit createDeposit2() { + ArrayList hashes = new ArrayList<>(); + hashes.add(Hashes.sha256(BytesValue.fromHexString("aa"))); + hashes.add(Hashes.sha256(BytesValue.fromHexString("bb"))); + hashes.addAll(Collections.nCopies(specConstants.getDepositContractTreeDepth().getIntValue() - hashes.size(), Hash32.ZERO)); + ReadVector proof = ReadVector.wrap(hashes, Integer::new); + Deposit deposit = new Deposit(proof, UInt64.ZERO, createDepositData()); + + return deposit; + } + + public VoluntaryExit createExit() { + VoluntaryExit voluntaryExit = new VoluntaryExit(EpochNumber.of(123), ValidatorIndex.MAX, BLSSignature.wrap(Bytes96.fromHexString("aa"))); + + return voluntaryExit; + } + + public ProposerSlashing createProposerSlashing(Random random) { + ProposerSlashing proposerSlashing = + new ProposerSlashing( + ValidatorIndex.MAX, + BeaconBlockTestUtil.createRandomHeader(random), + BeaconBlockTestUtil.createRandomHeader(random)); + + return proposerSlashing; + } + + public BeaconBlockBody createBeaconBlockBody() { + Random random = new Random(1); + List proposerSlashings = new ArrayList<>(); + proposerSlashings.add(createProposerSlashing(random)); + List attesterSlashings = new ArrayList<>(); + attesterSlashings.add(createAttesterSlashings()); + attesterSlashings.add(createAttesterSlashings()); + List attestations = new ArrayList<>(); + attestations.add(createAttestation()); + List deposits = new ArrayList<>(); + deposits.add(createDeposit1()); + deposits.add(createDeposit2()); + List voluntaryExits = new ArrayList<>(); + voluntaryExits.add(createExit()); + List transfers = new ArrayList<>(); + BeaconBlockBody beaconBlockBody = + BeaconBlockBody.create( + BLSSignature.ZERO, + new Eth1Data(Hash32.ZERO, UInt64.ZERO, Hash32.ZERO), + Bytes32.ZERO, + proposerSlashings, + attesterSlashings, + attestations, + deposits, + voluntaryExits, + transfers + ); + + return beaconBlockBody; + } + + public AttesterSlashing createAttesterSlashings() { + return new AttesterSlashing( + createSlashableAttestation(), + createSlashableAttestation()); + } + + public IndexedAttestation createSlashableAttestation() { + return new IndexedAttestation( + Arrays.asList(ValidatorIndex.of(234), ValidatorIndex.of(235)), + Arrays.asList(ValidatorIndex.of(678), ValidatorIndex.of(679)), + createAttestationData(), + BLSSignature.wrap(Bytes96.fromHexString("aa"))); + } + + public BeaconBlock createBeaconBlock() { + return createBeaconBlock(BytesValue.fromHexString("aa")); + } + + public BeaconBlock createBeaconBlock(BytesValue someValue) { + BeaconBlock beaconBlock = + new BeaconBlock( + SlotNumber.castFrom(UInt64.MAX_VALUE), + Hashes.sha256(someValue), + Hashes.sha256(BytesValue.fromHexString("bb")), + createBeaconBlockBody(), + BLSSignature.wrap(Bytes96.fromHexString("aa"))); + + return beaconBlock; + } + + public BeaconState createBeaconState() { + BeaconState beaconState = BeaconState.getEmpty(); + + return beaconState; + } + + public Crosslink createCrosslink() { + Crosslink crosslink = Crosslink.EMPTY; + + return crosslink; + } + + public Eth1DataVote createEth1DataVote() { + Eth1DataVote eth1DataVote = new Eth1DataVote(Eth1Data.EMPTY, UInt64.MAX_VALUE); + + return eth1DataVote; + } + + public Fork createFork() { + Fork fork = Fork.EMPTY; + + return fork; + } + + public PendingAttestation createPendingAttestation() { + PendingAttestation pendingAttestation = + new PendingAttestation( + Bitfield.of(BytesValue.fromHexString("aa")), + createAttestationData(), + SlotNumber.ZERO, + ValidatorIndex.ZERO); + + return pendingAttestation; + } + + public ValidatorRecord createValidatorRecord() { + ValidatorRecord validatorRecord = + ValidatorRecord.Builder.fromDepositData(createDepositData()) + .withPubKey(BLSPubkey.ZERO) + .withWithdrawalCredentials(Hash32.ZERO) + .withActivationEligibilityEpoch(EpochNumber.ZERO) + .withActivationEpoch(EpochNumber.ZERO) + .withExitEpoch(EpochNumber.ZERO) + .withWithdrawableEpoch(EpochNumber.ZERO) + .withSlashed(Boolean.FALSE) + .withEffectiveBalance(Gwei.ZERO) + .build(); + + return validatorRecord; + } +} diff --git a/settings.gradle b/settings.gradle index 3f66e2686..b83a97b7d 100644 --- a/settings.gradle +++ b/settings.gradle @@ -25,6 +25,8 @@ include 'ssz' include 'start:simulator' // Benchmarks CLI include 'start:benchmaker' +// Beacon node +include 'start:node' // Clients helpers include 'start:common' // Configuration parser diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/incremental/ObservableCompositeHelper.java b/ssz/src/main/java/org/ethereum/beacon/ssz/incremental/ObservableCompositeHelper.java index 025b74350..e77fd1f1d 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/incremental/ObservableCompositeHelper.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/incremental/ObservableCompositeHelper.java @@ -57,6 +57,11 @@ public UpdateListener fork() { public C get() { return value; } + + @Override + public String toString() { + return value == null ? "null" : value.toString(); + } } private Map listeners; diff --git a/start/benchmaker/src/main/java/org/ethereum/beacon/benchmaker/BenchmarkRunner.java b/start/benchmaker/src/main/java/org/ethereum/beacon/benchmaker/BenchmarkRunner.java index a79484435..a84a41e18 100644 --- a/start/benchmaker/src/main/java/org/ethereum/beacon/benchmaker/BenchmarkRunner.java +++ b/start/benchmaker/src/main/java/org/ethereum/beacon/benchmaker/BenchmarkRunner.java @@ -1,19 +1,13 @@ package org.ethereum.beacon.benchmaker; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.time.Duration; import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.Random; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.ethereum.beacon.Launcher; import org.ethereum.beacon.bench.BenchmarkController; import org.ethereum.beacon.bench.BenchmarkReport; import org.ethereum.beacon.bench.BenchmarkUtils; @@ -33,19 +27,16 @@ import org.ethereum.beacon.crypto.util.BlsKeyPairReader; import org.ethereum.beacon.pow.DepositContract; import org.ethereum.beacon.schedulers.ControlledSchedulers; -import org.ethereum.beacon.schedulers.LoggerMDCExecutor; -import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.schedulers.TimeController; -import org.ethereum.beacon.schedulers.TimeControllerImpl; -import org.ethereum.beacon.util.SimulateUtils; +import org.ethereum.beacon.start.common.Launcher; +import org.ethereum.beacon.start.common.util.MDCControlledSchedulers; +import org.ethereum.beacon.start.common.util.SimpleDepositContract; +import org.ethereum.beacon.start.common.util.SimulateUtils; import org.ethereum.beacon.util.stats.MeasurementsCollector; import org.ethereum.beacon.validator.crypto.BLS381Credentials; import org.ethereum.beacon.wire.LocalWireHub; -import org.ethereum.beacon.wire.WireApi; +import org.ethereum.beacon.wire.WireApiSub; import org.javatuples.Pair; -import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.uint.UInt64; @@ -113,7 +104,7 @@ public void run() { logger.info("Bootstrapping validators ..."); ControlledSchedulers schedulers = controlledSchedulers.createNew("V0"); - WireApi wireApi = localWireHub.createNewPeer("0"); + WireApiSub wireApi = localWireHub.createNewPeer("0"); List blsCreds; if (spec.isBlsVerify()) { @@ -235,79 +226,4 @@ static Stats createFrom(MeasurementsCollector collector) { return stats; } } - - private static class SimpleDepositContract implements DepositContract { - private final ChainStart chainStart; - - public SimpleDepositContract(ChainStart chainStart) { - this.chainStart = chainStart; - } - - @Override - public Publisher getChainStartMono() { - return Mono.just(chainStart); - } - - @Override - public Publisher getDepositStream() { - return Mono.empty(); - } - - @Override - public List peekDeposits( - int maxCount, Eth1Data fromDepositExclusive, Eth1Data tillDepositInclusive) { - return Collections.emptyList(); - } - - @Override - public boolean hasDepositRoot(Hash32 blockHash, Hash32 depositRoot) { - return true; - } - - @Override - public Optional getLatestEth1Data() { - return Optional.of(chainStart.getEth1Data()); - } - - @Override - public void setDistanceFromHead(long distanceFromHead) {} - } - - public static class MDCControlledSchedulers { - private DateFormat localTimeFormat = new SimpleDateFormat("HH:mm:ss.SSS"); - - private TimeController timeController = new TimeControllerImpl(); - - public ControlledSchedulers createNew(String validatorId) { - return createNew(validatorId, 0); - } - - public ControlledSchedulers createNew(String validatorId, long timeShift) { - ControlledSchedulers[] newSched = new ControlledSchedulers[1]; - LoggerMDCExecutor mdcExecutor = new LoggerMDCExecutor() - .add("validatorTime", () -> localTimeFormat.format(new Date(newSched[0].getCurrentTime()))) - .add("validatorIndex", () -> "" + validatorId); - newSched[0] = Schedulers.createControlled(() -> mdcExecutor); - newSched[0].getTimeController().setParent(timeController); - newSched[0].getTimeController().setTimeShift(timeShift); - - return newSched[0]; - } - - public void setCurrentTime(long time) { - timeController.setTime(time); - } - - void addTime(Duration duration) { - addTime(duration.toMillis()); - } - - void addTime(long millis) { - setCurrentTime(timeController.getTime() + millis); - } - - public long getCurrentTime() { - return timeController.getTime(); - } - } } diff --git a/start/common/build.gradle b/start/common/build.gradle index 3abb82135..e83a85628 100644 --- a/start/common/build.gradle +++ b/start/common/build.gradle @@ -6,6 +6,7 @@ dependencies { implementation project(':db:core') implementation project(':chain') implementation project(':pow:core') + implementation project(':ssz') implementation project(':validator') implementation project(':wire') implementation project(':util') @@ -22,4 +23,5 @@ dependencies { testImplementation project(':consensus').sourceSets.test.output testImplementation project(':chain').sourceSets.test.output testImplementation project(':pow:core').sourceSets.test.output + testImplementation project(':start:simulator') } diff --git a/start/common/src/main/java/org/ethereum/beacon/Launcher.java b/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java similarity index 98% rename from start/common/src/main/java/org/ethereum/beacon/Launcher.java rename to start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java index 2ee2e0590..439c86a91 100644 --- a/start/common/src/main/java/org/ethereum/beacon/Launcher.java +++ b/start/common/src/main/java/org/ethereum/beacon/start/common/Launcher.java @@ -1,4 +1,4 @@ -package org.ethereum.beacon; +package org.ethereum.beacon.start.common; import java.util.List; import org.ethereum.beacon.bench.BenchmarkController; @@ -37,7 +37,7 @@ import org.ethereum.beacon.validator.attester.BeaconChainAttesterImpl; import org.ethereum.beacon.validator.crypto.BLS381Credentials; import org.ethereum.beacon.validator.proposer.BeaconChainProposerImpl; -import org.ethereum.beacon.wire.WireApi; +import org.ethereum.beacon.wire.WireApiSub; import reactor.core.publisher.DirectProcessor; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -46,7 +46,7 @@ public class Launcher { private final BeaconChainSpec spec; private final DepositContract depositContract; private final List validatorCred; - private final WireApi wireApi; + private final WireApiSub wireApi; private final BeaconChainStorageFactory storageFactory; private final Schedulers schedulers; @@ -79,7 +79,7 @@ public Launcher( BeaconChainSpec spec, DepositContract depositContract, List validatorCred, - WireApi wireApi, + WireApiSub wireApi, BeaconChainStorageFactory storageFactory, Schedulers schedulers) { this(spec, depositContract, validatorCred, wireApi, storageFactory, schedulers, @@ -90,7 +90,7 @@ public Launcher( BeaconChainSpec spec, DepositContract depositContract, List validatorCred, - WireApi wireApi, + WireApiSub wireApi, BeaconChainStorageFactory storageFactory, Schedulers schedulers, BenchmarkController benchmarkController) { @@ -264,7 +264,7 @@ public List getValidatorCred() { return validatorCred; } - public WireApi getWireApi() { + public WireApiSub getWireApi() { return wireApi; } diff --git a/start/common/src/main/java/org/ethereum/beacon/start/common/NodeLauncher.java b/start/common/src/main/java/org/ethereum/beacon/start/common/NodeLauncher.java new file mode 100644 index 000000000..31e3a6475 --- /dev/null +++ b/start/common/src/main/java/org/ethereum/beacon/start/common/NodeLauncher.java @@ -0,0 +1,321 @@ +package org.ethereum.beacon.start.common; + +import java.time.Duration; +import java.util.List; +import org.ethereum.beacon.chain.DefaultBeaconChain; +import org.ethereum.beacon.chain.MutableBeaconChain; +import org.ethereum.beacon.chain.ProposedBlockProcessor; +import org.ethereum.beacon.chain.ProposedBlockProcessorImpl; +import org.ethereum.beacon.chain.SlotTicker; +import org.ethereum.beacon.chain.observer.ObservableStateProcessor; +import org.ethereum.beacon.chain.observer.ObservableStateProcessorImpl; +import org.ethereum.beacon.chain.storage.BeaconChainStorage; +import org.ethereum.beacon.chain.storage.BeaconChainStorageFactory; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.transition.EmptySlotTransition; +import org.ethereum.beacon.consensus.transition.ExtendedSlotTransition; +import org.ethereum.beacon.consensus.transition.InitialStateTransition; +import org.ethereum.beacon.consensus.transition.PerBlockTransition; +import org.ethereum.beacon.consensus.transition.PerEpochTransition; +import org.ethereum.beacon.consensus.transition.PerSlotTransition; +import org.ethereum.beacon.consensus.transition.StateCachingTransition; +import org.ethereum.beacon.consensus.verifier.BeaconBlockVerifier; +import org.ethereum.beacon.consensus.verifier.BeaconStateVerifier; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.db.InMemoryDatabase; +import org.ethereum.beacon.pow.DepositContract; +import org.ethereum.beacon.pow.DepositContract.ChainStart; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.ssz.SSZBuilder; +import org.ethereum.beacon.ssz.SSZSerializer; +import org.ethereum.beacon.validator.BeaconChainProposer; +import org.ethereum.beacon.validator.MultiValidatorService; +import org.ethereum.beacon.validator.attester.BeaconChainAttesterImpl; +import org.ethereum.beacon.validator.crypto.BLS381Credentials; +import org.ethereum.beacon.validator.proposer.BeaconChainProposerImpl; +import org.ethereum.beacon.wire.Feedback; +import org.ethereum.beacon.wire.MessageSerializer; +import org.ethereum.beacon.wire.SimplePeerManagerImpl; +import org.ethereum.beacon.wire.WireApiSub; +import org.ethereum.beacon.wire.WireApiSync; +import org.ethereum.beacon.wire.WireApiSyncServer; +import org.ethereum.beacon.wire.message.SSZMessageSerializer; +import org.ethereum.beacon.wire.net.ConnectionManager; +import org.ethereum.beacon.wire.sync.BeaconBlockTree; +import org.ethereum.beacon.wire.sync.SyncManagerImpl; +import org.ethereum.beacon.wire.sync.SyncQueue; +import org.ethereum.beacon.wire.sync.SyncQueueImpl; +import reactor.core.publisher.DirectProcessor; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import tech.pegasys.artemis.util.uint.UInt64; + +public class NodeLauncher { + private final BeaconChainSpec spec; + private final DepositContract depositContract; + private final List validatorCred; + private final BeaconChainStorageFactory storageFactory; + private final Schedulers schedulers; + + private InitialStateTransition initialTransition; + private StateCachingTransition stateCachingTransition; + private PerSlotTransition perSlotTransition; + private PerBlockTransition perBlockTransition; + private PerEpochTransition perEpochTransition; + private ExtendedSlotTransition extendedSlotTransition; + private EmptySlotTransition emptySlotTransition; + private BeaconBlockVerifier blockVerifier; + private BeaconStateVerifier stateVerifier; + + private InMemoryDatabase db; + private BeaconChainStorage beaconChainStorage; + private MutableBeaconChain beaconChain; + private SlotTicker slotTicker; + private ObservableStateProcessor observableStateProcessor; + private BeaconChainProposer beaconChainProposer; + private BeaconChainAttesterImpl beaconChainAttester; + private MultiValidatorService beaconChainValidator; + + private byte networkId = 1; + private UInt64 chainId = UInt64.valueOf(1); + private boolean startSyncManager = false; + + private WireApiSub wireApiSub; + private WireApiSync wireApiSyncRemote; + private final ConnectionManager connectionManager; + private SimplePeerManagerImpl peerManager; + private BeaconBlockTree blockTree; + private SyncQueue syncQueue; + private SyncManagerImpl syncManager; + private WireApiSyncServer syncServer; + + public NodeLauncher( + BeaconChainSpec spec, + DepositContract depositContract, + List validatorCred, + ConnectionManager connectionManager, + BeaconChainStorageFactory storageFactory, + Schedulers schedulers, + boolean startSyncManager) { + + this.spec = spec; + this.depositContract = depositContract; + this.validatorCred = validatorCred; + this.connectionManager = connectionManager; + this.storageFactory = storageFactory; + this.schedulers = schedulers; + this.startSyncManager = startSyncManager; + + if (depositContract != null) { + Mono.from(depositContract.getChainStartMono()).subscribe(this::chainStarted); + } + } + + void chainStarted(ChainStart chainStartEvent) { + initialTransition = new InitialStateTransition(chainStartEvent, spec); + stateCachingTransition = new StateCachingTransition(spec); + perSlotTransition = new PerSlotTransition(spec); + perBlockTransition = new PerBlockTransition(spec); + perEpochTransition = new PerEpochTransition(spec); + extendedSlotTransition = + new ExtendedSlotTransition( + stateCachingTransition, perEpochTransition, perSlotTransition, spec); + emptySlotTransition = new EmptySlotTransition(extendedSlotTransition); + + db = new InMemoryDatabase(); + beaconChainStorage = storageFactory.create(db); + + blockVerifier = BeaconBlockVerifier.createDefault(spec); + stateVerifier = BeaconStateVerifier.createDefault(spec); + + beaconChain = + new DefaultBeaconChain( + spec, + initialTransition, + emptySlotTransition, + perBlockTransition, + blockVerifier, + stateVerifier, + beaconChainStorage, + schedulers); + beaconChain.init(); + + slotTicker = + new SlotTicker(spec, beaconChain.getRecentlyProcessed().getState(), schedulers); + slotTicker.start(); + + DirectProcessor allAttestations = DirectProcessor.create(); + + observableStateProcessor = new ObservableStateProcessorImpl( + beaconChainStorage, + slotTicker.getTickerStream(), + allAttestations, + beaconChain.getBlockStatesStream(), + spec, + emptySlotTransition, + schedulers); + observableStateProcessor.start(); + + SSZSerializer ssz = new SSZBuilder().buildSerializer(); + MessageSerializer messageSerializer = new SSZMessageSerializer(ssz); + syncServer = new WireApiSyncServer(beaconChainStorage); + + peerManager = new SimplePeerManagerImpl( + networkId, + chainId, + connectionManager.channelsStream(), + ssz, + spec, + messageSerializer, + schedulers, + syncServer, + beaconChain.getBlockStatesStream()); + + wireApiSub = peerManager.getWireApiSub(); + wireApiSyncRemote = peerManager.getWireApiSync(); + + blockTree = new BeaconBlockTree(spec.getObjectHasher()); + syncQueue = new SyncQueueImpl(blockTree); + + Flux ownBlocks = Flux.empty(); + if (validatorCred != null) { + beaconChainProposer = new BeaconChainProposerImpl(spec, perBlockTransition, depositContract); + beaconChainAttester = new BeaconChainAttesterImpl(spec); + + beaconChainValidator = new MultiValidatorService( + validatorCred, + beaconChainProposer, + beaconChainAttester, + spec, + observableStateProcessor.getObservableStateStream(), + schedulers); + beaconChainValidator.start(); + + ProposedBlockProcessor proposedBlocksProcessor = new ProposedBlockProcessorImpl( + beaconChain, schedulers); + Flux.from(beaconChainValidator.getProposedBlocksStream()) + .subscribe(proposedBlocksProcessor::newBlockProposed); + Flux.from(proposedBlocksProcessor.processedBlocksStream()) + .subscribe(wireApiSub::sendProposedBlock); + + Flux.from(beaconChainValidator.getAttestationsStream()).subscribe(wireApiSub::sendAttestation); + Flux.from(beaconChainValidator.getAttestationsStream()).subscribe(allAttestations); + + ownBlocks = Flux.from(proposedBlocksProcessor.processedBlocksStream()); + } + + Flux.from(wireApiSub.inboundAttestationsStream()) + .publishOn(schedulers.reactorEvents()) + .subscribe(allAttestations); + + Flux allNewBlocks = Flux.merge(ownBlocks, wireApiSub.inboundBlocksStream()); + syncManager = new SyncManagerImpl( + beaconChain, + allNewBlocks.map(Feedback::of), + beaconChainStorage, + spec, + wireApiSyncRemote, + syncQueue, + 1, + schedulers.reactorEvents()); + syncManager.setRequestsDelay(Duration.ofSeconds(1), Duration.ofSeconds(5)); + + if (startSyncManager) { + syncManager.start(); + } + +// Flux.from(wireApiSub.inboundBlocksStream()) +// .publishOn(schedulers.reactorEvents()) +// .subscribe(beaconChain::insert); + } + + + public BeaconChainSpec getSpec() { + return spec; + } + + public DepositContract getDepositContract() { + return depositContract; + } + + public List getValidatorCred() { + return validatorCred; + } + + public WireApiSub getWireApiSub() { + return wireApiSub; + } + + public InitialStateTransition getInitialTransition() { + return initialTransition; + } + + public PerSlotTransition getPerSlotTransition() { + return perSlotTransition; + } + + public PerBlockTransition getPerBlockTransition() { + return perBlockTransition; + } + + public PerEpochTransition getPerEpochTransition() { + return perEpochTransition; + } + + public ExtendedSlotTransition getExtendedSlotTransition() { + return extendedSlotTransition; + } + + public BeaconBlockVerifier getBlockVerifier() { + return blockVerifier; + } + + public BeaconStateVerifier getStateVerifier() { + return stateVerifier; + } + + public InMemoryDatabase getDb() { + return db; + } + + public BeaconChainStorage getBeaconChainStorage() { + return beaconChainStorage; + } + + public MutableBeaconChain getBeaconChain() { + return beaconChain; + } + + public SlotTicker getSlotTicker() { + return slotTicker; + } + + public ObservableStateProcessor getObservableStateProcessor() { + return observableStateProcessor; + } + + public BeaconChainProposer getBeaconChainProposer() { + return beaconChainProposer; + } + + public BeaconChainAttesterImpl getBeaconChainAttester() { + return beaconChainAttester; + } + + public MultiValidatorService getValidatorService() { + return beaconChainValidator; + } + + public BeaconChainStorageFactory getStorageFactory() { + return storageFactory; + } + + public Schedulers getSchedulers() { + return schedulers; + } + + public SyncManagerImpl getSyncManager() { + return syncManager; + } +} diff --git a/start/common/src/main/java/org/ethereum/beacon/start/common/util/MDCControlledSchedulers.java b/start/common/src/main/java/org/ethereum/beacon/start/common/util/MDCControlledSchedulers.java new file mode 100644 index 000000000..f079427a5 --- /dev/null +++ b/start/common/src/main/java/org/ethereum/beacon/start/common/util/MDCControlledSchedulers.java @@ -0,0 +1,49 @@ +package org.ethereum.beacon.start.common.util; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.time.Duration; +import java.util.Date; +import org.ethereum.beacon.schedulers.ControlledSchedulers; +import org.ethereum.beacon.schedulers.LoggerMDCExecutor; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.schedulers.TimeController; +import org.ethereum.beacon.schedulers.TimeControllerImpl; + +public class MDCControlledSchedulers { + private DateFormat localTimeFormat = new SimpleDateFormat("HH:mm:ss.SSS"); + + private TimeController timeController = new TimeControllerImpl(); + + public ControlledSchedulers createNew(String validatorId) { + return createNew(validatorId, 0); + } + + public ControlledSchedulers createNew(String validatorId, long timeShift) { + ControlledSchedulers[] newSched = new ControlledSchedulers[1]; + LoggerMDCExecutor mdcExecutor = new LoggerMDCExecutor() + .add("validatorTime", () -> localTimeFormat.format(new Date(newSched[0].getCurrentTime()))) + .add("validatorIndex", () -> "" + validatorId); + newSched[0] = Schedulers.createControlled(() -> mdcExecutor); + newSched[0].getTimeController().setParent(timeController); + newSched[0].getTimeController().setTimeShift(timeShift); + + return newSched[0]; + } + + public void setCurrentTime(long time) { + timeController.setTime(time); + } + + public void addTime(Duration duration) { + addTime(duration.toMillis()); + } + + public void addTime(long millis) { + setCurrentTime(timeController.getTime() + millis); + } + + public long getCurrentTime() { + return timeController.getTime(); + } +} diff --git a/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimpleDepositContract.java b/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimpleDepositContract.java new file mode 100644 index 000000000..d3f4084a1 --- /dev/null +++ b/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimpleDepositContract.java @@ -0,0 +1,48 @@ +package org.ethereum.beacon.start.common.util; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import org.ethereum.beacon.core.operations.Deposit; +import org.ethereum.beacon.core.state.Eth1Data; +import org.ethereum.beacon.pow.DepositContract; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Mono; +import tech.pegasys.artemis.ethereum.core.Hash32; + +public class SimpleDepositContract implements DepositContract { + private final ChainStart chainStart; + + public SimpleDepositContract(ChainStart chainStart) { + this.chainStart = chainStart; + } + + @Override + public Publisher getChainStartMono() { + return Mono.just(chainStart); + } + + @Override + public Publisher getDepositStream() { + return Mono.empty(); + } + + @Override + public List peekDeposits( + int maxCount, Eth1Data fromDepositExclusive, Eth1Data tillDepositInclusive) { + return Collections.emptyList(); + } + + @Override + public boolean hasDepositRoot(Hash32 blockHash, Hash32 depositRoot) { + return true; + } + + @Override + public Optional getLatestEth1Data() { + return Optional.of(chainStart.getEth1Data()); + } + + @Override + public void setDistanceFromHead(long distanceFromHead) {} +} diff --git a/start/common/src/main/java/org/ethereum/beacon/util/SimulateUtils.java b/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulateUtils.java similarity index 78% rename from start/common/src/main/java/org/ethereum/beacon/util/SimulateUtils.java rename to start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulateUtils.java index e43161517..960e3c040 100644 --- a/start/common/src/main/java/org/ethereum/beacon/util/SimulateUtils.java +++ b/start/common/src/main/java/org/ethereum/beacon/start/common/util/SimulateUtils.java @@ -1,4 +1,4 @@ -package org.ethereum.beacon.util; +package org.ethereum.beacon.start.common.util; import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.core.operations.Deposit; @@ -6,6 +6,7 @@ import org.ethereum.beacon.core.spec.SignatureDomains; import org.ethereum.beacon.core.types.BLSPubkey; import org.ethereum.beacon.core.types.BLSSignature; +import org.ethereum.beacon.core.types.Gwei; import org.ethereum.beacon.crypto.BLS381; import org.ethereum.beacon.crypto.BLS381.PrivateKey; import org.ethereum.beacon.crypto.MessageParameters; @@ -42,14 +43,25 @@ public static synchronized Pair, List> getAnyDepos cachedDeposits.getValue0().subList(0, count), cachedDeposits.getValue1().subList(0, count)); } - public static synchronized Deposit getDepositForKeyPair(UInt64 depositIndex, Random rnd, + public static Deposit getDepositForKeyPair(UInt64 depositIndex, Random rnd, BLS381.KeyPair keyPair, BeaconChainSpec spec, boolean isProofVerifyEnabled) { + return getDepositForKeyPair( + depositIndex, + rnd, + keyPair, + spec, + spec.getConstants().getMaxEffectiveBalance(), + isProofVerifyEnabled); + } + + public static synchronized Deposit getDepositForKeyPair(UInt64 depositIndex, Random rnd, + BLS381.KeyPair keyPair, BeaconChainSpec spec, Gwei initBalance, boolean isProofVerifyEnabled) { Hash32 withdrawalCredentials = Hash32.random(rnd); DepositData depositDataWithoutSignature = new DepositData( BLSPubkey.wrap(Bytes48.leftPad(keyPair.getPublic().getEncodedBytes())), withdrawalCredentials, - spec.getConstants().getMaxEffectiveBalance(), + initBalance, BLSSignature.wrap(Bytes96.ZERO)); BLSSignature signature = BLSSignature.ZERO; @@ -75,12 +87,33 @@ public static synchronized Deposit getDepositForKeyPair(UInt64 depositIndex, Ran } public static synchronized List getDepositsForKeyPairs( - UInt64 startIndex, Random rnd, List keyPairs, BeaconChainSpec spec, boolean isProofVerifyEnabled) { + UInt64 startIndex, + Random rnd, + List keyPairs, + BeaconChainSpec spec, + boolean isProofVerifyEnabled) { + return getDepositsForKeyPairs( + startIndex, + rnd, + keyPairs, + spec, + spec.getConstants().getMaxEffectiveBalance(), + isProofVerifyEnabled); + } + + public static synchronized List getDepositsForKeyPairs( + UInt64 startIndex, + Random rnd, + List keyPairs, + BeaconChainSpec spec, + Gwei initBalance, + boolean isProofVerifyEnabled) { + List deposits = new ArrayList<>(); UInt64 index = startIndex; for (BLS381.KeyPair keyPair : keyPairs) { - deposits.add(getDepositForKeyPair(index, rnd, keyPair, spec, isProofVerifyEnabled)); + deposits.add(getDepositForKeyPair(index, rnd, keyPair, spec, initBalance, isProofVerifyEnabled)); index = index.increment(); } diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/ConfigException.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/ConfigException.java new file mode 100644 index 000000000..96e31f82b --- /dev/null +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/ConfigException.java @@ -0,0 +1,16 @@ +package org.ethereum.beacon.emulator.config; + +public class ConfigException extends RuntimeException { + + public ConfigException(String message) { + super(message); + } + + public ConfigException(String message, Throwable cause) { + super(message, cause); + } + + public ConfigException(Throwable cause) { + super(cause); + } +} diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/chainspec/SpecBuilder.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/chainspec/SpecBuilder.java index 1ba2f4c4d..5a368629e 100644 --- a/start/config/src/main/java/org/ethereum/beacon/emulator/config/chainspec/SpecBuilder.java +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/chainspec/SpecBuilder.java @@ -41,7 +41,7 @@ public BeaconChainSpec buildSpec( .withConstants(specConstants) .withBlsVerify(specHelpersOptions.isBlsVerify()) .withBlsVerifyProofOfPossession(specHelpersOptions.isBlsVerifyProofOfPossession()) - .enableCache() + .withCache(spec.getSpecHelpersOptions().isEnableCache()) .build(); } diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/chainspec/SpecHelpersData.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/chainspec/SpecHelpersData.java index 79c30ba89..e04267656 100644 --- a/start/config/src/main/java/org/ethereum/beacon/emulator/config/chainspec/SpecHelpersData.java +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/chainspec/SpecHelpersData.java @@ -9,6 +9,8 @@ public class SpecHelpersData { private boolean blsSign = true; + private boolean enableCache = true; + public boolean isBlsVerify() { return blsVerify; } @@ -32,4 +34,12 @@ public boolean isBlsSign() { public void setBlsSign(boolean blsSign) { this.blsSign = blsSign; } + + public boolean isEnableCache() { + return enableCache; + } + + public void setEnableCache(boolean enableCache) { + this.enableCache = enableCache; + } } diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/Configuration.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/Configuration.java index ef69a426b..1f6eb6ada 100644 --- a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/Configuration.java +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/Configuration.java @@ -1,10 +1,24 @@ package org.ethereum.beacon.emulator.config.main; +import java.util.ArrayList; +import java.util.List; +import org.ethereum.beacon.emulator.config.main.network.Network; + /** Beacon chain configuration */ public class Configuration { + private String name; private String db; + private List networks = new ArrayList<>(); private Validator validator; + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + public String getDb() { return db; } @@ -13,6 +27,14 @@ public void setDb(String db) { this.db = db; } + public List getNetworks() { + return networks; + } + + public void setNetworks(List networks) { + this.networks = networks; + } + public Validator getValidator() { return validator; } diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/Signer.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/Signer.java index bb67b88d4..2a0cb93bd 100644 --- a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/Signer.java +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/Signer.java @@ -1,41 +1,24 @@ package org.ethereum.beacon.emulator.config.main; -import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import java.util.List; -import java.util.Map; +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME) +@JsonSubTypes({ + @JsonSubTypes.Type(value = Signer.Insecure.class, name = "insecure"), +}) +public abstract class Signer { -/** Eth2.0 Signer settings */ -public class Signer { - private SignerImplementation implementation; + public static class Insecure extends Signer{ + List keys; - public SignerImplementation getImplementation() { - return implementation; - } - - public void setImplementation(SignerImplementation implementation) { - this.implementation = implementation; - } - - public static class SignerImplementation { - @JsonProperty("class") - private String clazz; - - private Map input; - - public String getClazz() { - return clazz; - } - - public void setClazz(String clazz) { - this.clazz = clazz; - } - - public Map getInput() { - return input; + public List getKeys() { + return keys; } - public void setInput(Map input) { - this.input = input; + public void setKeys(List keys) { + this.keys = keys; } } } diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/Validator.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/Validator.java index c9a94cdbe..800f1d181 100644 --- a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/Validator.java +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/Validator.java @@ -1,17 +1,17 @@ package org.ethereum.beacon.emulator.config.main; -import java.util.Map; +import org.ethereum.beacon.emulator.config.main.conract.Contract; /** Validator settings */ public class Validator { - private Map contract; + private Contract contract; private Signer signer; - public Map getContract() { + public Contract getContract() { return contract; } - public void setContract(Map contract) { + public void setContract(Contract contract) { this.contract = contract; } diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/ValidatorKeys.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/ValidatorKeys.java new file mode 100644 index 000000000..ee1de7a5e --- /dev/null +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/ValidatorKeys.java @@ -0,0 +1,68 @@ +package org.ethereum.beacon.emulator.config.main; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import java.util.List; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME) +@JsonSubTypes({ + @JsonSubTypes.Type(value = ValidatorKeys.Generate.class, name = "generate"), + @JsonSubTypes.Type(value = ValidatorKeys.Private.class, name = "private"), + @JsonSubTypes.Type(value = ValidatorKeys.Public.class, name = "public"), +}) +public abstract class ValidatorKeys { + + public static class Generate extends ValidatorKeys { + private int count; + private int seed = 0; + private int startIndex = 0; + + public int getCount() { + return count; + } + + public void setCount(int count) { + this.count = count; + } + + public int getSeed() { + return seed; + } + + public void setSeed(int seed) { + this.seed = seed; + } + + public int getStartIndex() { + return startIndex; + } + + public void setStartIndex(int startIndex) { + this.startIndex = startIndex; + } + } + + public static class ExplicitKeys extends ValidatorKeys { + private List keys; + + public List getKeys() { + return keys; + } + + public void setKeys(List keys) { + this.keys = keys; + } + } + + public static class Private extends ExplicitKeys { + + public Private() { + } + + public Private(List keys) { + setKeys(keys); + } + } + + public static class Public extends ExplicitKeys {} +} diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/Contract.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/Contract.java new file mode 100644 index 000000000..889516473 --- /dev/null +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/Contract.java @@ -0,0 +1,11 @@ +package org.ethereum.beacon.emulator.config.main.conract; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME) +@JsonSubTypes({ + @JsonSubTypes.Type(value = EmulatorContract.class, name = "emulator"), + @JsonSubTypes.Type(value = EthereumJContract.class, name = "ethereumj"), +}) +public abstract class Contract {} diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EmulatorContract.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EmulatorContract.java new file mode 100644 index 000000000..96a9350be --- /dev/null +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EmulatorContract.java @@ -0,0 +1,39 @@ +package org.ethereum.beacon.emulator.config.main.conract; + +import com.fasterxml.jackson.annotation.JsonFormat; +import java.util.Date; +import java.util.List; +import org.ethereum.beacon.emulator.config.main.ValidatorKeys; + +public class EmulatorContract extends Contract { + + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT") + private Date genesisTime; + + private Integer balance; + private List keys; + + public Date getGenesisTime() { + return genesisTime; + } + + public void setGenesisTime(Date genesisTime) { + this.genesisTime = genesisTime; + } + + public Integer getBalance() { + return balance; + } + + public void setBalance(Integer balance) { + this.balance = balance; + } + + public List getKeys() { + return keys; + } + + public void setKeys(List keys) { + this.keys = keys; + } +} diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EthereumJContract.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EthereumJContract.java new file mode 100644 index 000000000..8b7c0c754 --- /dev/null +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/conract/EthereumJContract.java @@ -0,0 +1,31 @@ +package org.ethereum.beacon.emulator.config.main.conract; + +public class EthereumJContract extends Contract { + private String contractAddress; + private long contractBlock; + private String contractAbiPath; + + public String getContractAddress() { + return contractAddress; + } + + public void setContractAddress(String contractAddress) { + this.contractAddress = contractAddress; + } + + public long getContractBlock() { + return contractBlock; + } + + public void setContractBlock(long contractBlock) { + this.contractBlock = contractBlock; + } + + public String getContractAbiPath() { + return contractAbiPath; + } + + public void setContractAbiPath(String contractAbiPath) { + this.contractAbiPath = contractAbiPath; + } +} diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/network/NettyNetwork.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/network/NettyNetwork.java new file mode 100644 index 000000000..3c3e134dd --- /dev/null +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/network/NettyNetwork.java @@ -0,0 +1,25 @@ +package org.ethereum.beacon.emulator.config.main.network; + +import java.util.ArrayList; +import java.util.List; + +public class NettyNetwork extends Network { + private Integer listenPort; + private List activePeers = new ArrayList<>(); + + public Integer getListenPort() { + return listenPort; + } + + public void setListenPort(Integer listenPort) { + this.listenPort = listenPort; + } + + public List getActivePeers() { + return activePeers; + } + + public void setActivePeers(List activePeers) { + this.activePeers = activePeers; + } +} diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/network/Network.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/network/Network.java new file mode 100644 index 000000000..93d51ca20 --- /dev/null +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/main/network/Network.java @@ -0,0 +1,10 @@ +package org.ethereum.beacon.emulator.config.main.network; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = NettyNetwork.class, name = "netty"), +}) +public abstract class Network {} diff --git a/start/config/src/test/java/org/ethereum/beacon/emulator/config/ConfigBuilderTest.java b/start/config/src/test/java/org/ethereum/beacon/emulator/config/ConfigBuilderTest.java index cfeaaa8f0..c5baa952f 100644 --- a/start/config/src/test/java/org/ethereum/beacon/emulator/config/ConfigBuilderTest.java +++ b/start/config/src/test/java/org/ethereum/beacon/emulator/config/ConfigBuilderTest.java @@ -1,13 +1,17 @@ package org.ethereum.beacon.emulator.config; import java.util.Collections; +import java.util.Date; import org.ethereum.beacon.consensus.BeaconChainSpec; import org.ethereum.beacon.core.spec.SpecConstants; import org.ethereum.beacon.emulator.config.chainspec.SpecData; import org.ethereum.beacon.emulator.config.chainspec.SpecBuilder; import org.ethereum.beacon.emulator.config.main.Configuration; import org.ethereum.beacon.emulator.config.main.MainConfig; +import org.ethereum.beacon.emulator.config.main.ValidatorKeys; import org.ethereum.beacon.emulator.config.main.action.ActionRun; +import org.ethereum.beacon.emulator.config.main.conract.EmulatorContract; +import org.ethereum.beacon.emulator.config.main.conract.EthereumJContract; import org.ethereum.beacon.emulator.config.main.plan.GeneralPlan; import org.junit.Test; import tech.pegasys.artemis.util.uint.UInt64; @@ -16,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; public class ConfigBuilderTest { @@ -46,14 +51,17 @@ public void test1() { MainConfig merged = configBuilder.build(); assertEquals("file://second/path", merged.getConfig().getDb()); assertEquals(2, ((GeneralPlan) merged.getPlan()).getValidator().size()); - assertEquals("ethereumj", merged.getConfig().getValidator().getContract().get("handler")); + assertTrue(merged.getConfig().getValidator().getContract() instanceof EmulatorContract); + assertEquals(3, (long) ((ValidatorKeys.Private)((EmulatorContract) + merged.getConfig().getValidator().getContract()).getKeys().get(1)).getKeys().size()); + assertEquals( + new Date(1526835300000L), + ((EmulatorContract) merged.getConfig().getValidator().getContract()).getGenesisTime()); configBuilder.addConfigOverride("config.db", "file://test-db"); - configBuilder.addConfigOverride("config.validator.contract.handler", "unknown"); MainConfig overrided = configBuilder.build(); assertEquals("file://test-db", overrided.getConfig().getDb()); assertEquals(2, ((GeneralPlan) overrided.getPlan()).getValidator().size()); - assertEquals("unknown", overrided.getConfig().getValidator().getContract().get("handler")); } @Test diff --git a/start/config/src/test/resources/config.yml b/start/config/src/test/resources/config.yml index e0603fe36..f7fe1bc01 100644 --- a/start/config/src/test/resources/config.yml +++ b/start/config/src/test/resources/config.yml @@ -1,18 +1,32 @@ config: db: file://db + networks: + - type: netty + listenPort: 40001 + activePeers: + - tcp://localhost:40002 + - tcp://localhost:40003 validator: - contract: - handler: ethereumj - distanceFromHead: 1000 - contractAddress: 0xd47c61f2c25aaa677dcf23e65765fac04c85d6a0 - contractBlock: 8000000 - contractAbi: file://Contract.abi - signer: - implementation: - class: InsecureBLS381MessageSigner - input: - pubKey: 1 - privKey: 1 + contract: !emulator + genesisTime: 2018-05-20 16:55:00 + keys: + - !generate + count: 16 + seed: 666 + - !private + keys: + - 0x3564c032070e518026e47b32b6d34bca57c192d56f62e41b33e4b952e7b04d7a + - 0x3564c032070e518026e47b32b6d34bca57c192d56f62e41b33e4b952e7b04d7b + - 0x3564c032070e518026e47b32b6d34bca57c192d56f62e41b33e4b952e7b04d7c + signer: !insecure + keys: + - !generate + startIndex: 15 + count: 1 + seed: 666 + - !private + keys: + - 0x3564c032070e518026e47b32b6d34bca57c192d56f62e41b33e4b952e7b04d7a plan: !general sync: diff --git a/start/node/build.gradle b/start/node/build.gradle new file mode 100644 index 000000000..d6eb5edca --- /dev/null +++ b/start/node/build.gradle @@ -0,0 +1,33 @@ +plugins { + id 'application' +} + +application { + mainClassName = 'org.ethereum.beacon.node.Node' + applicationDefaultJvmArgs = ['-Xmx2g'] +} + +dependencies { + implementation project(':types') + implementation project(':wire') + implementation project(':util') + implementation project(':start:common') + implementation project(':start:config') + implementation project(':crypto') + implementation project(':core') + implementation project(':consensus') + implementation project(':db:core') + implementation project(':chain') + implementation project(':ssz') + implementation project(':pow:core') + implementation project(':validator') + + implementation 'info.picocli:picocli' + implementation 'com.google.guava:guava' + implementation 'io.projectreactor:reactor-core' + implementation 'org.apache.logging.log4j:log4j-core' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'io.netty:netty-all' + + testImplementation 'org.mockito:mockito-core' +} diff --git a/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java b/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java new file mode 100644 index 000000000..52ae6edae --- /dev/null +++ b/start/node/src/main/java/org/ethereum/beacon/node/ConfigUtils.java @@ -0,0 +1,105 @@ +package org.ethereum.beacon.node; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.operations.Deposit; +import org.ethereum.beacon.core.state.Eth1Data; +import org.ethereum.beacon.core.types.Gwei; +import org.ethereum.beacon.core.types.Time; +import org.ethereum.beacon.crypto.BLS381.KeyPair; +import org.ethereum.beacon.crypto.BLS381.PrivateKey; +import org.ethereum.beacon.emulator.config.main.Signer; +import org.ethereum.beacon.emulator.config.main.Signer.Insecure; +import org.ethereum.beacon.emulator.config.main.ValidatorKeys; +import org.ethereum.beacon.emulator.config.main.ValidatorKeys.Generate; +import org.ethereum.beacon.emulator.config.main.ValidatorKeys.Private; +import org.ethereum.beacon.emulator.config.main.ValidatorKeys.Public; +import org.ethereum.beacon.emulator.config.main.conract.Contract; +import org.ethereum.beacon.emulator.config.main.conract.EmulatorContract; +import org.ethereum.beacon.pow.DepositContract; +import org.ethereum.beacon.start.common.util.SimpleDepositContract; +import org.ethereum.beacon.start.common.util.SimulateUtils; +import org.ethereum.beacon.validator.crypto.BLS381Credentials; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +public class ConfigUtils { + + public static List createCredentials(Signer config, boolean isBlsSign) { + if (config == null) { + return null; + } + if (config instanceof Signer.Insecure) { + return createKeyPairs(((Insecure) config).getKeys()) + .stream() + .map( + key -> + isBlsSign + ? BLS381Credentials.createWithInsecureSigner(key) + : BLS381Credentials.createWithDummySigner(key)) + .collect(Collectors.toList()); + } else { + throw new IllegalArgumentException( + "This config class is not yet supported: " + config.getClass()); + } + } + + public static List createKeyPairs(List keys) { + return keys.stream().flatMap(k -> createKeyPairs(k).stream()).collect(Collectors.toList()); + } + + public static List createKeyPairs(ValidatorKeys keys) { + if (keys instanceof Public) { + throw new IllegalArgumentException("Can't generate key pairs from public keys: " + keys); + } else if (keys instanceof Private) { + return ((Private) keys).getKeys().stream() + .map(Bytes32::fromHexString) + .map(PrivateKey::create) + .map(KeyPair::create) + .collect(Collectors.toList()); + } else if (keys instanceof Generate) { + Generate genKeys = (Generate) keys; + Random random = new Random(genKeys.getSeed()); + for (int i = 0; i < genKeys.getStartIndex(); i++) { + Bytes32.random(random); + } + List ret = new ArrayList<>(); + for (int i = 0; i < genKeys.getCount(); i++) { + ret.add(KeyPair.create(PrivateKey.create(Bytes32.random(random)))); + } + return ret; + } else { + throw new IllegalArgumentException("Unknown ValidatorKeys subclass: " + keys.getClass()); + } + } + + public static DepositContract createDepositContract(Contract config, BeaconChainSpec spec, boolean verifyProof) { + if (config instanceof EmulatorContract) { + EmulatorContract eConfig = (EmulatorContract) config; + List keyPairs = createKeyPairs(eConfig.getKeys()); + Random random = new Random(1); + Gwei amount = + eConfig.getBalance() != null + ? Gwei.ofEthers(eConfig.getBalance()) + : spec.getConstants().getMaxEffectiveBalance(); + List deposits = SimulateUtils + .getDepositsForKeyPairs(UInt64.ZERO, random, keyPairs, spec, amount, verifyProof); + Eth1Data eth1Data = + new Eth1Data( + Hash32.random(random), UInt64.valueOf(deposits.size()), Hash32.random(random)); + DepositContract.ChainStart chainStart = + new DepositContract.ChainStart( + Time.of(eConfig.getGenesisTime().getTime() / 1000), eth1Data, deposits); + SimpleDepositContract depositContract = new SimpleDepositContract(chainStart); + return depositContract; + } else { + throw new IllegalArgumentException( + "This config class is not yet supported: " + config.getClass()); + } + } +} diff --git a/start/node/src/main/java/org/ethereum/beacon/node/Node.java b/start/node/src/main/java/org/ethereum/beacon/node/Node.java new file mode 100644 index 000000000..7a59d64fa --- /dev/null +++ b/start/node/src/main/java/org/ethereum/beacon/node/Node.java @@ -0,0 +1,37 @@ +package org.ethereum.beacon.node; + +import org.ethereum.beacon.node.command.RunNode; +import picocli.CommandLine; +import picocli.CommandLine.RunLast; + +@CommandLine.Command( + description = "Beacon chain node", + name = "node", + version = "node " + Node.VERSION, + mixinStandardHelpOptions = true, + subcommands = {RunNode.class}) +public class Node implements Runnable { + + static final String VERSION = "0.1.0"; + + private static final int SUCCESS_EXIT_CODE = 0; + private static final int ERROR_EXIT_CODE = 1; + + public static void main(String[] args) { + try { + CommandLine commandLine = new CommandLine(new Node()); + commandLine.setCaseInsensitiveEnumValuesAllowed(true); + commandLine.parseWithHandlers( + new RunLast().andExit(SUCCESS_EXIT_CODE), + CommandLine.defaultExceptionHandler().andExit(ERROR_EXIT_CODE), + args); + } catch (Exception e) { + System.out.println(String.format((char) 27 + "[31m" + "FATAL ERROR: %s", e.getMessage())); + } + } + + @Override + public void run() { + CommandLine.usage(this, System.out); + } +} diff --git a/start/node/src/main/java/org/ethereum/beacon/node/NodeCommandLauncher.java b/start/node/src/main/java/org/ethereum/beacon/node/NodeCommandLauncher.java new file mode 100644 index 000000000..9b618297c --- /dev/null +++ b/start/node/src/main/java/org/ethereum/beacon/node/NodeCommandLauncher.java @@ -0,0 +1,356 @@ +package org.ethereum.beacon.node; + +import io.netty.channel.ChannelFutureListener; +import java.io.File; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.net.URI; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Random; +import java.util.TimeZone; +import java.util.concurrent.ThreadFactory; +import java.util.stream.IntStream; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.ThreadContext; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.ethereum.beacon.chain.storage.impl.MemBeaconChainStorageFactory; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.spec.SpecConstants; +import org.ethereum.beacon.core.state.Eth1Data; +import org.ethereum.beacon.core.types.Time; +import org.ethereum.beacon.crypto.BLS381.KeyPair; +import org.ethereum.beacon.emulator.config.ConfigBuilder; +import org.ethereum.beacon.emulator.config.ConfigException; +import org.ethereum.beacon.emulator.config.chainspec.SpecBuilder; +import org.ethereum.beacon.emulator.config.chainspec.SpecData; +import org.ethereum.beacon.emulator.config.main.MainConfig; +import org.ethereum.beacon.emulator.config.main.Signer.Insecure; +import org.ethereum.beacon.emulator.config.main.ValidatorKeys.Private; +import org.ethereum.beacon.emulator.config.main.conract.EmulatorContract; +import org.ethereum.beacon.emulator.config.main.network.NettyNetwork; +import org.ethereum.beacon.emulator.config.main.network.Network; +import org.ethereum.beacon.node.command.RunNode; +import org.ethereum.beacon.pow.DepositContract; +import org.ethereum.beacon.schedulers.DefaultSchedulers; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.start.common.NodeLauncher; +import org.ethereum.beacon.start.common.util.MDCControlledSchedulers; +import org.ethereum.beacon.validator.crypto.BLS381Credentials; +import org.ethereum.beacon.wire.net.ConnectionManager; +import org.ethereum.beacon.wire.net.netty.NettyClient; +import org.ethereum.beacon.wire.net.netty.NettyServer; + +public class NodeCommandLauncher implements Runnable { + private static final Logger logger = LogManager.getLogger("node"); + + private final MainConfig config; + private final SpecConstants specConstants; + private final BeaconChainSpec spec; + private final Level logLevel; + private final SpecBuilder specBuilder; + + private Random rnd; + private Time genesisTime; + private MDCControlledSchedulers controlledSchedulers; + private List keyPairs; + private Eth1Data eth1Data; + private DepositContract depositContract; + + /** + * Creates launcher with following settings + * + * @param config configuration and run plan. + * @param specBuilder chain specification builder. + * @param logLevel Log level, Apache log4j type. + */ + public NodeCommandLauncher( + MainConfig config, + SpecBuilder specBuilder, + Level logLevel) { + this.config = config; + this.specBuilder = specBuilder; + this.specConstants = specBuilder.buildSpecConstants(); + this.spec = specBuilder.buildSpec(); + this.logLevel = logLevel; + + init(); + } + + private void setupLogging() { + // set logLevel + if (logLevel != null) { + LoggerContext context = + (LoggerContext) LogManager.getContext(false); + Configuration config = context.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig("node"); + loggerConfig.setLevel(logLevel); + context.updateLoggers(); + } + } + + public void init() { + setupLogging(); + } + + public void run() { + String nodeName = config.getConfig().getName(); + if (nodeName != null) { + ThreadContext.put("validatorIndex", nodeName); + } + + if (config.getChainSpec().isDefined()) + logger.info("Overridden beacon chain parameters:\n{}", config.getChainSpec()); + + Schedulers schedulers = + new DefaultSchedulers() { + @Override + protected ThreadFactory createThreadFactory(String namePattern) { + ThreadFactory factory = + createThreadFactoryBuilder((nodeName == null ? "" : nodeName + "-") + namePattern).build(); + if (nodeName == null) { + return factory; + } else { + return r -> + factory.newThread( + () -> { + ThreadContext.put("validatorIndex", nodeName); + r.run(); + }); + } + } + }; + + depositContract = ConfigUtils.createDepositContract( + config.getConfig().getValidator().getContract(), + spec, + config.getChainSpec().getSpecHelpersOptions().isBlsVerifyProofOfPossession()); + + List credentials = ConfigUtils.createCredentials( + config.getConfig().getValidator().getSigner(), + config.getChainSpec().getSpecHelpersOptions().isBlsSign()); + + ConnectionManager connectionManager; + if (config.getConfig().getNetworks().size() != 1) { + throw new IllegalArgumentException("1 network should be specified in config"); + } + Network networkCfg = config.getConfig().getNetworks().get(0); + if (networkCfg instanceof NettyNetwork) { + NettyNetwork nettyConfig = (NettyNetwork) networkCfg; + NettyServer nettyServer = null; + if (nettyConfig.getListenPort() != null) { + Scheduler serverScheduler = schedulers.newParallelDaemon("netty-server-%d", 16); + nettyServer = new NettyServer(nettyConfig.getListenPort(), serverScheduler::executeR); + nettyServer.start().addListener((ChannelFutureListener) channelFuture -> { + try { + channelFuture.get(); + logger.info("Listening for inbound connections on port " + nettyConfig.getListenPort()); + } catch (Exception e) { + logger.error("Unable to open inbound port " + nettyConfig.getListenPort(), e); + } + }); + } + Scheduler clientScheduler = schedulers.newParallelDaemon("netty-client-%d", 2); + NettyClient nettyClient = new NettyClient(clientScheduler::executeR); + ConnectionManager tcpConnectionManager = + new ConnectionManager<>(nettyServer, nettyClient, schedulers.reactorEvents()); + connectionManager = tcpConnectionManager; + for (String addr : nettyConfig.getActivePeers()) { + URI uri = URI.create(addr); + tcpConnectionManager.addActivePeer(InetSocketAddress.createUnresolved(uri.getHost(), uri.getPort())); + } + } else { + throw new IllegalArgumentException( + "This type of network is not supported yet: " + networkCfg.getClass()); + } + + NodeLauncher node = new NodeLauncher( + specBuilder.buildSpec(), + this.depositContract, + credentials, + connectionManager, + new MemBeaconChainStorageFactory(spec.getObjectHasher()), + schedulers, + true); + + while (true) { + try { + Thread.sleep(1000000L); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + public static class Builder { + private MainConfig config; + private Level logLevel = Level.INFO; + private RunNode cliOptions; + + public Builder() {} + + public NodeCommandLauncher build() { + assert config != null; + + ConfigBuilder specConfigBuilder = + new ConfigBuilder<>(SpecData.class).addYamlConfigFromResources("/config/spec-constants.yml"); + if (config.getChainSpec().isDefined()) { + specConfigBuilder.addConfig(config.getChainSpec()); + } + + SpecData spec = specConfigBuilder.build(); + + SpecBuilder specBuilder = new SpecBuilder().withSpec(spec); + + if (cliOptions.getName() != null) { + config.getConfig().setName(cliOptions.getName()); + } + + + if (cliOptions.getListenPort() != null || cliOptions.getActivePeers() != null) { + NettyNetwork network = (NettyNetwork) config + .getConfig() + .getNetworks() + .stream() + .filter(n -> n instanceof NettyNetwork) + .findFirst().orElse(null); + + if (network == null) { + network = new NettyNetwork(); + config.getConfig().getNetworks().add(network); + } + + if (cliOptions.getListenPort() != null) { + network.setListenPort(cliOptions.getListenPort()); + } + + if (cliOptions.getActivePeers() != null) { + network.setActivePeers(cliOptions.getActivePeers()); + } + } + + if (cliOptions.getValidators() != null) { + List validatorKeys = new ArrayList<>(); + List depositKeypairs = null; + for (String key : cliOptions.getValidators()) { + if (key.startsWith("0x")) { + validatorKeys.add(key); + } else { + if (depositKeypairs == null) { + depositKeypairs = ConfigUtils + .createKeyPairs(((EmulatorContract)config.getConfig().getValidator().getContract()).getKeys()); + } + List finalDepositKeypairs = depositKeypairs; + + IntStream indices; + if (key.contains("-")) { + int idx = key.indexOf("-"); + int start = Integer.parseInt(key.substring(0, idx)); + int end = Integer.parseInt(key.substring(idx + 1)); + indices = IntStream.range(start, end + 1); + } else { + indices = IntStream.of(Integer.parseInt(key)); + } + indices + .mapToObj(i -> finalDepositKeypairs.get(i).getPrivate().getEncodedBytes().toString()) + .forEach(validatorKeys::add); + } + } + Insecure signer = new Insecure(); + signer.setKeys(Collections.singletonList(new Private(validatorKeys))); + config.getConfig().getValidator().setSigner(signer); + } + + if (cliOptions.getGenesisTime() != null) { + SimpleDateFormat[] supportedFormats = new SimpleDateFormat[] { + new SimpleDateFormat("yyyy-MM-dd HH:mm"), + new SimpleDateFormat("HH:mm")}; + + Date time = null; + for (SimpleDateFormat format : supportedFormats) { + format.setTimeZone(TimeZone.getTimeZone("GMT")); + try { + time = format.parse(cliOptions.getGenesisTime()); + break; + } catch (ParseException e) { + continue; + } + } + if (time == null) { + throw new ConfigException( + "Couldn't parse --genesisTime option value: '" + cliOptions.getGenesisTime() + "'"); + } + if (time.getYear() + 1900 == 1970) { + Date now = new Date(); + time.setYear(now.getYear()); + time.setMonth(now.getMonth()); + time.setDate(now.getDate()); + } + + if (!(config.getConfig().getValidator().getContract() instanceof EmulatorContract)) { + throw new ConfigException("Genesis time can only be set for 'emulator' contract type"); + } + EmulatorContract contract = (EmulatorContract) config.getConfig().getValidator().getContract(); + contract.setGenesisTime(time); + + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + format.setTimeZone(TimeZone.getTimeZone("GMT")); + logger.info("Genesis time from cli option: " + + format.format(time) + " GMT"); + } + + if (config.getConfig().getValidator().getContract() instanceof EmulatorContract) { + EmulatorContract contract = (EmulatorContract) config.getConfig().getValidator().getContract(); + if (contract.getGenesisTime() == null) { + Date defaultTime = new Date(); + defaultTime.setMinutes(0); + defaultTime.setSeconds(0); + defaultTime = new Date(defaultTime.getTime() / 1000 * 1000); + contract.setGenesisTime(defaultTime); + + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + format.setTimeZone(TimeZone.getTimeZone("GMT")); + logger.warn("Genesis time not specified. Default genesisTime was generated: " + + format.format(defaultTime) + " GMT"); + } + } + + return new NodeCommandLauncher( + config, + specBuilder, + logLevel); + } + + public Builder withConfigFromFile(File file) { + this.config = new ConfigBuilder<>(MainConfig.class).addYamlConfig(file).build(); + return this; + } + + public Builder withConfigFromResource(String resourceName) { + this.config = + new ConfigBuilder<>(MainConfig.class) + .addYamlConfigFromResources(resourceName) + .build(); + return this; + } + + public Builder withLogLevel(Level logLevel) { + this.logLevel = logLevel; + return this; + } + + public Builder withCliOptions(RunNode runNode) { + cliOptions = runNode; + return this; + } + } +} diff --git a/start/node/src/main/java/org/ethereum/beacon/node/command/LogLevel.java b/start/node/src/main/java/org/ethereum/beacon/node/command/LogLevel.java new file mode 100644 index 000000000..9153ef569 --- /dev/null +++ b/start/node/src/main/java/org/ethereum/beacon/node/command/LogLevel.java @@ -0,0 +1,25 @@ +package org.ethereum.beacon.node.command; + +import org.apache.logging.log4j.Level; + +public enum LogLevel { + all, + debug, + info, + error; + + public Level toLog4j() { + switch (this) { + case all: + return Level.ALL; + case debug: + return Level.DEBUG; + case info: + return Level.INFO; + case error: + return Level.ERROR; + default: + throw new IllegalArgumentException("Unsupported log level " + this); + } + } +} diff --git a/start/node/src/main/java/org/ethereum/beacon/node/command/RunNode.java b/start/node/src/main/java/org/ethereum/beacon/node/command/RunNode.java new file mode 100644 index 000000000..d4a371a26 --- /dev/null +++ b/start/node/src/main/java/org/ethereum/beacon/node/command/RunNode.java @@ -0,0 +1,109 @@ +package org.ethereum.beacon.node.command; + +import java.io.File; +import java.util.List; +import org.ethereum.beacon.node.NodeCommandLauncher; +import picocli.CommandLine; + +@CommandLine.Command( + name = "run", + description = "Runs beacon chain node", + mixinStandardHelpOptions = true, + sortOptions = false +) +public class RunNode implements Runnable { + + @CommandLine.Parameters( + index = "0", + paramLabel = "node-config.yml", + description = + "A path to a config file containing node config in YAML format\nuse 'default' to run a node with default setup" + ) + private String config; + + @CommandLine.Option( + names = {"--loglevel"}, + paramLabel = "level", + description = "Log verbosity level: all, debug, info, error\ninfo is set by default" + ) + private LogLevel logLevel = null; + + @CommandLine.Option( + names = {"--listen"}, + paramLabel = "port", + description = "Listen for inbound connections on TCP port" + ) + private Integer listenPort; + + @CommandLine.Option( + names = {"--connect"}, + paramLabel = "URL", + split = ",", + description = "Actively connects to remote peers. URL in form 'tcp://:'" + ) + private List activePeers; + + @CommandLine.Option( + names = {"--validators"}, + paramLabel = "key", + split = ",", + description = { + "List of signers. Entry is either hex private key (starting from '0x'), " + + "or index of keypair specified in the 'contract emulator' config deposits " + + "or a range of such indices", + "Example: --validators=1,2,5-9,0x1234567...ef" + } + ) + private List validators; + + @CommandLine.Option( + names = {"--name"}, + paramLabel = "node-name", + description = "Node name for logs identification (when several nodes running)" + ) + private String name; + + @CommandLine.Option( + names = {"--genesis-time"}, + paramLabel = "time", + description = "Genesis time in GMT+0 timezone. In either form: '2019-05-24 11:23', or just" + + " '11:23' (current day is taken). Default value is start of the current hour." + ) + private String genesisTime; + + public String getName() { + return name; + } + + public Integer getListenPort() { + return listenPort; + } + + public List getActivePeers() { + return activePeers; + } + + public List getValidators() { + return validators; + } + + public String getGenesisTime() { + return genesisTime; + } + + @Override + public void run() { + NodeCommandLauncher.Builder nodeBuilder = + new NodeCommandLauncher.Builder() + .withLogLevel(logLevel == null ? null : logLevel.toLog4j()) + .withCliOptions(this); + + if ("default".equals(config)) { + nodeBuilder.withConfigFromResource("/config/default-node-config.yml"); + } else { + nodeBuilder.withConfigFromFile(new File(config)); + } + + nodeBuilder.build().run(); + } +} diff --git a/start/node/src/main/resources/config/default-node-config.yml b/start/node/src/main/resources/config/default-node-config.yml new file mode 100644 index 000000000..e1eba1d80 --- /dev/null +++ b/start/node/src/main/resources/config/default-node-config.yml @@ -0,0 +1,44 @@ +config: + db: file://db + networks: + - type: netty + + validator: + contract: !emulator + balance: 55 + keys: + - !generate + count: 16 + seed: 0 +# signer: !insecure +# keys: +# - !generate +# count: 16 +# seed: 0 + +chainSpec: + specConstants: + initialValues: + GENESIS_SLOT: 0 + miscParameters: + SHARD_COUNT: 4 + TARGET_COMMITTEE_SIZE: 2 + timeParameters: + SECONDS_PER_SLOT: 10 + MIN_ATTESTATION_INCLUSION_DELAY: 1 + SLOTS_PER_EPOCH: 4 + SLOTS_PER_HISTORICAL_ROOT: 64 + + honestValidatorParameters: + ETH1_FOLLOW_DISTANCE: 1 + stateListLengths: + LATEST_RANDAO_MIXES_LENGTH: 64 + LATEST_ACTIVE_INDEX_ROOTS_LENGTH: 64 + LATEST_SLASHED_EXIT_LENGTH: 64 + + specHelpersOptions: + blsVerify: false + blsVerifyProofOfPossession: false + blsSign: false + enableCache: false + diff --git a/start/node/src/main/resources/config/node-config.yml b/start/node/src/main/resources/config/node-config.yml new file mode 100644 index 000000000..58b6821b7 --- /dev/null +++ b/start/node/src/main/resources/config/node-config.yml @@ -0,0 +1,46 @@ +config: + db: file://db + networks: + - type: netty + listenPort: 40001 + activePeers: + - tcp://localhost:40002 + - tcp://localhost:40003 + + validator: + contract: !emulator + genesisTime: 2018-05-20 16:55:00 + keys: + - !generate + count: 16 + seed: 0 + signer: !insecure + keys: + - !generate + count: 16 + seed: 0 + +chainSpec: + specConstants: + initialValues: + GENESIS_SLOT: 1000000 + miscParameters: + SHARD_COUNT: 4 + TARGET_COMMITTEE_SIZE: 2 + timeParameters: + SECONDS_PER_SLOT: 10 + MIN_ATTESTATION_INCLUSION_DELAY: 1 + SLOTS_PER_EPOCH: 4 + SLOTS_PER_HISTORICAL_ROOT: 64 + + honestValidatorParameters: + ETH1_FOLLOW_DISTANCE: 1 + stateListLengths: + LATEST_RANDAO_MIXES_LENGTH: 64 + LATEST_ACTIVE_INDEX_ROOTS_LENGTH: 64 + LATEST_SLASHED_EXIT_LENGTH: 64 + + specHelpersOptions: + blsVerify: false + blsVerifyProofOfPossession: false + blsSign: false diff --git a/start/node/src/main/resources/log4j2.xml b/start/node/src/main/resources/log4j2.xml new file mode 100644 index 000000000..00f89c297 --- /dev/null +++ b/start/node/src/main/resources/log4j2.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + %d{HH:mm:ss.SSS} %p %c{1.} [%t] %m%n + + + + + + + + + %d{HH:mm:ss.SSS} [#%X{validatorIndex}] %p %c{1.} [%t] %m%n + + + + + + + %d{HH:mm:ss.SSS} #%X{validatorIndex} %X{validatorTime} %-5level - %msg%n + + + + + + + + + + + + + + + diff --git a/start/node/src/test/resources/config/fast-chainSpec.yml b/start/node/src/test/resources/config/fast-chainSpec.yml new file mode 100644 index 000000000..41be4da49 --- /dev/null +++ b/start/node/src/test/resources/config/fast-chainSpec.yml @@ -0,0 +1,8 @@ +initialValues: + GENESIS_SLOT: 1000000 +miscParameters: + TARGET_COMMITTEE_SIZE: 1 +timeParameters: + SECONDS_PER_SLOT: 10 + MIN_ATTESTATION_INCLUSION_DELAY: 1 + SLOTS_PER_EPOCH: 2 diff --git a/start/simulator/src/main/java/org/ethereum/beacon/simulator/SimulatorLauncher.java b/start/simulator/src/main/java/org/ethereum/beacon/simulator/SimulatorLauncher.java index ab81ee2ce..680b8f06c 100644 --- a/start/simulator/src/main/java/org/ethereum/beacon/simulator/SimulatorLauncher.java +++ b/start/simulator/src/main/java/org/ethereum/beacon/simulator/SimulatorLauncher.java @@ -1,18 +1,13 @@ package org.ethereum.beacon.simulator; import java.io.File; -import java.io.InputStream; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Random; import java.util.stream.Collectors; import org.apache.logging.log4j.Level; @@ -20,10 +15,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.config.Configuration; -import org.apache.logging.log4j.core.config.ConfigurationSource; -import org.apache.logging.log4j.core.config.Configurator; import org.apache.logging.log4j.core.config.LoggerConfig; -import org.ethereum.beacon.Launcher; import org.ethereum.beacon.chain.observer.ObservableBeaconState; import org.ethereum.beacon.chain.storage.impl.MemBeaconChainStorageFactory; import org.ethereum.beacon.consensus.BeaconChainSpec; @@ -40,25 +32,22 @@ import org.ethereum.beacon.crypto.BLS381.KeyPair; import org.ethereum.beacon.crypto.BLS381.PrivateKey; import org.ethereum.beacon.emulator.config.ConfigBuilder; -import org.ethereum.beacon.emulator.config.chainspec.SpecData; import org.ethereum.beacon.emulator.config.chainspec.SpecBuilder; +import org.ethereum.beacon.emulator.config.chainspec.SpecData; import org.ethereum.beacon.emulator.config.main.MainConfig; import org.ethereum.beacon.emulator.config.main.plan.SimulationPlan; import org.ethereum.beacon.emulator.config.simulator.PeersConfig; import org.ethereum.beacon.pow.DepositContract; import org.ethereum.beacon.schedulers.ControlledSchedulers; -import org.ethereum.beacon.schedulers.LoggerMDCExecutor; -import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.schedulers.TimeController; -import org.ethereum.beacon.schedulers.TimeControllerImpl; -import org.ethereum.beacon.util.SimulateUtils; +import org.ethereum.beacon.start.common.Launcher; +import org.ethereum.beacon.start.common.util.MDCControlledSchedulers; +import org.ethereum.beacon.start.common.util.SimpleDepositContract; +import org.ethereum.beacon.start.common.util.SimulateUtils; import org.ethereum.beacon.validator.crypto.BLS381Credentials; import org.ethereum.beacon.wire.LocalWireHub; -import org.ethereum.beacon.wire.WireApi; +import org.ethereum.beacon.wire.WireApiSub; import org.javatuples.Pair; -import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.uint.UInt64; @@ -76,6 +65,16 @@ public class SimulatorLauncher implements Runnable { private final Level logLevel; private final SpecBuilder specBuilder; + private Random rnd; + private Time genesisTime; + private MDCControlledSchedulers controlledSchedulers; + private LocalWireHub localWireHub; + private List keyPairs; + private Eth1Data eth1Data; + private DepositContract depositContract; + + private List peers; + /** * Creates Simulator launcher with following settings * @@ -99,16 +98,11 @@ public SimulatorLauncher( this.validators = validators; this.observers = observers; this.logLevel = logLevel; + + init(); } private void setupLogging() { - try (InputStream inputStream = ClassLoader.class.getResourceAsStream("/log4j2.xml")) { - ConfigurationSource source = new ConfigurationSource(inputStream); - Configurator.initialize(null, source); - } catch (Exception e) { - throw new RuntimeException("Cannot read log4j default configuration", e); - } - // set logLevel if (logLevel != null) { LoggerContext context = @@ -138,43 +132,67 @@ private Pair, List> getValidatorDeposits(Random rn return deposits; } - public void run() { - logger.info("Simulation parameters:\n{}", simulationPlan); - if (config.getChainSpec().isDefined()) - logger.info("Overridden beacon chain parameters:\n{}", config.getChainSpec()); - - Random rnd = new Random(simulationPlan.getSeed()); + public void init() { + rnd = new Random(simulationPlan.getSeed()); setupLogging(); Pair, List> validatorDeposits = getValidatorDeposits(rnd); List deposits = validatorDeposits.getValue0().stream() .filter(Objects::nonNull).collect(Collectors.toList()); - List keyPairs = validatorDeposits.getValue1(); + keyPairs = validatorDeposits.getValue1(); - Time genesisTime = Time.of(simulationPlan.getGenesisTime()); + genesisTime = Time.of(simulationPlan.getGenesisTime()); - MDCControlledSchedulers controlledSchedulers = new MDCControlledSchedulers(); + controlledSchedulers = new MDCControlledSchedulers(); controlledSchedulers.setCurrentTime(genesisTime.getMillis().getValue() + 1000); - Eth1Data eth1Data = new Eth1Data(Hash32.random(rnd), UInt64.ZERO, Hash32.random(rnd)); + eth1Data = new Eth1Data(Hash32.random(rnd), UInt64.ZERO, Hash32.random(rnd)); - LocalWireHub localWireHub = - new LocalWireHub(s -> wire.trace(s), controlledSchedulers.createNew("wire")); + localWireHub = new LocalWireHub(s -> wire.trace(s), controlledSchedulers.createNew("wire")); DepositContract.ChainStart chainStart = new DepositContract.ChainStart(genesisTime, eth1Data, deposits); - DepositContract depositContract = new SimpleDepositContract(chainStart); + depositContract = new SimpleDepositContract(chainStart); + } + + public Launcher createPeer(String name) { + return createPeer(new PeersConfig(), null, name); + } + public Launcher createPeer(PeersConfig config, BLS381Credentials bls, String name) { + WireApiSub wireApi = + localWireHub.createNewPeer( + name, + config.getWireInboundDelay(), + config.getWireOutboundDelay()); + return createPeer(config, bls, wireApi, name); + } + + public Launcher createPeer(PeersConfig config, BLS381Credentials bls, WireApiSub wireApi, String name) { + ControlledSchedulers schedulers = + controlledSchedulers.createNew(name, config.getSystemTimeShift()); + + BeaconChainSpec spec = specBuilder.buildSpec(); + return new Launcher( + spec, + depositContract, + bls == null ? null : Collections.singletonList(bls), + wireApi, + new MemBeaconChainStorageFactory(spec.getObjectHasher()), + schedulers); + } - List peers = new ArrayList<>(); + public void run() { + run(Integer.MAX_VALUE); + } + + public void run(int slotsCount) { + logger.info("Simulation parameters:\n{}", simulationPlan); + if (config.getChainSpec().isDefined()) + logger.info("Overridden beacon chain parameters:\n{}", config.getChainSpec()); + + peers = new ArrayList<>(); logger.info("Creating validators..."); for (int i = 0; i < validators.size(); i++) { - ControlledSchedulers schedulers = - controlledSchedulers.createNew("V" + i, validators.get(i).getSystemTimeShift()); - WireApi wireApi = - localWireHub.createNewPeer( - "" + i, - validators.get(i).getWireInboundDelay(), - validators.get(i).getWireOutboundDelay()); BLS381Credentials bls; if (keyPairs.get(i) == null) { @@ -185,17 +203,7 @@ public void run() { BLS381Credentials.createWithDummySigner(keyPairs.get(i)); } - BeaconChainSpec spec = specBuilder.buildSpec(); - Launcher launcher = - new Launcher( - spec, - depositContract, - Collections.singletonList(bls), - wireApi, - new MemBeaconChainStorageFactory(spec.getObjectHasher()), - schedulers); - - peers.add(launcher); + peers.add(createPeer(validators.get(i), bls, "V" + i)); if ((i + 1) % 100 == 0) logger.info("{} validators created", (i + 1)); @@ -204,19 +212,7 @@ public void run() { logger.info("Creating observer peers..."); for (int i = 0; i < observers.size(); i++) { - PeersConfig config = observers.get(i); - String name = "O" + i; - BeaconChainSpec spec = specBuilder.buildSpec(); - Launcher launcher = - new Launcher( - spec, - depositContract, - null, - localWireHub.createNewPeer( - name, config.getWireInboundDelay(), config.getWireOutboundDelay()), - new MemBeaconChainStorageFactory(spec.getObjectHasher()), - controlledSchedulers.createNew(name, config.getSystemTimeShift())); - peers.add(launcher); + peers.add(createPeer(observers.get(i), null, "O" + i)); } Map latestStates = new HashMap<>(); @@ -245,18 +241,7 @@ public void run() { } // system observer - ControlledSchedulers schedulers = controlledSchedulers.createNew("X"); - WireApi wireApi = localWireHub.createNewPeer("X"); - - Launcher observer = - new Launcher( - spec, - depositContract, - null, - wireApi, - new MemBeaconChainStorageFactory(spec.getObjectHasher()), - schedulers); - + Launcher observer = createPeer("X"); peers.add(observer); List slots = new ArrayList<>(); @@ -290,7 +275,7 @@ public void run() { logger.info("Time starts running ..."); controlledSchedulers.setCurrentTime( genesisTime.plus(specConstants.getSecondsPerSlot()).getMillis().getValue() - 9); - while (true) { + for (int i = 0; i < slotsCount; i++) { controlledSchedulers.addTime( Duration.ofMillis(specConstants.getSecondsPerSlot().getMillis().getValue())); @@ -363,86 +348,39 @@ public void run() { } } - private static String getValidators(String info, Map records) { - if (records.isEmpty()) return ""; - return info + " [" - + records.entrySet().stream().map(e -> e.getKey().toString()).collect(Collectors.joining(",")) - + "]"; + public List getPeers() { + return peers; } - private static class SimpleDepositContract implements DepositContract { - private final ChainStart chainStart; - - public SimpleDepositContract(ChainStart chainStart) { - this.chainStart = chainStart; - } - - @Override - public Publisher getChainStartMono() { - return Mono.just(chainStart); - } - - @Override - public Publisher getDepositStream() { - return Mono.empty(); - } - - @Override - public List peekDeposits( - int maxCount, Eth1Data fromDepositExclusive, Eth1Data tillDepositInclusive) { - return Collections.emptyList(); - } - - @Override - public boolean hasDepositRoot(Hash32 blockHash, Hash32 depositRoot) { - return true; - } - - @Override - public Optional getLatestEth1Data() { - return Optional.of(chainStart.getEth1Data()); - } - - @Override - public void setDistanceFromHead(long distanceFromHead) {} + public BeaconChainSpec getSpec() { + return spec; } - public static class MDCControlledSchedulers { - private DateFormat localTimeFormat = new SimpleDateFormat("HH:mm:ss.SSS"); - - private TimeController timeController = new TimeControllerImpl(); - - public ControlledSchedulers createNew(String validatorId) { - return createNew(validatorId, 0); - } - - public ControlledSchedulers createNew(String validatorId, long timeShift) { - ControlledSchedulers[] newSched = new ControlledSchedulers[1]; - LoggerMDCExecutor mdcExecutor = new LoggerMDCExecutor() - .add("validatorTime", () -> localTimeFormat.format(new Date(newSched[0].getCurrentTime()))) - .add("validatorIndex", () -> "" + validatorId); - newSched[0] = Schedulers.createControlled(() -> mdcExecutor); - newSched[0].getTimeController().setParent(timeController); - newSched[0].getTimeController().setTimeShift(timeShift); + public Random getRnd() { + return rnd; + } - return newSched[0]; - } + public Time getGenesisTime() { + return genesisTime; + } - public void setCurrentTime(long time) { - timeController.setTime(time); - } + public MDCControlledSchedulers getControlledSchedulers() { + return controlledSchedulers; + } - void addTime(Duration duration) { - addTime(duration.toMillis()); - } + public LocalWireHub getLocalWireHub() { + return localWireHub; + } - void addTime(long millis) { - setCurrentTime(timeController.getTime() + millis); - } + public DepositContract getDepositContract() { + return depositContract; + } - public long getCurrentTime() { - return timeController.getTime(); - } + private static String getValidators(String info, Map records) { + if (records.isEmpty()) return ""; + return info + " [" + + records.entrySet().stream().map(e -> e.getKey().toString()).collect(Collectors.joining(",")) + + "]"; } public static class Builder { diff --git a/util/src/main/java/org/ethereum/beacon/schedulers/DefaultSchedulers.java b/util/src/main/java/org/ethereum/beacon/schedulers/DefaultSchedulers.java index 27d665eb6..8a7b701e5 100644 --- a/util/src/main/java/org/ethereum/beacon/schedulers/DefaultSchedulers.java +++ b/util/src/main/java/org/ethereum/beacon/schedulers/DefaultSchedulers.java @@ -3,15 +3,16 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; import java.util.function.Consumer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class DefaultSchedulers extends AbstractSchedulers { - private Consumer errorHandler = - t -> { - System.err.println("ErrorHandlingScheduler (default error handler):"); - t.printStackTrace(); - }; + private static final Logger logger = LogManager.getLogger(DefaultSchedulers.class); + + private Consumer errorHandler = t -> logger.error("Unhandled exception:", t); private volatile boolean started; public void setErrorHandler(Consumer errorHandler) { @@ -29,7 +30,17 @@ protected Scheduler createExecutorScheduler(ScheduledExecutorService executorSer @Override protected ScheduledExecutorService createExecutor(String namePattern, int threads) { started = true; - return Executors.newScheduledThreadPool( - threads, new ThreadFactoryBuilder().setDaemon(true).setNameFormat(namePattern).build()); + return Executors.newScheduledThreadPool(threads, createThreadFactory(namePattern)); + } + + protected ThreadFactory createThreadFactory(String namePattern) { + return createThreadFactoryBuilder(namePattern).build(); + } + + protected ThreadFactoryBuilder createThreadFactoryBuilder(String namePattern) { + return new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat(namePattern) + .setUncaughtExceptionHandler((thread, thr) -> errorHandler.accept(thr)); } } diff --git a/util/src/main/java/org/ethereum/beacon/schedulers/Scheduler.java b/util/src/main/java/org/ethereum/beacon/schedulers/Scheduler.java index e3d3c373f..b4d74da61 100644 --- a/util/src/main/java/org/ethereum/beacon/schedulers/Scheduler.java +++ b/util/src/main/java/org/ethereum/beacon/schedulers/Scheduler.java @@ -3,6 +3,7 @@ import java.time.Duration; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; /** * Analog for standard ScheduledExecutorService @@ -15,11 +16,26 @@ public interface Scheduler { CompletableFuture executeAtFixedRate(Duration initialDelay, Duration period, RunnableEx task); + default CompletableFuture executeR(Runnable task) { + return execute(task::run); + } + default CompletableFuture execute(RunnableEx task) { return execute(() -> {task.run(); return null;}); } + default CompletableFuture executeWithDelayR(Duration delay, Runnable task) { + return executeWithDelay(delay, task::run); + } + default CompletableFuture executeWithDelay(Duration delay, RunnableEx task) { return executeWithDelay(delay, () -> {task.run(); return null;}); } + + default CompletableFuture orTimeout(CompletableFuture future, Duration futureTimeout, Supplier exceptionSupplier) { + return (CompletableFuture) CompletableFuture.anyOf( + future, + executeWithDelay(futureTimeout, + () -> {throw exceptionSupplier.get();})); + } } diff --git a/util/src/main/java/org/ethereum/beacon/stream/RxUtil.java b/util/src/main/java/org/ethereum/beacon/stream/RxUtil.java new file mode 100644 index 000000000..32ee5d523 --- /dev/null +++ b/util/src/main/java/org/ethereum/beacon/stream/RxUtil.java @@ -0,0 +1,35 @@ +package org.ethereum.beacon.stream; + +import java.util.ArrayList; +import java.util.List; +import org.javatuples.Pair; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; + +public class RxUtil { + + + public static Publisher join(Publisher s1, Publisher s2, int bufferLen) { + throw new UnsupportedOperationException(); + } + + enum Op { + ADDED, REMOVED + } + + public static Flux> collect(Publisher addedStream, Publisher removedStream) { + return Flux.merge( + Flux.from(addedStream).map(e -> Pair.with(Op.ADDED, e)), + Flux.from(removedStream).map(e -> Pair.with(Op.REMOVED, e)) + ).scan(new ArrayList(), (arr, op) -> { + ArrayList ret = new ArrayList<>(arr); + if (op.getValue0() == Op.ADDED) { + ret.add(op.getValue1()); + } else { + ret.remove(op.getValue1()); + } + return ret; + }); + } + +} diff --git a/util/src/main/java/org/ethereum/beacon/stream/SimpleProcessor.java b/util/src/main/java/org/ethereum/beacon/stream/SimpleProcessor.java index cbd0b349c..2c803afc9 100644 --- a/util/src/main/java/org/ethereum/beacon/stream/SimpleProcessor.java +++ b/util/src/main/java/org/ethereum/beacon/stream/SimpleProcessor.java @@ -5,17 +5,20 @@ import org.reactivestreams.Subscription; import reactor.core.publisher.Flux; import reactor.core.publisher.FluxProcessor; +import reactor.core.publisher.FluxSink; import reactor.core.publisher.ReplayProcessor; import reactor.core.scheduler.Scheduler; public class SimpleProcessor implements Processor { FluxProcessor subscriber; + FluxSink sink; Flux publisher; boolean subscribed; public SimpleProcessor(Scheduler scheduler, String name) { ReplayProcessor processor = ReplayProcessor.cacheLast(); subscriber = processor; + sink = subscriber.sink(); publisher = Flux.from(processor) .publishOn(scheduler) .onBackpressureError() @@ -54,16 +57,16 @@ public void onSubscribe(Subscription subscription) { @Override public void onNext(T t) { - subscriber.onNext(t); + sink.next(t); } @Override public void onError(Throwable throwable) { - subscriber.onError(throwable); + sink.error(throwable); } @Override public void onComplete() { - subscriber.onComplete(); + sink.complete(); } } diff --git a/util/src/main/java/org/ethereum/beacon/util/Utils.java b/util/src/main/java/org/ethereum/beacon/util/Utils.java new file mode 100644 index 000000000..668498c23 --- /dev/null +++ b/util/src/main/java/org/ethereum/beacon/util/Utils.java @@ -0,0 +1,40 @@ +package org.ethereum.beacon.util; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import java.util.stream.Stream; + +public class Utils { + + public static Function, Stream> optionalFlatMap(Function func) { + return opt -> opt.map(a -> Stream.of(func.apply(a))).orElseGet(Stream::empty); + } + + public static Function> nullableFlatMap(Function func) { + return n -> n != null ? Stream.of(func.apply(n)) : Stream.empty(); + } + + public static void futureForward(CompletableFuture result, CompletableFuture forwardToFuture) { + result.whenComplete( + (res, t) -> { + if (t != null) { + forwardToFuture.completeExceptionally(t); + } else { + forwardToFuture.complete(res); + } + }); + } + + public static Set newLRUSet(int size) { + return Collections.newSetFromMap(new LinkedHashMap() { + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > size; + } + }); + } +} diff --git a/util/src/main/java/org/ethereum/beacon/util/cache/DebugCacheFactory.java b/util/src/main/java/org/ethereum/beacon/util/cache/DebugCacheFactory.java new file mode 100644 index 000000000..b7025d903 --- /dev/null +++ b/util/src/main/java/org/ethereum/beacon/util/cache/DebugCacheFactory.java @@ -0,0 +1,32 @@ +package org.ethereum.beacon.util.cache; + +import java.util.Optional; +import java.util.function.Function; + +/** + * Checks validity of cache entry on access + */ +public class DebugCacheFactory implements CacheFactory { + + @Override + public Cache createLRUCache(int capacity) { + return new Cache() { + + LRUCache cache = new LRUCache<>(capacity); + + @Override + public V get(K key, Function fallback) { + Optional cacheEntry = cache.getExisting(key); + if (cacheEntry.isPresent()) { + V goldenVal = fallback.apply(key); + if (!cacheEntry.get().equals(goldenVal)) { + throw new IllegalStateException("Cache broken: key=" + key + ", cacheEntry: " + cacheEntry.get() + ", but should be: " + goldenVal); + } + return goldenVal; + } else { + return cache.get(key, fallback); + } + } + }; + } +} diff --git a/util/src/main/java/org/ethereum/beacon/util/cache/LRUCache.java b/util/src/main/java/org/ethereum/beacon/util/cache/LRUCache.java index 4d1d94ba1..f8b71190a 100644 --- a/util/src/main/java/org/ethereum/beacon/util/cache/LRUCache.java +++ b/util/src/main/java/org/ethereum/beacon/util/cache/LRUCache.java @@ -3,6 +3,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import org.ethereum.beacon.util.cache.Cache; @@ -60,6 +61,10 @@ public V get(K key, Function fallback) { return result; } + public Optional getExisting(K key) { + return Optional.ofNullable(cacheData.get(key)); + } + public long getHits() { return hits.get(); } diff --git a/validator/src/main/java/org/ethereum/beacon/validator/MultiValidatorService.java b/validator/src/main/java/org/ethereum/beacon/validator/MultiValidatorService.java index c7d532441..2341b1c8d 100644 --- a/validator/src/main/java/org/ethereum/beacon/validator/MultiValidatorService.java +++ b/validator/src/main/java/org/ethereum/beacon/validator/MultiValidatorService.java @@ -361,7 +361,10 @@ private void propagateAttestation(Attestation attestation) { } private void subscribeToStateUpdates(Consumer payload) { - Flux.from(stateStream).subscribe(payload); + Flux.from(stateStream) + .doOnNext(payload) + .onErrorContinue((t,o) -> logger.warn("Validator error: ", t)) + .subscribe(); } @VisibleForTesting diff --git a/versions.gradle b/versions.gradle index 16fb5374b..d934da377 100644 --- a/versions.gradle +++ b/versions.gradle @@ -25,6 +25,7 @@ dependencyManagement { dependency "net.consensys.cava:cava-ssz:${cavaVersion}" dependency "net.consensys.cava:cava-units:${cavaVersion}" dependency "io.projectreactor:reactor-core:3.2.5.RELEASE" + dependency "io.projectreactor:reactor-test:3.2.5.RELEASE" dependency "org.reactivestreams:reactive-streams:1.0.2" dependency "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonVersion}" @@ -32,5 +33,6 @@ dependencyManagement { dependency "commons-beanutils:commons-beanutils:1.9.3" dependency "info.picocli:picocli:3.9.4" + dependency "io.netty:netty-all:4.1.36.Final" } } diff --git a/wire/build.gradle b/wire/build.gradle index 3eb7d631c..b319c2a5a 100644 --- a/wire/build.gradle +++ b/wire/build.gradle @@ -1,12 +1,26 @@ dependencies { implementation project(':types') implementation project(':core') + implementation project(':consensus') implementation project(':util') + implementation project(':ssz') + implementation project(':chain') + implementation project(':db:core') implementation 'com.google.guava:guava' implementation 'io.projectreactor:reactor-core' + implementation 'io.netty:netty-all' + implementation 'io.vertx:vertx-core' testImplementation 'org.mockito:mockito-core' + testImplementation 'io.projectreactor:reactor-test' + testImplementation project(':crypto') + testImplementation project(':validator') + testImplementation project(':pow:core') + testImplementation project(':start:common') + testImplementation project(':start:config') + testImplementation project(':start:simulator') + testImplementation project(':start:simulator').sourceSets.test.output // Gradle does not import test sources alongside with main sources // use a workaround until better solution will be found diff --git a/wire/src/main/java/org/ethereum/beacon/wire/Feedback.java b/wire/src/main/java/org/ethereum/beacon/wire/Feedback.java new file mode 100644 index 000000000..62bbe1faa --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/Feedback.java @@ -0,0 +1,94 @@ +package org.ethereum.beacon.wire; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; + +/** + * Wrapper for some asynchronous result for which the result consumer may leave a feedback. + * + * E.g. blocks downloaded from a remote peer are then later process and verified. The PeerManager + * may want to know if a peer sends invalid blocks and thus ban it. + * + * @param + */ +public interface Feedback { + + static Feedback of(T result) { + return new Impl<>(result); + } + + /** + * Return the wrapped value + */ + TResult get(); + + /** + * Report the value is OK + */ + void feedbackSuccess(); + + /** + * Report the value is erroneous + */ + void feedbackError(Throwable e); + + /** + * Creates a CompletableFuture which is done when feedback left + */ + CompletableFuture getFeedback(); + + /** + * Creates another Feedback with other value which forwards feedback to this instance + * @see #map(Function) + */ + Feedback delegate(TOtherResult otherResult); + + /** + * Convenient shortcut for {@link #delegate(Object)} method. + * Converts wrapped value to another Feedback wrapped value + */ + default Feedback map(Function mapper) { + return delegate(mapper.apply(get())); + } + + class Impl implements Feedback { + private final TResult result; + private final CompletableFuture feedback = new CompletableFuture<>(); + + private Impl(TResult result) { + this.result = result; + } + + @Override + public TResult get() { + return result; + } + + @Override + public void feedbackSuccess() { + feedback.complete(null); + } + + @Override + public void feedbackError(Throwable e) { + feedback.completeExceptionally(e); + } + + public CompletableFuture getFeedback() { + return feedback; + } + + @Override + public Feedback delegate(TOtherResult otherResult) { + Impl ret = new Impl<>(otherResult); + ret.getFeedback().whenComplete((v, t) -> { + if (t == null) { + feedbackSuccess(); + } else { + feedbackError(t); + } + }); + return ret; + } + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/LocalWireHub.java b/wire/src/main/java/org/ethereum/beacon/wire/LocalWireHub.java index 2c09d0d1b..98a63e099 100644 --- a/wire/src/main/java/org/ethereum/beacon/wire/LocalWireHub.java +++ b/wire/src/main/java/org/ethereum/beacon/wire/LocalWireHub.java @@ -14,7 +14,7 @@ import reactor.core.publisher.Flux; public class LocalWireHub { - private class WireImpl implements WireApi { + private class WireImpl implements WireApiSub { Processor blocks = DirectProcessor.create(); Processor attestations = DirectProcessor.create(); String name; @@ -95,11 +95,11 @@ public LocalWireHub(Consumer logger, Schedulers schedulers) { this.schedulers = schedulers; } - public WireApi createNewPeer(String name) { + public WireApiSub createNewPeer(String name) { return createNewPeer(name, 0, 0); } - public WireApi createNewPeer(String name, long inboundDelay, long outboundDelay) { + public WireApiSub createNewPeer(String name, long inboundDelay, long outboundDelay) { WireImpl ret = new WireImpl(name, inboundDelay, outboundDelay); peers.add(ret); return ret; diff --git a/wire/src/main/java/org/ethereum/beacon/wire/MessageSerializer.java b/wire/src/main/java/org/ethereum/beacon/wire/MessageSerializer.java new file mode 100644 index 000000000..30383a830 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/MessageSerializer.java @@ -0,0 +1,14 @@ +package org.ethereum.beacon.wire; + +import org.ethereum.beacon.wire.message.Message; +import tech.pegasys.artemis.util.bytes.BytesValue; + +/** + * Serialize/deserialize RPC messages envelop + */ +public interface MessageSerializer { + + BytesValue serialize(Message message); + + Message deserialize(BytesValue messageBytes); +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/Peer.java b/wire/src/main/java/org/ethereum/beacon/wire/Peer.java new file mode 100644 index 000000000..f021da60d --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/Peer.java @@ -0,0 +1,26 @@ +package org.ethereum.beacon.wire; + +import org.ethereum.beacon.wire.channel.Channel; +import tech.pegasys.artemis.util.bytes.BytesValue; + +/** + * Represent connected peer + */ +public interface Peer { + + /** + * Returns raw bytes {@link Channel} of this peer + */ + Channel getRawChannel(); + + /** + * Returns {@link WireApiSync} instance linked to this peer + */ + WireApiSync getSyncApi(); + + /** + * Returns {@link WireApiSub} instance linked to this peer + */ + WireApiSub getSubApi(); + +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/PeerImpl.java b/wire/src/main/java/org/ethereum/beacon/wire/PeerImpl.java new file mode 100644 index 000000000..966569496 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/PeerImpl.java @@ -0,0 +1,112 @@ +package org.ethereum.beacon.wire; + +import java.util.concurrent.CompletableFuture; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.ssz.SSZSerializer; +import org.ethereum.beacon.wire.channel.Channel; +import org.ethereum.beacon.wire.channel.beacon.BeaconPipeline; +import org.ethereum.beacon.wire.channel.beacon.WireApiSubAdapter; +import org.ethereum.beacon.wire.message.payload.GoodbyeMessage; +import org.ethereum.beacon.wire.message.payload.HelloMessage; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public class PeerImpl implements Peer { + + private final Channel channel; + private final WireApiPeer apiPeerRemote; + private final HelloMessage localHelloMessage; + private final CompletableFuture remoteHelloMessageFut = new CompletableFuture<>(); + private final CompletableFuture peerActiveFut = new CompletableFuture<>(); + private final BeaconPipeline beaconPipeline; + private final WireApiSubAdapter wireApiSubAdapter = new WireApiSubAdapter(); + private final Schedulers schedulers; + + private GoodbyeMessage remoteGoodbye; + private GoodbyeMessage localGoodbye; + + public PeerImpl( + Channel channel, + HelloMessage helloMessage, + SSZSerializer ssz, + MessageSerializer messageSerializer, + WireApiSync syncServer, + Schedulers schedulers) { + + this.channel = channel; + this.localHelloMessage = helloMessage; + this.schedulers = schedulers; + + beaconPipeline = new BeaconPipeline(ssz, new WireApiPeer() { + public void hello(HelloMessage message) { + onHello(message); + } + public void goodbye(GoodbyeMessage message) { + onGoodbye(message); + } + }, wireApiSubAdapter, syncServer, schedulers); + beaconPipeline.initFromBytesChannel(channel, messageSerializer); + wireApiSubAdapter.setSubClient(beaconPipeline.getSubClient()); + + apiPeerRemote = beaconPipeline.getPeerClient(); + apiPeerRemote.hello(helloMessage); + } + + @Override + public Channel getRawChannel() { + return channel; + } + + public CompletableFuture getRemoteHelloMessage() { + return remoteHelloMessageFut; + } + + public CompletableFuture getPeerActiveFuture() { + return peerActiveFut; + } + + private void onHello(HelloMessage message) { + remoteHelloMessageFut.complete(message); + + if (localHelloMessage.getNetworkId() != message.getNetworkId()) { + disconnect(new GoodbyeMessage(GoodbyeMessage.IRRELEVANT_NETWORK)); + } + if (!localHelloMessage.getChainId().equals(message.getChainId())) { + disconnect(new GoodbyeMessage(GoodbyeMessage.IRRELEVANT_NETWORK)); + } + + peerActiveFut.complete(message); + } + + private void onGoodbye(GoodbyeMessage message) { + remoteGoodbye = message; + } + + public void disconnect(GoodbyeMessage message) { + localGoodbye = message; + apiPeerRemote.goodbye(message); + channel.close(); + } + + @Override + public WireApiSync getSyncApi() { + return beaconPipeline.getSyncClient(); + } + + @Override + public WireApiSub getSubApi() { + return wireApiSubAdapter; + } + + @Override + public String toString() { + String bestSlot; + try { + bestSlot = + getRemoteHelloMessage().isDone() ? getRemoteHelloMessage().get().getBestSlot().toString() : null; + } catch (Exception e) { + bestSlot = "(err )" + e; + } + return "Peer[" + channel + (bestSlot == null ? "" : ", slot: " + bestSlot) + "]"; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/PeerManager.java b/wire/src/main/java/org/ethereum/beacon/wire/PeerManager.java new file mode 100644 index 000000000..3f71033ae --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/PeerManager.java @@ -0,0 +1,48 @@ +package org.ethereum.beacon.wire; + +import java.util.List; +import org.ethereum.beacon.stream.RxUtil; +import org.reactivestreams.Publisher; + +/** + * Manages connected peers and aggregates their `high-level` APIs + */ +public interface PeerManager { + + /** + * Stream of new peer connections + */ + Publisher connectedPeerStream(); + + /** + * Stream of peer disconnects + * Peer must occur in this stream strictly after occurring in the {@link #connectedPeerStream()} + */ + Publisher disconnectedPeerStream(); + + /** + * Steam of new active peers which are connected and handshake was done. + * A peer appearing in this stream is available for 'high-level' APIs calls + */ + Publisher activatedPeerStream(); + + /** + * Stream of currently active peers list + */ + default Publisher> activePeersStream() { + return RxUtil.collect(activatedPeerStream(), disconnectedPeerStream()); + } + + /** + * Returns WireApiSync instance which is the aggregation of all connected peer APIs + * When currently no active peers the API enqueues invocations and execute them upon any + * active peer connected + */ + WireApiSync getWireApiSync(); + + /** + * Returns WireApiSub instance which is the aggregation of all connected peer APIs. + * When currently no active peers the instance just ignores outbound notifications. + */ + WireApiSub getWireApiSub(); +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/SimplePeerManagerImpl.java b/wire/src/main/java/org/ethereum/beacon/wire/SimplePeerManagerImpl.java new file mode 100644 index 000000000..0fe23f267 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/SimplePeerManagerImpl.java @@ -0,0 +1,126 @@ +package org.ethereum.beacon.wire; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.chain.BeaconTupleDetails; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.ssz.SSZSerializer; +import org.ethereum.beacon.wire.channel.Channel; +import org.ethereum.beacon.wire.message.payload.HelloMessage; +import org.ethereum.beacon.wire.sync.WireApiSyncRouter; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +public class SimplePeerManagerImpl implements PeerManager { + private static final Logger logger = LogManager.getLogger(SimplePeerManagerImpl.class); + + private final byte networkId; + private final UInt64 chainId; + + private final Publisher> channelsStream; + private final SSZSerializer ssz; + private final BeaconChainSpec spec; + private final MessageSerializer messageSerializer; + private final Schedulers schedulers; + private final WireApiSync syncServer; + private final Publisher headStream; + private final WireApiSyncRouter wireApiSyncRouter; + private final WireApiSubRouter wireApiSubRouter; + + + private final Flux connectedPeersStream; + private final List activePeers = Collections.synchronizedList(new ArrayList<>()); + + public SimplePeerManagerImpl( + byte networkId, + UInt64 chainId, + Publisher> channelsStream, + SSZSerializer ssz, + BeaconChainSpec spec, + MessageSerializer messageSerializer, + Schedulers schedulers, + WireApiSync syncServer, + Publisher headStream) { + + this.networkId = networkId; + this.chainId = chainId; + this.channelsStream = channelsStream; + this.ssz = ssz; + this.spec = spec; + this.messageSerializer = messageSerializer; + this.schedulers = schedulers; + this.syncServer = syncServer; + this.headStream = headStream; + + connectedPeersStream = Flux.from(channelsStream) + .map(this::createPeer) + .replay(1).autoConnect(); + + Flux.from(activatedPeerStream()).subscribe(this::onNewActivePeer); + + wireApiSyncRouter = new WireApiSyncRouter( + Flux.from(activatedPeerStream()).map(Peer::getSyncApi), + Flux.from(disconnectedPeerStream()).map(Peer::getSyncApi)); + + wireApiSubRouter = new WireApiSubRouter( + Flux.from(activatedPeerStream()).map(Peer::getSubApi), + Flux.from(disconnectedPeerStream()).map(Peer::getSubApi)); + } + + protected HelloMessage createLocalHello() { + BeaconTupleDetails head = Mono.from(headStream).block(Duration.ofSeconds(10)); // TODO + return new HelloMessage( + networkId, + chainId, + head.getFinalState().getFinalizedRoot(), + head.getFinalState().getFinalizedEpoch(), + spec.getObjectHasher().getHashTruncateLast(head.getBlock()), + head.getBlock().getSlot()); + } + + protected PeerImpl createPeer(Channel channel) { + logger.info("Creating a peer from new channel: " + channel); + return new PeerImpl(channel, createLocalHello(), ssz, messageSerializer, syncServer, schedulers); + } + + @Override + public Publisher connectedPeerStream() { + return connectedPeersStream.map(p -> p); + } + + @Override + public Publisher disconnectedPeerStream() { + return connectedPeersStream.flatMap( + peer -> Mono.fromFuture(peer.getRawChannel().getCloseFuture().thenApply(v -> peer))); + } + + @Override + public Publisher activatedPeerStream() { + return connectedPeersStream.flatMap( + peer -> Mono.fromFuture(peer.getPeerActiveFuture().thenApply(v -> peer))); + } + + protected void onNewActivePeer(Peer peer) { + logger.info("New active peer: " + peer); + activePeers.add(peer); + peer.getRawChannel().getCloseFuture().thenAccept(v -> activePeers.remove(peer)); + } + + @Override + public WireApiSync getWireApiSync() { + return wireApiSyncRouter; + } + + @Override + public WireApiSub getWireApiSub() { + return wireApiSubRouter; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/WireApi.java b/wire/src/main/java/org/ethereum/beacon/wire/WireApi.java deleted file mode 100644 index 2ab40b2f7..000000000 --- a/wire/src/main/java/org/ethereum/beacon/wire/WireApi.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.ethereum.beacon.wire; - -import org.ethereum.beacon.core.BeaconBlock; -import org.ethereum.beacon.core.operations.Attestation; -import org.reactivestreams.Publisher; - -public interface WireApi { - - void sendProposedBlock(BeaconBlock block); - - void sendAttestation(Attestation attestation); - - Publisher inboundBlocksStream(); - - Publisher inboundAttestationsStream(); -} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/WireApiPeer.java b/wire/src/main/java/org/ethereum/beacon/wire/WireApiPeer.java new file mode 100644 index 000000000..37c4507b1 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/WireApiPeer.java @@ -0,0 +1,15 @@ +package org.ethereum.beacon.wire; + +import org.ethereum.beacon.wire.message.payload.GoodbyeMessage; +import org.ethereum.beacon.wire.message.payload.HelloMessage; + +/** + * Represents synchronous interface for peers 'low level' interaction + */ +public interface WireApiPeer { + + void hello(HelloMessage message); + + void goodbye(GoodbyeMessage message); + +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/WireApiSub.java b/wire/src/main/java/org/ethereum/beacon/wire/WireApiSub.java new file mode 100644 index 000000000..aaa6fa787 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/WireApiSub.java @@ -0,0 +1,33 @@ +package org.ethereum.beacon.wire; + +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.reactivestreams.Publisher; + +/** + * Represents asynchronous wire interface for subscription-like messages + */ +public interface WireApiSub { + + /** + * Sends a new block to remote peer(s) + */ + void sendProposedBlock(BeaconBlock block); + + /** + * Sends a new attestation to remote peer(s) + */ + void sendAttestation(Attestation attestation); + + /** + * Stream of new blocks from remote peer(s) + * This stream must be distinct, i.e. doesn't contain duplicate blocks + */ + Publisher inboundBlocksStream(); + + /** + * Stream of new attestations from remote peer(s) + * This stream must be distinct, i.e. doesn't contain duplicate attestations + */ + Publisher inboundAttestationsStream(); +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/WireApiSubRouter.java b/wire/src/main/java/org/ethereum/beacon/wire/WireApiSubRouter.java new file mode 100644 index 000000000..f3ed98232 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/WireApiSubRouter.java @@ -0,0 +1,121 @@ +package org.ethereum.beacon.wire; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.function.Function; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.stream.RxUtil; +import org.ethereum.beacon.util.Utils; +import org.javatuples.Pair; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; + +/** + * Tracks and aggregates {@link WireApiSub} instances from separate peers + * The class realizes flood pub strategy for notifications propagation + */ +public class WireApiSubRouter implements WireApiSub { + private static final int DUPLICATE_DETECTION_SET_SIZE = 64; + + private final Flux inboundAttestationsStream; + private final Flux inboundBlocksStream; + private FluxSink localAttestationSink; + private FluxSink localBlocksSink; + + private volatile List activeApis = Collections.emptyList(); + + Set seenBlocks; + + public WireApiSubRouter( + Publisher addedPeersStream, + Publisher removedPeersStream) { + + RxUtil.collect(addedPeersStream, removedPeersStream) + .subscribe(l -> activeApis = l); + + // flood pub realization: upon receiving new block from remote or local + // broadcasting it to all except sender + // also filtering already known blocks + Flux localBlocks = Flux.create(e -> localBlocksSink = e); + + Flux> allNewBlocks = Flux.from(addedPeersStream) + .flatMap(api -> Flux.from(api.inboundBlocksStream()).map(block -> new ApiData<>(api, block))) + .mergeWith(localBlocks.map(block -> new ApiData<>(this, block))) + .distinct(ApiData::getData, () -> seenBlocks = Utils.newLRUSet(DUPLICATE_DETECTION_SET_SIZE)); + + allNewBlocks.subscribe( + p -> { + activeApis + .stream() + .filter(api -> !api.equals(p.getApi())) + .forEach(api -> api.sendProposedBlock(p.getData())); + }); + + // the same flood pub for attestations + Flux localAttestations = Flux.create(e -> localAttestationSink = e); + + Flux> allNewAttestations = Flux.from(addedPeersStream) + .flatMap(api -> Flux.from(api.inboundAttestationsStream()).map(attest -> new ApiData<>(api, attest))) + .mergeWith(localAttestations.map(attest -> new ApiData<>(this, attest))) + .distinct(ApiData::getData, () -> Utils.newLRUSet(DUPLICATE_DETECTION_SET_SIZE)); + + allNewAttestations.subscribe( + p -> { + activeApis + .stream() + .filter(api -> !api.equals(p.getApi())) + .forEach(api -> api.sendAttestation(p.getData())); + }); + + inboundBlocksStream = Flux.from(addedPeersStream) + .flatMap(WireApiSub::inboundBlocksStream) + .distinct(Function.identity(), () -> Utils.newLRUSet(DUPLICATE_DETECTION_SET_SIZE)); + + inboundAttestationsStream = Flux + .from(addedPeersStream) + .flatMap(WireApiSub::inboundAttestationsStream) + .distinct(Function.identity(), () -> Utils.newLRUSet(DUPLICATE_DETECTION_SET_SIZE)); + + } + + @Override + public void sendProposedBlock(BeaconBlock block) { + localBlocksSink.next(block); + } + + @Override + public void sendAttestation(Attestation attestation) { + localAttestationSink.next(attestation); + } + + @Override + public Publisher inboundBlocksStream() { + return inboundBlocksStream; + } + + @Override + public Publisher inboundAttestationsStream() { + return inboundAttestationsStream; + } + + static class ApiData { + private final WireApiSub api; + private final T block; + + public ApiData(WireApiSub api, T block) { + this.api = api; + this.block = block; + } + + public WireApiSub getApi() { + return api; + } + + public T getData() { + return block; + } + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/WireApiSync.java b/wire/src/main/java/org/ethereum/beacon/wire/WireApiSync.java new file mode 100644 index 000000000..e61690b57 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/WireApiSync.java @@ -0,0 +1,77 @@ +package org.ethereum.beacon.wire; + +import static org.ethereum.beacon.util.Utils.optionalFlatMap; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import org.ethereum.beacon.consensus.hasher.ObjectHasher; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.BeaconBlockBody; +import org.ethereum.beacon.core.BeaconBlockHeader; +import org.ethereum.beacon.wire.message.payload.BlockBodiesRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockBodiesResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockHeadersResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockHeadersRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsResponseMessage; +import tech.pegasys.artemis.ethereum.core.Hash32; + +/** + * Asynchronous wire interface for downloading blockchain sync data from remote peer(s) + */ +public interface WireApiSync { + int MAX_BLOCK_ROOTS_COUNT = 32768; + + /** + * Requests block roots from remote peer(s) + */ + CompletableFuture requestBlockRoots( + BlockRootsRequestMessage requestMessage); + + /** + * Requests block headers from remote peer(s) + */ + CompletableFuture requestBlockHeaders( + BlockHeadersRequestMessage requestMessage); + + /** + * Requests block bodies from remote peer(s) + */ + CompletableFuture> requestBlockBodies( + BlockBodiesRequestMessage requestMessage); + + /** + * Handy shortcut to download headers+bodies + */ + default CompletableFuture>> requestBlocks( + BlockHeadersRequestMessage requestMessage, ObjectHasher hasher) { + + CompletableFuture> headersFuture = requestBlockHeaders( + requestMessage).thenApply(BlockHeadersResponseMessage::getHeaders); + + CompletableFuture>> bodiesFuture = + headersFuture.thenCompose( + headers -> { + List blockHashes = + headers.stream().map(hasher::getHashTruncateLast).collect(Collectors.toList()); + return requestBlockBodies(new BlockBodiesRequestMessage(blockHashes)) + .thenApply(bb -> bb.map(BlockBodiesResponseMessage::getBlockBodies)); + }); + + return headersFuture.thenCombine( + bodiesFuture, + (headers, bodies) -> { + Map bodyMap = + bodies.get().stream().collect(Collectors.toMap(hasher::getHash, b -> b, (b1, b2) -> b1)); + return bodies.delegate( + headers.stream() + .map(h -> Optional.ofNullable(bodyMap.get(h.getBlockBodyRoot())) + .map(body -> new BeaconBlock(h, body))) + .flatMap(optionalFlatMap(b -> b)) + .collect(Collectors.toList())); + }); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/WireApiSyncServer.java b/wire/src/main/java/org/ethereum/beacon/wire/WireApiSyncServer.java new file mode 100644 index 000000000..1b19e0c2c --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/WireApiSyncServer.java @@ -0,0 +1,117 @@ +package org.ethereum.beacon.wire; + +import static org.ethereum.beacon.util.Utils.optionalFlatMap; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import org.ethereum.beacon.chain.storage.BeaconChainStorage; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.BeaconBlockBody; +import org.ethereum.beacon.core.BeaconBlockHeader; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.wire.exceptions.WireIllegalArgumentsException; +import org.ethereum.beacon.wire.message.payload.BlockBodiesRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockBodiesResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockHeadersRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockHeadersResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsResponseMessage.BlockRootSlot; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.uint.UInt64; + +/** + * Serves {@link WireApiSync} requests supplying local blockchain information to remote party + */ +public class WireApiSyncServer implements WireApiSync { + + private final BeaconChainStorage storage; + + public WireApiSyncServer(BeaconChainStorage storage) { + this.storage = storage; + } + + @Override + public CompletableFuture requestBlockRoots( + BlockRootsRequestMessage requestMessage) { + CompletableFuture ret = new CompletableFuture<>(); + if (requestMessage.getCount().compareTo(UInt64.valueOf(MAX_BLOCK_ROOTS_COUNT)) > 0) { + ret.completeExceptionally(new WireIllegalArgumentsException( + "Too many block roots requested: " + requestMessage.getCount())); + } else { + List roots = new ArrayList<>(); + for (SlotNumber slot : requestMessage.getStartSlot().iterateTo( + requestMessage.getStartSlot().plus(requestMessage.getCount()))) { + List slotRoots = storage.getBlockStorage().getSlotBlocks(slot); + for (Hash32 slotRoot : slotRoots) { + roots.add(new BlockRootSlot(slotRoot, slot)); + } + } + ret.complete(new BlockRootsResponseMessage(roots)); + } + return ret; + } + + @Override + public CompletableFuture requestBlockHeaders( + BlockHeadersRequestMessage requestMessage) { + CompletableFuture ret = new CompletableFuture<>(); + + SlotNumber slot; + if (!BlockHeadersRequestMessage.NULL_START_SLOT.equals(requestMessage.getStartSlot())) { + slot = requestMessage.getStartSlot(); + } else { + Optional blockOpt = storage.getBlockStorage().get(requestMessage.getStartRoot()); + slot = blockOpt.map(BeaconBlock::getSlot).orElse(null); + } + + if (slot != null) { + List headers = new ArrayList<>(); + int increment = requestMessage.getSkipSlots().getIntValue() + 1; + SlotNumber maxSlot = storage.getBlockStorage().getMaxSlot(); + SlotNumber prevSlot = SlotNumber.ZERO; + for(int i = 0; i < requestMessage.getMaxHeaders().intValue(); i++) { + if (slot.greater(maxSlot)) { + break; + } + List slotBlocks = Collections.emptyList(); + SlotNumber nonEmptySlot = slot; + while (nonEmptySlot.greater(prevSlot)) { + slotBlocks = storage.getBlockStorage().getSlotBlocks(nonEmptySlot); + if (!slotBlocks.isEmpty()) { + break; + } + nonEmptySlot = nonEmptySlot.decrement(); + } + + if (nonEmptySlot.greater(prevSlot)) { + headers.add(storage.getBlockHeaderStorage().get(slotBlocks.get(0)).get()); + } + slot = slot.plus(increment); + prevSlot = nonEmptySlot; + } + ret.complete(new BlockHeadersResponseMessage(headers)); + } else { + ret.complete(new BlockHeadersResponseMessage(Collections.emptyList())); + } + return ret; + } + + + + @Override + public CompletableFuture> requestBlockBodies( + BlockBodiesRequestMessage requestMessage) { + + List bodyList = requestMessage.getBlockTreeRoots().stream() + .map(blockRoot -> storage.getBlockStorage().get(blockRoot)) + .flatMap(optionalFlatMap(BeaconBlock::getBody)) + .collect(Collectors.toList()); + return CompletableFuture.completedFuture( + Feedback.of(new BlockBodiesResponseMessage(bodyList))); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/Channel.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/Channel.java new file mode 100644 index 000000000..71a91d4a0 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/Channel.java @@ -0,0 +1,46 @@ +package org.ethereum.beacon.wire.channel; + +import java.util.concurrent.CompletableFuture; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Mono; + +/** + * Duplex channel handling one (inbound) or two (inbound/outbound) streams of abstract messages + * The channel is assumed closed when {@link #inboundMessageStream()} is in Complete + * state. + */ +public interface Channel { + + /** + * Returns the steam of inbound messages. When the stream completes the channel is assumed + * closed. + * The publisher returned must cache all messages and flush them upon the first subscription. + * It may handle several subscribers and it's upon implementation what past messages + * to replay to later subscribers + */ + Publisher inboundMessageStream(); + + /** + * This method is called if the client wants to send any messages to this channel. + * Normally the outboundMessageStream is immediately subscribed to during the call + * When this method called more than than once the behaviour is not specified + * but advanced implementations may merge messages from different publishers + */ + void subscribeToOutbound(Publisher outboundMessageStream); + + /** + * Returns the future which completes when this channel is closed. + * This default implementation just subscribes to {@link #inboundMessageStream()} and + * waits for it to complete. Implementing classes may override it for more effective implementation. + */ + default CompletableFuture getCloseFuture() { + return Mono.ignoreElements(inboundMessageStream()).toFuture().thenApply(ignore -> null); + } + + /** + * Closes this channel. {@link #inboundMessageStream()} will complete + * synchronously/asynchronously during/after this call. + */ + default void close() { + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/ChannelCodec.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/ChannelCodec.java new file mode 100644 index 000000000..6c26434f6 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/ChannelCodec.java @@ -0,0 +1,35 @@ +package org.ethereum.beacon.wire.channel; + +import java.util.function.Function; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; + +public class ChannelCodec implements ChannelOp { + + private final Channel inChannel; + private final Function messageDecoder; + private final Function messageEncoder; + + public ChannelCodec(Channel inChannel, + Function messageDecoder, + Function messageEncoder) { + this.inChannel = inChannel; + this.messageDecoder = messageDecoder; + this.messageEncoder = messageEncoder; + } + + @Override + public Publisher inboundMessageStream() { + return Flux.from(inChannel.inboundMessageStream()).map(messageDecoder); + } + + @Override + public void subscribeToOutbound(Publisher outboundMessageStream) { + inChannel.subscribeToOutbound(Flux.from(outboundMessageStream).map(messageEncoder)); + } + + @Override + public Channel getInChannel() { + return inChannel; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/ChannelFilter.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/ChannelFilter.java new file mode 100644 index 000000000..51198c405 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/ChannelFilter.java @@ -0,0 +1,30 @@ +package org.ethereum.beacon.wire.channel; + +import java.util.function.Predicate; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; + +public class ChannelFilter implements ChannelOp { + private final Channel inChannel; + private final Predicate filter; + + public ChannelFilter(Channel inChannel, Predicate filter) { + this.inChannel = inChannel; + this.filter = filter; + } + + @Override + public Channel getInChannel() { + return inChannel; + } + + @Override + public Publisher inboundMessageStream() { + return Flux.from(getInChannel().inboundMessageStream()).filter(filter); + } + + @Override + public void subscribeToOutbound(Publisher outboundMessageStream) { + getInChannel().subscribeToOutbound(Flux.from(outboundMessageStream).filter(filter)); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/ChannelHub.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/ChannelHub.java new file mode 100644 index 000000000..8c38d348c --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/ChannelHub.java @@ -0,0 +1,36 @@ +package org.ethereum.beacon.wire.channel; + +import org.reactivestreams.Publisher; +import reactor.core.publisher.ConnectableFlux; +import reactor.core.publisher.Flux; + +public class ChannelHub implements Channel { + + Channel inChannel; + Flux inMessagePublisher; + ConnectableFlux connectableFlux; + + public ChannelHub(Channel inChannel, boolean autoConnect) { + this.inChannel = inChannel; + connectableFlux = Flux.from(inChannel.inboundMessageStream()).publish(); + if (autoConnect) { + inMessagePublisher = connectableFlux.autoConnect(); + } else { + inMessagePublisher = connectableFlux; + } + } + + public void connect() { + connectableFlux.connect(); + } + + @Override + public Publisher inboundMessageStream() { + return inMessagePublisher; + } + + @Override + public void subscribeToOutbound(Publisher outboundMessageStream) { + inChannel.subscribeToOutbound(outboundMessageStream); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/ChannelOp.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/ChannelOp.java new file mode 100644 index 000000000..dd9037552 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/ChannelOp.java @@ -0,0 +1,14 @@ +package org.ethereum.beacon.wire.channel; + +/** + * Represents {@link Channel} operation which has inbound {@link Channel} with messages of + * type TInMessage, maps these messages to TOutMessage type + * and serves them as a {@link Channel} itself + */ +public interface ChannelOp extends Channel { + + /** + * Returns the source {@link Channel} + */ + Channel getInChannel(); +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/IdentityChannel.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/IdentityChannel.java new file mode 100644 index 000000000..c10712db4 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/IdentityChannel.java @@ -0,0 +1,34 @@ +package org.ethereum.beacon.wire.channel; + +import java.util.function.Predicate; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; + +public abstract class IdentityChannel implements ChannelOp { + private final Channel inChannel; + + public IdentityChannel(Channel inChannel) { + this.inChannel = inChannel; + } + + @Override + public Channel getInChannel() { + return inChannel; + } + + @Override + public Publisher inboundMessageStream() { + return Flux.from(getInChannel().inboundMessageStream()) + .doOnNext(this::onInbound); + } + + @Override + public void subscribeToOutbound(Publisher outboundMessageStream) { + getInChannel().subscribeToOutbound(Flux.from(outboundMessageStream) + .doOnNext(this::onOutbound)); + } + + protected void onInbound(TMessage msg) throws RuntimeException {} + + protected void onOutbound(TMessage msg)throws RuntimeException {} +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcChannel.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcChannel.java new file mode 100644 index 000000000..45a323e46 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcChannel.java @@ -0,0 +1,21 @@ +package org.ethereum.beacon.wire.channel; + +import org.reactivestreams.Publisher; + +public interface RpcChannel extends Channel> { + + static RpcChannel from(Channel> channel) { + return new RpcChannel() { + @Override + public Publisher> inboundMessageStream() { + return channel.inboundMessageStream(); + } + + @Override + public void subscribeToOutbound( + Publisher> outboundMessageStream) { + channel.subscribeToOutbound(outboundMessageStream); + } + } ; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcChannelAdapter.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcChannelAdapter.java new file mode 100644 index 000000000..26530b82e --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcChannelAdapter.java @@ -0,0 +1,125 @@ +package org.ethereum.beacon.wire.channel; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.wire.exceptions.WireException; +import org.ethereum.beacon.wire.exceptions.WireRpcClosedException; +import org.ethereum.beacon.wire.exceptions.WireRpcTimeoutException; +import reactor.core.publisher.DirectProcessor; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; + +public class RpcChannelAdapter { + private static final Logger logger = LogManager.getLogger(RpcChannelAdapter.class); + + private static final Object CONTEXT_KEY_FUTURE = new Object(); + public static final Duration DEFAULT_RPC_TIMEOUT = Duration.ofSeconds(10); + + private final Scheduler timeoutScheduler; + private Duration rpcCallTimeout = DEFAULT_RPC_TIMEOUT; + private final RpcChannel inChannel; + private final Function> serverHandler; + private FluxSink> outboundSink; + private volatile boolean closed; + + + public RpcChannelAdapter(RpcChannel inChannel, + Function> serverHandler, + Scheduler timeoutScheduler) { + this.inChannel = inChannel; + this.serverHandler = serverHandler; + this.timeoutScheduler = timeoutScheduler; + inChannel.subscribeToOutbound( + Flux.>create(s -> outboundSink = s) + .publish(1) + .autoConnect()); + Flux.from(inChannel.inboundMessageStream()) + .subscribe(this::onInbound, err -> logger.warn("Unexpected error", err), this::onClose); + } + + public RpcChannelAdapter withRpcCallTimeout( + Duration rpcCallTimeout) { + this.rpcCallTimeout = rpcCallTimeout; + return this; + } + + private void onClose() { + closed = true; + } + + private void onInbound(RpcMessage msg) { + if (msg.isRequest()) { + if (msg.isNotification()) { + handleNotify(msg.getRequest()); + } else { + handleInvoke(msg); + } + } else { + handleResponse(msg); + } + } + + private void handleResponse(RpcMessage msg) { + CompletableFuture respFut = + (CompletableFuture) msg.getRequestContext(CONTEXT_KEY_FUTURE); + + if (msg.getResponse().isPresent()) { + respFut.complete(msg.getResponse().get()); + } else { + respFut.completeExceptionally(msg.getError().get()); + } + } + + private void handleNotify(TRequestMessage msg) { + if (serverHandler != null) { + serverHandler.apply(msg); + } + } + + private void handleInvoke(RpcMessage msg) { + try { + if (serverHandler == null) { + throw new WireException("No server to process RPC invoke: " + msg); + } + CompletableFuture fut = serverHandler.apply(msg.getRequest()); + fut.whenComplete( + (r, t) -> + outboundSink.next( + t != null ? msg.copyWithResponseError(t) : msg.copyWithResponse(r))) + .whenComplete( + (r, t) -> { + if (t != null) t.printStackTrace(); + }); + } catch (Exception e) { + outboundSink.next(msg.copyWithResponseError(e)); + } + } + + public CompletableFuture invokeRemote(TRequestMessage request) { + CompletableFuture ret = new CompletableFuture<>(); + + if(closed) { + ret.completeExceptionally(new WireRpcClosedException("Channel already closed: " + inChannel)); + return ret; + } else { + RpcMessage requestRpcMsg = + new RpcMessage<>(request, false); + requestRpcMsg.setRequestContext(CONTEXT_KEY_FUTURE, ret); + outboundSink.next(requestRpcMsg); + return timeoutScheduler.orTimeout( + ret, + rpcCallTimeout, + () -> closed + ? new WireRpcClosedException("Channel was closed during call execution") + : new WireRpcTimeoutException("RPC call timeout.")); + } + } + + public void notifyRemote(TRequestMessage request) { + outboundSink.next(new RpcMessage<>(request, true)); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcChannelClassFilter.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcChannelClassFilter.java new file mode 100644 index 000000000..3b35a9ff6 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcChannelClassFilter.java @@ -0,0 +1,47 @@ +package org.ethereum.beacon.wire.channel; + +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; + +public class RpcChannelClassFilter + implements ChannelOp, RpcMessage>, + RpcChannel { + + private static final Object CONTEXT_KEY_REQ_CLASS = new Object(); + + private final RpcChannel inChannel; + private final Class requestMessageClass; + + public RpcChannelClassFilter(RpcChannel inChannel, Class requestMessageClass) { + this.inChannel = inChannel; + this.requestMessageClass = requestMessageClass; + } + + @Override + public Publisher> inboundMessageStream() { + return Flux.from(inChannel.inboundMessageStream()) + .filter(rpcMsg -> rpcMsg.isRequest() && requestMessageClass.isInstance(rpcMsg.getRequest()) + || rpcMsg.isResponse() && requestMessageClass == rpcMsg.getRequestContext(CONTEXT_KEY_REQ_CLASS)) + .map(msg -> (RpcMessage) msg); + } + + @Override + public void subscribeToOutbound(Publisher> outboundMessageStream) { + inChannel.subscribeToOutbound(Flux.from(outboundMessageStream) + .doOnNext(rpcMsg -> { + if (rpcMsg.isRequest()) { + if (!requestMessageClass.isInstance(rpcMsg.getRequest())) { + throw new IllegalArgumentException("Invalid request class: " + rpcMsg.getRequest().getClass() + ", expected " + requestMessageClass); + } + rpcMsg.setRequestContext(CONTEXT_KEY_REQ_CLASS, requestMessageClass); + } + }) + .map(msg -> (RpcMessage) msg) + ); + } + + @Override + public Channel> getInChannel() { + return inChannel; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcChannelMapper.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcChannelMapper.java new file mode 100644 index 000000000..a9bd0752c --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcChannelMapper.java @@ -0,0 +1,86 @@ +package org.ethereum.beacon.wire.channel; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.ethereum.beacon.wire.exceptions.WireException; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; + +public abstract class RpcChannelMapper + implements RpcChannel, + ChannelOp> { + + private static final Object CONTEXT_ID_KEY = new Object(); + + private final Map> idToContextMap = new ConcurrentHashMap<>(); + private final Channel inChannel; + + protected RpcChannelMapper(Channel inChannel) { + this.inChannel = inChannel; + } + + @Override + public Publisher> inboundMessageStream() { + return Flux.from(inChannel.inboundMessageStream()).map(this::fromIn); + } + + protected RpcMessage fromIn(TInMessage msg) { + if (isRequest(msg)) { + if (isNotification(msg)) { + return new RpcMessage((TOutRequest) msg, true); + } else { + RpcMessage rpcMessage = new RpcMessage<>( + (TOutRequest) msg, false); + rpcMessage.setRequestContext(CONTEXT_ID_KEY, getId(msg)); + return rpcMessage; + } + } else { + Object id = getId(msg); + Map requestContext = idToContextMap.remove(id); + if (requestContext == null) { + throw new WireException("Invalid response from remote: can't find request ID: " + id + ", " + msg); + } + RpcMessage rpcMessage = new RpcMessage<>(null, (TOutResponse) msg); + rpcMessage.getRequestContext().putAll(requestContext); + return rpcMessage; + } + } + + protected TInMessage toIn(RpcMessage msg) { + if (msg.isRequest()) { + TInMessage inMessage = (TInMessage) msg.getRequest(); + Object id = generateNextId(); + setId(inMessage, id); + if (msg.isNotification()) { + return (TInMessage) msg.getRequest(); + } else { + idToContextMap.put(id, msg.getRequestContext()); + return inMessage; + } + } else { + TInMessage inMessage = (TInMessage) msg.getResponse().get(); + setId(inMessage, msg.getRequestContext(CONTEXT_ID_KEY)); + return inMessage; + } + } + + @Override + public void subscribeToOutbound(Publisher> outboundMessageStream) { + inChannel.subscribeToOutbound(Flux.from(outboundMessageStream).map(this::toIn)); + } + + public Channel getInChannel() { + return inChannel; + } + + protected abstract boolean isRequest(TInMessage msg); + + protected abstract boolean isNotification(TInMessage msg); + + protected abstract Object generateNextId(); + + protected abstract Object getId(TInMessage msg); + + protected abstract void setId(TInMessage msg, Object id); + +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcMessage.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcMessage.java new file mode 100644 index 000000000..9e7b2db1d --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/RpcMessage.java @@ -0,0 +1,104 @@ +package org.ethereum.beacon.wire.channel; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +public class RpcMessage { + private final TRequest request; + private final boolean isNotification; + private final TResponse response; + private final Throwable error; + private final Map requestContext = new HashMap<>(); + + public RpcMessage(TRequest request, boolean isNotification) { + this(request, isNotification, null, null); + } + + public RpcMessage(TRequest request, TResponse response) { + this(request, false, response, null); + } + + public RpcMessage(TRequest request, Throwable error) { + this(request, false, null, error); + } + + private RpcMessage(TRequest request, boolean isNotification, TResponse response, Throwable error) { + this.request = request; + this.isNotification = isNotification; + this.response = response; + this.error = error; + } + + public TRequest getRequest() { + return request; + } + + public boolean isNotification() { + return isNotification; + } + + public Optional getResponse() { + return Optional.ofNullable(response); + } + + public Optional getError() { + return Optional.ofNullable(error); + } + + public boolean isRequest() { + return !(getResponse().isPresent() || getError().isPresent()); + } + + public boolean isResponse() { + return !isRequest(); + } + + public RpcMessage copyWithRequest(TNewRequest newRequest) { + if (!isRequest()) { + throw new IllegalStateException(""); + } + RpcMessage ret = new RpcMessage<>(newRequest, + isNotification, null, null); + ret.requestContext.putAll(requestContext); + return ret; + } + + public RpcMessage copyWithResponse(TNewResponse newResponse) { + if (getError().isPresent()) { + throw new IllegalStateException(""); + } + RpcMessage ret = new RpcMessage<>(null, + false, newResponse, null); + ret.requestContext.putAll(requestContext); + return ret; + } + + public RpcMessage copyWithResponseError(Throwable error) { + if (getError().isPresent()) { + throw new IllegalStateException(""); + } + RpcMessage ret = new RpcMessage<>(null, + false, null, error); + ret.requestContext.putAll(requestContext); + return ret; + } + + public void setRequestContext(Object key, Object value) { + if (!isRequest()) { + throw new IllegalStateException("Context can be added to request only"); + } + requestContext.put(key, value); + } + + public Object getRequestContext(Object key) { + if (!isResponse()) { + throw new IllegalStateException("Context can be pushed from response only"); + } + return requestContext.get(key); + } + + Map getRequestContext() { + return requestContext; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/BeaconPayloadCodec.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/BeaconPayloadCodec.java new file mode 100644 index 000000000..2f792254e --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/BeaconPayloadCodec.java @@ -0,0 +1,79 @@ +package org.ethereum.beacon.wire.channel.beacon; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.ssz.SSZSerializer; +import org.ethereum.beacon.wire.channel.Channel; +import org.ethereum.beacon.wire.channel.ChannelCodec; +import org.ethereum.beacon.wire.channel.RpcMessage; +import org.ethereum.beacon.wire.exceptions.WireRemoteRpcError; +import org.ethereum.beacon.wire.message.RequestMessage; +import org.ethereum.beacon.wire.message.RequestMessagePayload; +import org.ethereum.beacon.wire.message.ResponseMessage; +import org.ethereum.beacon.wire.message.ResponseMessagePayload; +import org.ethereum.beacon.wire.message.payload.MessageType; +import tech.pegasys.artemis.util.bytes.BytesValue; + +class BeaconPayloadCodec extends ChannelCodec< + RpcMessage, + RpcMessage> { + + private static final Logger logger = LogManager.getLogger(BeaconPayloadCodec.class); + + private static final Object CONTEXT_REQUEST_MESSAGE_ID = new Object(); + + public BeaconPayloadCodec( + Channel> inChannel, + SSZSerializer sszSerializer) { + + super(inChannel, msg -> decode(sszSerializer, msg), msg -> encode(sszSerializer, msg)); + } + + static RpcMessage decode( + SSZSerializer sszSerializer, RpcMessage msg) { + + if (msg.isRequest()) { + MessageType messageType = MessageType.getById(msg.getRequest().getMethodId()); + RequestMessagePayload messagePayload = sszSerializer + .decode(msg.getRequest().getBody(), messageType.getRequestClass()); + return msg.copyWithRequest(messagePayload); + } else { + int methodId = (int) msg.getRequestContext(CONTEXT_REQUEST_MESSAGE_ID); + if (msg.getResponse().get().getResponseCode() == 0) { + ResponseMessagePayload messagePayload = + sszSerializer.decode( + msg.getResponse().get().getResult(), + MessageType.getById(methodId).getResponseClass()); + return msg.copyWithResponse(messagePayload); + } else { + return msg.copyWithResponseError(deserializeError(sszSerializer, + msg.getResponse().get().getResponseCode(), msg.getResponse().get().getResult())); + } + } + } + + static RpcMessage encode( + SSZSerializer sszSerializer, RpcMessage msg) { + if (msg.isRequest()) { + int methodId = msg.getRequest().getMethodId(); + msg.setRequestContext(CONTEXT_REQUEST_MESSAGE_ID, methodId); + BytesValue payloadBytes = sszSerializer.encode2(msg.getRequest()); + return msg.copyWithRequest(new RequestMessage(methodId, payloadBytes)); + } else { + if (msg.getError().isPresent()) { + return msg.copyWithResponse(serializeError(sszSerializer, msg.getError().get())); + } else { + BytesValue payloadBytes = sszSerializer.encode2(msg.getResponse().get()); + return msg.copyWithResponse(new ResponseMessage(0, payloadBytes)); + } + } + } + + protected static ResponseMessage serializeError(SSZSerializer sszSerializer, Throwable t) { + return new ResponseMessage(0xFF, BytesValue.EMPTY); + } + + protected static Throwable deserializeError(SSZSerializer sszSerializer, int respCode, BytesValue data) { + return new WireRemoteRpcError("Remote peer call error: code = " + respCode + ", payload: " + data); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/BeaconPipeline.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/BeaconPipeline.java new file mode 100644 index 000000000..2d4c3c1e4 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/BeaconPipeline.java @@ -0,0 +1,246 @@ +package org.ethereum.beacon.wire.channel.beacon; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.ssz.SSZSerializer; +import org.ethereum.beacon.wire.Feedback; +import org.ethereum.beacon.wire.MessageSerializer; +import org.ethereum.beacon.wire.WireApiPeer; +import org.ethereum.beacon.wire.WireApiSync; +import org.ethereum.beacon.wire.channel.Channel; +import org.ethereum.beacon.wire.channel.ChannelCodec; +import org.ethereum.beacon.wire.channel.ChannelHub; +import org.ethereum.beacon.wire.channel.IdentityChannel; +import org.ethereum.beacon.wire.channel.RpcChannel; +import org.ethereum.beacon.wire.channel.RpcChannelAdapter; +import org.ethereum.beacon.wire.channel.RpcChannelClassFilter; +import org.ethereum.beacon.wire.channel.RpcChannelMapper; +import org.ethereum.beacon.wire.channel.RpcMessage; +import org.ethereum.beacon.wire.message.Message; +import org.ethereum.beacon.wire.message.RequestMessage; +import org.ethereum.beacon.wire.message.RequestMessagePayload; +import org.ethereum.beacon.wire.message.ResponseMessage; +import org.ethereum.beacon.wire.message.ResponseMessagePayload; +import org.ethereum.beacon.wire.message.payload.BlockBodiesRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockBodiesResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockHeadersRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockHeadersResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsResponseMessage; +import org.ethereum.beacon.wire.message.payload.GoodbyeMessage; +import org.ethereum.beacon.wire.message.payload.HelloMessage; +import org.ethereum.beacon.wire.message.payload.NotifyNewAttestationMessage; +import org.ethereum.beacon.wire.message.payload.NotifyNewBlockMessage; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public class BeaconPipeline { + private static final Logger logger = LogManager.getLogger(BeaconPipeline.class); + + private final SSZSerializer sszSerializer; + private final WireApiPeer peerServer; + private final WireApiSubRpc subServer; + private final WireApiSync syncServer; + private final Schedulers schedulers; + private WireApiPeer peerClient; + private WireApiSubRpc subClient; + private WireApiSync syncClient; + private Duration rpsTimeout = RpcChannelAdapter.DEFAULT_RPC_TIMEOUT; + + private RpcChannel rpcHub; + + public BeaconPipeline(SSZSerializer sszSerializer, WireApiPeer peerServer, + WireApiSubRpc subServer, WireApiSync syncServer, + Schedulers schedulers) { + this.sszSerializer = sszSerializer; + this.peerServer = peerServer; + this.subServer = subServer; + this.syncServer = syncServer; + this.schedulers = schedulers; + } + + public BeaconPipeline setRpsTimeout(Duration rpsTimeout) { + this.rpsTimeout = rpsTimeout; + return this; + } + + public void initFromBytesChannel(Channel rawChannel, MessageSerializer messageSerializer) { + + Channel messageChannel = new ChannelCodec<>(rawChannel, + messageSerializer::deserialize, messageSerializer::serialize); + + initFromMessageChannel(messageChannel); + } + + public void initFromMessageChannel(Channel messageChannel) { + RpcChannelMapper rpcMessageChannel = + new BeaconRpcMapper(messageChannel); + + messageChannel.getCloseFuture().thenAccept(v -> System.out.println("### Closed")); + + ChannelCodec< + RpcMessage, + RpcMessage> + payloadCodec = new BeaconPayloadCodec(rpcMessageChannel, sszSerializer); + + IdentityChannel> loggerChannel = + new IdentityChannel>( + payloadCodec) { + @Override + protected void onInbound(RpcMessage msg) + throws RuntimeException { + logger.debug(" ==> " + (msg.isRequest() ? msg.getRequest() : msg.getResponse().get())); + } + + @Override + protected void onOutbound(RpcMessage msg) + throws RuntimeException { + logger.debug(" <== " + (msg.isRequest() ? msg.getRequest() : msg.getResponse().get())); + } + }; + + RpcChannel inboundResponsePayloadValidator = RpcChannel + .from(new IdentityChannel>( + loggerChannel) { + @Override + protected void onOutbound(RpcMessage msg) + throws RuntimeException { + if (msg.isRequest()) { + msg.setRequestContext("validatorRequestMessgae", msg.getRequest()); + } + } + + @Override + protected void onInbound(RpcMessage msg) + throws RuntimeException { + if (msg.isResponse()) { + RequestMessagePayload request = (RequestMessagePayload) msg + .getRequestContext("validatorRequestMessgae"); + ResponseMessagePayload response = msg.getResponse().get(); + + // validate response against request + } + } + }); + + ChannelHub> channelHub = new ChannelHub<>( + inboundResponsePayloadValidator, false); + rpcHub = RpcChannel.from(channelHub); + syncClient = createWireApiSync(syncServer); + subClient = createWireApiSub(subServer); + peerClient = createWireApiPeer(peerServer); + + channelHub.connect(); + } + + public WireApiPeer getPeerClient() { + return peerClient; + } + + public WireApiSubRpc getSubClient() { + return subClient; + } + + public WireApiSync getSyncClient() { + return syncClient; + } + + private WireApiSync createWireApiSync(WireApiSync syncServer) { + RpcChannelAdapter blockRootsAsync = + new RpcChannelAdapter<>(new RpcChannelClassFilter<>(rpcHub, BlockRootsRequestMessage.class), + syncServer != null ? syncServer::requestBlockRoots : null, schedulers.events()) + .withRpcCallTimeout(rpsTimeout); + RpcChannelAdapter blockHeadersAsync = + new RpcChannelAdapter<>(new RpcChannelClassFilter<>(rpcHub, BlockHeadersRequestMessage.class), + syncServer != null ? syncServer::requestBlockHeaders : null, schedulers.events()) + .withRpcCallTimeout(rpsTimeout); + RpcChannelAdapter blockBodiesAsync = + new RpcChannelAdapter<>(new RpcChannelClassFilter<>(rpcHub, BlockBodiesRequestMessage.class), + syncServer != null ? req -> syncServer.requestBlockBodies(req).thenApply(Feedback::get) : null, + schedulers.events()); + blockBodiesAsync.withRpcCallTimeout(rpsTimeout); + + + return new WireApiSync() { + @Override + public CompletableFuture requestBlockRoots( + BlockRootsRequestMessage requestMessage) { + return blockRootsAsync.invokeRemote(requestMessage); + } + + @Override + public CompletableFuture requestBlockHeaders( + BlockHeadersRequestMessage requestMessage) { + return blockHeadersAsync.invokeRemote(requestMessage); + } + + @Override + public CompletableFuture> requestBlockBodies( + BlockBodiesRequestMessage requestMessage) { + return blockBodiesAsync.invokeRemote(requestMessage).thenApply(Feedback::of); + } + }; + } + + private WireApiSubRpc createWireApiSub(WireApiSubRpc subServer) { + RpcChannelAdapter blocks = + new RpcChannelAdapter<>(new RpcChannelClassFilter<>(rpcHub, NotifyNewBlockMessage.class), + subServer == null ? null : newBlock -> { + subServer.newBlock(newBlock.getBlock()); + return null; + }, schedulers.events()) + .withRpcCallTimeout(rpsTimeout); + + RpcChannelAdapter attestations = + new RpcChannelAdapter<>(new RpcChannelClassFilter<>(rpcHub, NotifyNewAttestationMessage.class), + subServer == null ? null : newAttest -> { + subServer.newAttestation(newAttest.getAttestation()); + return null; + }, schedulers.events()) + .withRpcCallTimeout(rpsTimeout); + + return new WireApiSubRpc() { + @Override + public void newBlock(BeaconBlock block) { + blocks.notifyRemote(new NotifyNewBlockMessage(block)); + } + + @Override + public void newAttestation(Attestation attestation) { + attestations.notifyRemote(new NotifyNewAttestationMessage(attestation)); + } + }; + } + + private WireApiPeer createWireApiPeer(WireApiPeer peerServer) { + RpcChannelAdapter helloRpc = + new RpcChannelAdapter<>(new RpcChannelClassFilter<>(rpcHub, HelloMessage.class), + peerServer == null ? null : msg -> { + peerServer.hello(msg); + return null; + }, schedulers.events()); + + RpcChannelAdapter goodbyeRpc = + new RpcChannelAdapter<>(new RpcChannelClassFilter<>(rpcHub, GoodbyeMessage.class), + peerServer == null ? null : msg -> { + peerServer.goodbye(msg); + return null; + }, schedulers.events()); + + return new WireApiPeer() { + @Override + public void hello(HelloMessage message) { + helloRpc.notifyRemote(message); + } + + @Override + public void goodbye(GoodbyeMessage message) { + goodbyeRpc.notifyRemote(message); + } + }; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/BeaconRpcMapper.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/BeaconRpcMapper.java new file mode 100644 index 000000000..20c7be6ec --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/BeaconRpcMapper.java @@ -0,0 +1,43 @@ +package org.ethereum.beacon.wire.channel.beacon; + +import java.util.concurrent.atomic.AtomicLong; +import org.ethereum.beacon.wire.channel.Channel; +import org.ethereum.beacon.wire.channel.RpcChannelMapper; +import org.ethereum.beacon.wire.message.Message; +import org.ethereum.beacon.wire.message.RequestMessage; +import org.ethereum.beacon.wire.message.ResponseMessage; +import org.ethereum.beacon.wire.message.payload.MessageType; +import tech.pegasys.artemis.util.uint.UInt64; + +class BeaconRpcMapper extends RpcChannelMapper { + private AtomicLong idGen = new AtomicLong(1); + + public BeaconRpcMapper(Channel inChannel) { + super(inChannel); + } + + @Override + protected boolean isRequest(Message msg) { + return msg instanceof RequestMessage; + } + + @Override + protected boolean isNotification(Message msg) { + return MessageType.getById(((RequestMessage) msg).getMethodId()).isNotification(); + } + + @Override + protected Object generateNextId() { + return UInt64.valueOf(idGen.getAndIncrement()); + } + + @Override + protected Object getId(Message msg) { + return msg.getId(); + } + + @Override + protected void setId(Message msg, Object id) { + msg.setId((UInt64) id); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/WireApiSubAdapter.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/WireApiSubAdapter.java new file mode 100644 index 000000000..5fe7c6a68 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/WireApiSubAdapter.java @@ -0,0 +1,52 @@ +package org.ethereum.beacon.wire.channel.beacon; + +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.wire.WireApiSub; +import org.ethereum.beacon.wire.channel.beacon.WireApiSubRpc; +import org.reactivestreams.Publisher; +import reactor.core.publisher.FluxSink; +import reactor.core.publisher.ReplayProcessor; + +public class WireApiSubAdapter implements WireApiSub, WireApiSubRpc { + private final ReplayProcessor blockProcessor = ReplayProcessor.cacheLast(); + private final FluxSink blockSink = blockProcessor.sink(); + private final ReplayProcessor attestProcessor = ReplayProcessor.cacheLast(); + private final FluxSink attestSink = attestProcessor.sink(); + + private WireApiSubRpc subClient; + + public void setSubClient(WireApiSubRpc subClient) { + this.subClient = subClient; + } + + @Override + public void sendProposedBlock(BeaconBlock block) { + subClient.newBlock(block); + } + + @Override + public void sendAttestation(Attestation attestation) { + subClient.newAttestation(attestation); + } + + @Override + public Publisher inboundBlocksStream() { + return blockProcessor; + } + + @Override + public Publisher inboundAttestationsStream() { + return attestProcessor; + } + + @Override + public void newBlock(BeaconBlock block) { + blockSink.next(block); + } + + @Override + public void newAttestation(Attestation attestation) { + attestSink.next(attestation); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/WireApiSubRpc.java b/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/WireApiSubRpc.java new file mode 100644 index 000000000..9467dc40e --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/channel/beacon/WireApiSubRpc.java @@ -0,0 +1,13 @@ +package org.ethereum.beacon.wire.channel.beacon; + +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.reactivestreams.Publisher; + +public interface WireApiSubRpc { + + void newBlock(BeaconBlock block); + + void newAttestation(Attestation attestation); + +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireException.java b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireException.java new file mode 100644 index 000000000..d5032da8a --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireException.java @@ -0,0 +1,16 @@ +package org.ethereum.beacon.wire.exceptions; + +public class WireException extends RuntimeException { + + public WireException(String message) { + super(message); + } + + public WireException(String message, Throwable cause) { + super(message, cause); + } + + public WireException(Throwable cause) { + super(cause); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireIllegalArgumentsException.java b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireIllegalArgumentsException.java new file mode 100644 index 000000000..4aae7e508 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireIllegalArgumentsException.java @@ -0,0 +1,11 @@ +package org.ethereum.beacon.wire.exceptions; + +/** + * Is thrown on malformed request from remote side + */ +public class WireIllegalArgumentsException extends WireException { + + public WireIllegalArgumentsException(String message) { + super(message); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireInvalidConsensusDataException.java b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireInvalidConsensusDataException.java new file mode 100644 index 000000000..503d0425e --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireInvalidConsensusDataException.java @@ -0,0 +1,15 @@ +package org.ethereum.beacon.wire.exceptions; + +/** + * Is thrown when the data from a remote party violates consensus rules + */ +public class WireInvalidConsensusDataException extends WireException { + + public WireInvalidConsensusDataException(String message) { + super(message); + } + + public WireInvalidConsensusDataException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireInvalidResponseException.java b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireInvalidResponseException.java new file mode 100644 index 000000000..2836efd78 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireInvalidResponseException.java @@ -0,0 +1,11 @@ +package org.ethereum.beacon.wire.exceptions; + +/** + * When a remote party replied with invalid or inconsistent data to our request + */ +public class WireInvalidResponseException extends WireException { + + public WireInvalidResponseException(String message) { + super(message); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireRemoteRpcError.java b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireRemoteRpcError.java new file mode 100644 index 000000000..6188a0b1a --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireRemoteRpcError.java @@ -0,0 +1,11 @@ +package org.ethereum.beacon.wire.exceptions; + +/** + * This exception is a 'deserialized version' of error answer from remote RPC party + */ +public class WireRemoteRpcError extends WireRpcException { + + public WireRemoteRpcError(String message) { + super(message); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireRpcClosedException.java b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireRpcClosedException.java new file mode 100644 index 000000000..ba6064fc7 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireRpcClosedException.java @@ -0,0 +1,15 @@ +package org.ethereum.beacon.wire.exceptions; + +/** + * Thrown when calling RPC method on closed channel + */ +public class WireRpcClosedException extends WireRpcException { + + public WireRpcClosedException(String message) { + super(message); + } + + public WireRpcClosedException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireRpcException.java b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireRpcException.java new file mode 100644 index 000000000..31cd811d0 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireRpcException.java @@ -0,0 +1,15 @@ +package org.ethereum.beacon.wire.exceptions; + +/** + * Any RPC interaction exception + */ +public class WireRpcException extends WireException { + + public WireRpcException(String message) { + super(message); + } + + public WireRpcException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireRpcTimeoutException.java b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireRpcTimeoutException.java new file mode 100644 index 000000000..c0a6a3c60 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/exceptions/WireRpcTimeoutException.java @@ -0,0 +1,15 @@ +package org.ethereum.beacon.wire.exceptions; + +/** + * Thrown when no reply to RPC request for some time + */ +public class WireRpcTimeoutException extends WireRpcException { + + public WireRpcTimeoutException(String message) { + super(message); + } + + public WireRpcTimeoutException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/Message.java b/wire/src/main/java/org/ethereum/beacon/wire/message/Message.java new file mode 100644 index 000000000..99dd73b9a --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/Message.java @@ -0,0 +1,13 @@ +package org.ethereum.beacon.wire.message; + +import tech.pegasys.artemis.util.uint.UInt64; + +public abstract class Message { + + public abstract MessagePayload getPayload(); + + public abstract UInt64 getId(); + + public abstract void setId(UInt64 id); + +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/MessagePayload.java b/wire/src/main/java/org/ethereum/beacon/wire/message/MessagePayload.java new file mode 100644 index 000000000..140757443 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/MessagePayload.java @@ -0,0 +1,6 @@ +package org.ethereum.beacon.wire.message; + +import tech.pegasys.artemis.util.uint.UInt64; + +public abstract class MessagePayload { +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/RequestMessage.java b/wire/src/main/java/org/ethereum/beacon/wire/message/RequestMessage.java new file mode 100644 index 000000000..6bead59db --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/RequestMessage.java @@ -0,0 +1,48 @@ +package org.ethereum.beacon.wire.message; + +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +@SSZSerializable +public class RequestMessage extends Message { + @SSZ + private UInt64 id; + @SSZ(type = "uint16") + private final int methodId; + @SSZ + private final BytesValue body; + + public RequestMessage(int methodId, BytesValue body) { + this.methodId = methodId; + this.body = body; + } + + public RequestMessage(UInt64 id, int methodId, BytesValue body) { + this.id = id; + this.methodId = methodId; + this.body = body; + } + + public UInt64 getId() { + return id; + } + + public void setId(UInt64 id) { + this.id = id; + } + + public int getMethodId() { + return methodId; + } + + public BytesValue getBody() { + return body; + } + + @Override + public RequestMessagePayload getPayload() { + return null; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/RequestMessagePayload.java b/wire/src/main/java/org/ethereum/beacon/wire/message/RequestMessagePayload.java new file mode 100644 index 000000000..8a6ae50cd --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/RequestMessagePayload.java @@ -0,0 +1,6 @@ +package org.ethereum.beacon.wire.message; + +public abstract class RequestMessagePayload extends MessagePayload { + + public abstract int getMethodId(); +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/ResponseMessage.java b/wire/src/main/java/org/ethereum/beacon/wire/message/ResponseMessage.java new file mode 100644 index 000000000..a78d1bf3c --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/ResponseMessage.java @@ -0,0 +1,49 @@ +package org.ethereum.beacon.wire.message; + +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +@SSZSerializable +public class ResponseMessage extends Message { + @SSZ + private UInt64 id; + @SSZ(type = "uint16") + private final int responseCode; + @SSZ + private final BytesValue result; + + public ResponseMessage(int responseCode, BytesValue result) { + this.responseCode = responseCode; + this.result = result; + } + + public ResponseMessage(UInt64 id, int responseCode, + BytesValue result) { + this.id = id; + this.responseCode = responseCode; + this.result = result; + } + + public UInt64 getId() { + return id; + } + + public void setId(UInt64 id) { + this.id = id; + } + + public int getResponseCode() { + return responseCode; + } + + public BytesValue getResult() { + return result; + } + + @Override + public ResponseMessagePayload getPayload() { + return null; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/ResponseMessagePayload.java b/wire/src/main/java/org/ethereum/beacon/wire/message/ResponseMessagePayload.java new file mode 100644 index 000000000..ef705e615 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/ResponseMessagePayload.java @@ -0,0 +1,5 @@ +package org.ethereum.beacon.wire.message; + +public abstract class ResponseMessagePayload + extends MessagePayload { +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/SSZMessageSerializer.java b/wire/src/main/java/org/ethereum/beacon/wire/message/SSZMessageSerializer.java new file mode 100644 index 000000000..2c6952bcf --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/SSZMessageSerializer.java @@ -0,0 +1,27 @@ +package org.ethereum.beacon.wire.message; + +import org.ethereum.beacon.ssz.SSZSerializer; +import org.ethereum.beacon.wire.MessageSerializer; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public class SSZMessageSerializer implements MessageSerializer { + + private final SSZSerializer ssz; + + public SSZMessageSerializer(SSZSerializer ssz) { + this.ssz = ssz; + } + + @Override + public BytesValue serialize(Message message) { + return BytesValue.concat( + message instanceof RequestMessage ? BytesValue.of(1) : BytesValue.of(2), + ssz.encode2(message)); + } + + @Override + public Message deserialize(BytesValue messageBytes) { + BytesValue body = messageBytes.slice(1); + return ssz.decode(body, messageBytes.get(0) == 1 ? RequestMessage.class : ResponseMessage.class); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockBodiesRequestMessage.java b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockBodiesRequestMessage.java new file mode 100644 index 000000000..fe7ac5768 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockBodiesRequestMessage.java @@ -0,0 +1,36 @@ +package org.ethereum.beacon.wire.message.payload; + +import java.util.List; +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.ethereum.beacon.wire.message.RequestMessagePayload; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.uint.UInt64; + +@SSZSerializable +public class BlockBodiesRequestMessage extends RequestMessagePayload { + public static final int METHOD_ID = 0x0E; + + @SSZ private final List blockTreeRoots; + + public BlockBodiesRequestMessage( + List blockTreeRoots) { + this.blockTreeRoots = blockTreeRoots; + } + + @Override + public int getMethodId() { + return METHOD_ID; + } + + public List getBlockTreeRoots() { + return blockTreeRoots; + } + + @Override + public String toString() { + return "BlockBodiesRequestMessage{" + + "blockTreeRoots=" + blockTreeRoots + + '}'; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockBodiesResponseMessage.java b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockBodiesResponseMessage.java new file mode 100644 index 000000000..32d1a924e --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockBodiesResponseMessage.java @@ -0,0 +1,28 @@ +package org.ethereum.beacon.wire.message.payload; + +import java.util.List; +import org.ethereum.beacon.core.BeaconBlockBody; +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.ethereum.beacon.wire.message.ResponseMessagePayload; + +@SSZSerializable +public class BlockBodiesResponseMessage extends ResponseMessagePayload { + + @SSZ private final List blockBodies; + + public BlockBodiesResponseMessage(List blockBodies) { + this.blockBodies = blockBodies; + } + + public List getBlockBodies() { + return blockBodies; + } + + @Override + public String toString() { + return "BlockBodiesResponseMessage{" + + "blockBodies=" + blockBodies + + '}'; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockHeadersRequestMessage.java b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockHeadersRequestMessage.java new file mode 100644 index 000000000..549f58fe1 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockHeadersRequestMessage.java @@ -0,0 +1,59 @@ +package org.ethereum.beacon.wire.message.payload; + +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.ethereum.beacon.wire.message.RequestMessagePayload; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.uint.UInt64; + +@SSZSerializable +public class BlockHeadersRequestMessage extends RequestMessagePayload { + public static final int METHOD_ID = 0x0D; + public static final Hash32 NULL_START_ROOT = Hash32.fromHexString("11223344556677889900aabbccddeeff11223344556677889900aabbccddeeff"); + public static final SlotNumber NULL_START_SLOT = SlotNumber.castFrom(UInt64.MAX_VALUE); + + @SSZ private final Hash32 startRoot; + @SSZ private final SlotNumber startSlot; + @SSZ private final UInt64 maxHeaders; + @SSZ private final UInt64 skipSlots; + + public BlockHeadersRequestMessage(Hash32 startRoot, + SlotNumber startSlot, UInt64 maxHeaders, UInt64 skipSlots) { + this.startRoot = startRoot; + this.startSlot = startSlot; + this.maxHeaders = maxHeaders; + this.skipSlots = skipSlots; + } + + @Override + public int getMethodId() { + return METHOD_ID; + } + + public Hash32 getStartRoot() { + return startRoot; + } + + public SlotNumber getStartSlot() { + return startSlot; + } + + public UInt64 getMaxHeaders() { + return maxHeaders; + } + + public UInt64 getSkipSlots() { + return skipSlots; + } + + @Override + public String toString() { + return "BlockHeadersRequestMessage{" + + (NULL_START_ROOT.equals(startRoot) ? "" : "startRoot=" + startRoot + ", ") + + (NULL_START_SLOT.equals(startSlot) ? "" : "startSlot=" + startSlot + ", ") + + "maxHeaders=" + maxHeaders + + ", skipSlots=" + skipSlots + + '}'; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockHeadersResponseMessage.java b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockHeadersResponseMessage.java new file mode 100644 index 000000000..fa427fde3 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockHeadersResponseMessage.java @@ -0,0 +1,28 @@ +package org.ethereum.beacon.wire.message.payload; + +import java.util.List; +import org.ethereum.beacon.core.BeaconBlockHeader; +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.ethereum.beacon.wire.message.ResponseMessagePayload; + +@SSZSerializable +public class BlockHeadersResponseMessage extends ResponseMessagePayload { + + @SSZ private final List headers; + + public BlockHeadersResponseMessage(List headers) { + this.headers = headers; + } + + public List getHeaders() { + return headers; + } + + @Override + public String toString() { + return "BlockHeadersResponseMessage{" + + "headers=" + headers + + '}'; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockRootsRequestMessage.java b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockRootsRequestMessage.java new file mode 100644 index 000000000..31261b321 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockRootsRequestMessage.java @@ -0,0 +1,41 @@ +package org.ethereum.beacon.wire.message.payload; + +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.ethereum.beacon.wire.message.RequestMessagePayload; +import tech.pegasys.artemis.util.uint.UInt64; + +@SSZSerializable +public class BlockRootsRequestMessage extends RequestMessagePayload { + public static final int METHOD_ID = 0x0F; + + @SSZ private final SlotNumber startSlot; + @SSZ private final UInt64 count; + + public BlockRootsRequestMessage(SlotNumber startSlot, UInt64 count) { + this.startSlot = startSlot; + this.count = count; + } + + @Override + public int getMethodId() { + return METHOD_ID; + } + + public SlotNumber getStartSlot() { + return startSlot; + } + + public UInt64 getCount() { + return count; + } + + @Override + public String toString() { + return "BlockRootsRequestMessage{" + + "startSlot=" + startSlot + + ", count=" + count + + '}'; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockRootsResponseMessage.java b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockRootsResponseMessage.java new file mode 100644 index 000000000..fa5f40329 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/BlockRootsResponseMessage.java @@ -0,0 +1,48 @@ +package org.ethereum.beacon.wire.message.payload; + +import java.util.List; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.ethereum.beacon.wire.message.ResponseMessagePayload; +import tech.pegasys.artemis.ethereum.core.Hash32; + +@SSZSerializable +public class BlockRootsResponseMessage extends ResponseMessagePayload { + + @SSZSerializable + public static class BlockRootSlot { + @SSZ private final Hash32 blockRoot; + @SSZ private final SlotNumber slot; + + public BlockRootSlot(Hash32 blockRoot, SlotNumber slot) { + this.blockRoot = blockRoot; + this.slot = slot; + } + + public Hash32 getBlockRoot() { + return blockRoot; + } + + public SlotNumber getSlot() { + return slot; + } + } + + @SSZ private final List roots; + + public BlockRootsResponseMessage(List roots) { + this.roots = roots; + } + + public List getRoots() { + return roots; + } + + @Override + public String toString() { + return "BlockRootsResponseMessage{" + + "roots=" + roots + + '}'; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/payload/GoodbyeMessage.java b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/GoodbyeMessage.java new file mode 100644 index 000000000..cb2b15497 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/GoodbyeMessage.java @@ -0,0 +1,37 @@ +package org.ethereum.beacon.wire.message.payload; + +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.ethereum.beacon.wire.message.RequestMessagePayload; +import tech.pegasys.artemis.util.uint.UInt64; + +@SSZSerializable +public class GoodbyeMessage extends RequestMessagePayload { + public static final int METHOD_ID = 0x1; + + public static final UInt64 CLIENT_SHUTDOWN = UInt64.valueOf(1); + public static final UInt64 IRRELEVANT_NETWORK = UInt64.valueOf(2); + public static final UInt64 ERROR = UInt64.valueOf(3); + + @SSZ private final UInt64 reason; + + public GoodbyeMessage(UInt64 reason) { + this.reason = reason; + } + + @Override + public int getMethodId() { + return METHOD_ID; + } + + public UInt64 getReason() { + return reason; + } + + @Override + public String toString() { + return "GoodbyeMessage{" + + "reason=" + reason + + '}'; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/payload/HelloMessage.java b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/HelloMessage.java new file mode 100644 index 000000000..0b47eeffa --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/HelloMessage.java @@ -0,0 +1,74 @@ +package org.ethereum.beacon.wire.message.payload; + +import org.ethereum.beacon.core.types.EpochNumber; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.ethereum.beacon.wire.message.RequestMessagePayload; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.uint.UInt64; + +@SSZSerializable +public class HelloMessage extends RequestMessagePayload { + public static final int METHOD_ID = 0x0; + + @SSZ(type = "uint8") + private final int networkId; + @SSZ private final UInt64 chainId; + @SSZ private final Hash32 latestFinalizedRoot; + @SSZ private final EpochNumber latestFinalizedEpoch; + @SSZ private final Hash32 bestRoot; + @SSZ private final SlotNumber bestSlot; + + public HelloMessage(int networkId, UInt64 chainId, + Hash32 latestFinalizedRoot, EpochNumber latestFinalizedEpoch, + Hash32 bestRoot, SlotNumber bestSlot) { + this.networkId = networkId; + this.chainId = chainId; + this.latestFinalizedRoot = latestFinalizedRoot; + this.latestFinalizedEpoch = latestFinalizedEpoch; + this.bestRoot = bestRoot; + this.bestSlot = bestSlot; + } + + @Override + public int getMethodId() { + return METHOD_ID; + } + + public int getNetworkId() { + return (byte) networkId; + } + + public UInt64 getChainId() { + return chainId; + } + + public Hash32 getLatestFinalizedRoot() { + return latestFinalizedRoot; + } + + public EpochNumber getLatestFinalizedEpoch() { + return latestFinalizedEpoch; + } + + public Hash32 getBestRoot() { + return bestRoot; + } + + public SlotNumber getBestSlot() { + return bestSlot; + } + + @Override + public String toString() { + return "HelloMessage{" + + "networkId=" + networkId + + ", chainId=" + chainId + + ", latestFinalizedRoot=" + latestFinalizedRoot + + ", latestFinalizedEpoch=" + latestFinalizedEpoch + + ", bestRoot=" + bestRoot + + ", bestSlot=" + bestSlot + + '}'; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/payload/MessageType.java b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/MessageType.java new file mode 100644 index 000000000..b4dea178b --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/MessageType.java @@ -0,0 +1,63 @@ +package org.ethereum.beacon.wire.message.payload; + +import org.ethereum.beacon.wire.message.RequestMessagePayload; +import org.ethereum.beacon.wire.message.ResponseMessagePayload; + +public enum MessageType { + + Hello(HelloMessage.METHOD_ID, HelloMessage.class, null), + Goodbye(GoodbyeMessage.METHOD_ID, GoodbyeMessage.class, null), + BlockRoots(BlockRootsRequestMessage.METHOD_ID, BlockRootsRequestMessage.class, BlockRootsResponseMessage.class), + BlockHeaders(BlockHeadersRequestMessage.METHOD_ID, BlockHeadersRequestMessage.class, BlockHeadersResponseMessage.class), + BlockBodies(BlockBodiesRequestMessage.METHOD_ID, BlockBodiesRequestMessage.class, BlockBodiesResponseMessage.class), + NewBlock(NotifyNewBlockMessage.METHOD_ID, NotifyNewBlockMessage.class, null), + NewAttestation(NotifyNewAttestationMessage.METHOD_ID, NotifyNewAttestationMessage.class, null); + + public static MessageType getById(int id) { + for (MessageType message : MessageType.values()) { + if (id == message.id) { + return message; + } + } + return null; + } + + public static MessageType getByClass(Class messageClass) { + for (MessageType message : MessageType.values()) { + if (message.requestClass.isAssignableFrom(messageClass) + || (message.responseClass != null + && message.responseClass.isAssignableFrom(messageClass))) { + return message; + } + } + return null; + } + + private final int id; + private final Class requestClass; + private final Class responseClass; + + MessageType(int id, + Class requestClass, + Class responseClass) { + this.id = id; + this.requestClass = requestClass; + this.responseClass = responseClass; + } + + public int getId() { + return id; + } + + public Class getRequestClass() { + return requestClass; + } + + public Class getResponseClass() { + return responseClass; + } + + public boolean isNotification() { + return getResponseClass() == null; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/payload/NotifyNewAttestationMessage.java b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/NotifyNewAttestationMessage.java new file mode 100644 index 000000000..8a5df9095 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/NotifyNewAttestationMessage.java @@ -0,0 +1,34 @@ +package org.ethereum.beacon.wire.message.payload; + +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.ethereum.beacon.wire.message.MessagePayload; +import org.ethereum.beacon.wire.message.RequestMessagePayload; + +@SSZSerializable +public class NotifyNewAttestationMessage extends RequestMessagePayload { + public static final int METHOD_ID = 0xF02; + + @SSZ private final Attestation attestation; + + public NotifyNewAttestationMessage(Attestation attestation) { + this.attestation = attestation; + } + + public Attestation getAttestation() { + return attestation; + } + + @Override + public int getMethodId() { + return METHOD_ID; + } + + @Override + public String toString() { + return "NotifyNewAttestationMessage{" + + "attestation=" + attestation + + '}'; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/message/payload/NotifyNewBlockMessage.java b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/NotifyNewBlockMessage.java new file mode 100644 index 000000000..88d40b84f --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/message/payload/NotifyNewBlockMessage.java @@ -0,0 +1,34 @@ +package org.ethereum.beacon.wire.message.payload; + +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.ssz.annotation.SSZ; +import org.ethereum.beacon.ssz.annotation.SSZSerializable; +import org.ethereum.beacon.wire.message.MessagePayload; +import org.ethereum.beacon.wire.message.RequestMessagePayload; + +@SSZSerializable +public class NotifyNewBlockMessage extends RequestMessagePayload { + public static final int METHOD_ID = 0xF01; + + @SSZ private final BeaconBlock block; + + public NotifyNewBlockMessage(BeaconBlock block) { + this.block = block; + } + + public BeaconBlock getBlock() { + return block; + } + + @Override + public int getMethodId() { + return METHOD_ID; + } + + @Override + public String toString() { + return "NotifyNewBlockMessage{" + + "block=" + block + + '}'; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/net/Client.java b/wire/src/main/java/org/ethereum/beacon/wire/net/Client.java new file mode 100644 index 000000000..db0274d49 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/net/Client.java @@ -0,0 +1,18 @@ +package org.ethereum.beacon.wire.net; + +import java.util.concurrent.CompletableFuture; +import org.ethereum.beacon.wire.channel.Channel; +import tech.pegasys.artemis.util.bytes.BytesValue; + +/** + * An abstract client which can connect to remote party by supplying its abstract TAddress + */ +public interface Client { + + /** + * Connects to remote party and returns the bytes {@link Channel} upon connection + * If connecting fails the {@link CompletableFuture} return will be completed with exception + */ + > CompletableFuture connect(TAddress address); + +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/net/ConnectionManager.java b/wire/src/main/java/org/ethereum/beacon/wire/net/ConnectionManager.java new file mode 100644 index 000000000..d7cfc2425 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/net/ConnectionManager.java @@ -0,0 +1,96 @@ +package org.ethereum.beacon.wire.net; + +import java.time.Duration; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.wire.channel.Channel; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; +import reactor.core.publisher.Mono; +import reactor.core.publisher.ReplayProcessor; +import reactor.core.scheduler.Scheduler; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public class ConnectionManager { + private static final Logger logger = LogManager.getLogger(ConnectionManager.class); + private static final int RECONNECT_TIMEOUT_SECONDS = 1; + + private final Server server; + private final Client client; + private final Scheduler rxScheduler; + + private final ReplayProcessor> clientConnections = ReplayProcessor.create(); + private final FluxSink> clientConnectionsSink = clientConnections.sink(); + private final Set activePeers = Collections.synchronizedSet(new HashSet<>()); + private final Map> activePeerConnections = new ConcurrentHashMap<>(); + + public ConnectionManager(Server server, Client client, + Scheduler rxScheduler) { + this.server = server; + this.client = client; + this.rxScheduler = rxScheduler; + } + + public CompletableFuture> connect(TAddress peerAddress) { + return client + .connect(peerAddress) + .thenApply( + a -> { + clientConnectionsSink.next(a); + return a; + }); + } + + public void addActivePeer(TAddress peerAddress) { + if (activePeers.contains(peerAddress)) { + throw new RuntimeException("Already have active peer address: " + peerAddress); + } + activePeers.add(peerAddress); + + Flux.just(peerAddress) + .doOnNext(addr -> logger.info("Connecting to active peer " + peerAddress)) + .map(client::connect) + .flatMap(Mono::fromFuture, 1, 1) + .doOnError(t-> logger.info("Couldn't connect to active peer " + peerAddress + ": " + t)) + .doOnNext(ch -> logger.info("Connected to active peer " + peerAddress)) + .doOnNext(ch -> { + activePeerConnections.put(peerAddress, ch); + clientConnectionsSink.next(ch); + }) + .map(Channel::getCloseFuture) + .onErrorResume(t -> Flux.just(CompletableFuture.completedFuture(null))) + .flatMap(f -> Mono.fromFuture(f.thenApply(v -> "")), 1, 1) + .doOnNext(ch -> { + activePeerConnections.remove(peerAddress); + logger.info("Disconnected from active peer " + peerAddress); + }) + .delayElements(Duration.ofSeconds(RECONNECT_TIMEOUT_SECONDS), rxScheduler) + .repeat(() -> activePeers.contains(peerAddress)) + .subscribe(); + } + + public void removeActivePeer(TAddress peerAddress, boolean disconnect) { + activePeers.remove(peerAddress); + if (disconnect) { + Channel channel = activePeerConnections.remove(peerAddress); + if (channel != null) { + channel.close(); + } + } + } + + public Publisher> channelsStream() { + return Flux.merge( + server == null ? Flux.empty() : server.channelsStream(), + client == null ? Flux.empty() : + clientConnections) + .doOnNext(ch -> System.out.println(ch)); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/net/Server.java b/wire/src/main/java/org/ethereum/beacon/wire/net/Server.java new file mode 100644 index 000000000..e7104c3c9 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/net/Server.java @@ -0,0 +1,36 @@ +package org.ethereum.beacon.wire.net; + +import io.netty.channel.ChannelFuture; +import org.ethereum.beacon.wire.channel.Channel; +import org.reactivestreams.Publisher; +import tech.pegasys.artemis.util.bytes.BytesValue; + +/** + * Abstract server which accepts inbound connections making bytes {@link Channel}'s from them. + */ +public interface Server extends AutoCloseable { + + /** + * Stream of connected channels. + * This publisher should queue and connections made before anyone subscribed and replay them + * to the first subscriber. The same rule applies to the Channel's inbound bytes + * (see {@link Channel#inboundMessageStream()}) + */ + Publisher> channelsStream(); + + /** + * Start listening for inbound connections + * @return Future which indicates when ready to accept connections or error + */ + ChannelFuture start(); + + /** + * Stops listening and release any system resources allocated + */ + void stop(); + + @Override + default void close() throws Exception { + stop(); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/net/netty/NettyChannel.java b/wire/src/main/java/org/ethereum/beacon/wire/net/netty/NettyChannel.java new file mode 100644 index 000000000..078be81ad --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/net/netty/NettyChannel.java @@ -0,0 +1,94 @@ +package org.ethereum.beacon.wire.net.netty; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import java.net.SocketAddress; +import java.util.function.Consumer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.wire.channel.Channel; +import org.reactivestreams.Publisher; +import reactor.core.Disposable; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; +import reactor.core.publisher.ReplayProcessor; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public class NettyChannel extends SimpleChannelInboundHandler implements Channel { + private static final Logger logger = LogManager.getLogger(NettyChannel.class); + + private final Consumer activeChannelListener; + private final ReplayProcessor inMessages = ReplayProcessor.create(); + private final FluxSink inMessagesSink = inMessages.sink(); + private ChannelHandlerContext ctx; + private Disposable outboundSubscription; + + public NettyChannel(Consumer activeChannelListener) { + this.activeChannelListener = activeChannelListener; + } + + @Override + public Publisher inboundMessageStream() { + return inMessages; + } + + @Override + public void subscribeToOutbound(Publisher outboundMessageStream) { + outboundSubscription = Flux.from(outboundMessageStream).subscribe(this::send); + } + + private void send(BytesValue bytesValue) { + ByteBuf buffer = ctx.alloc().buffer(bytesValue.size()); + bytesValue.appendTo(buffer); + ctx.writeAndFlush(buffer); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + super.channelActive(ctx); + this.ctx = ctx; + activeChannelListener.accept(this); + ctx.channel().closeFuture().addListener((ChannelFutureListener) future -> closed()); + } + + private void closed() { + inMessagesSink.complete(); + if (outboundSubscription != null) { + outboundSubscription.dispose(); + } + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + logger.warn("Exception caught", cause); +// inMessagesSink.error(cause); + } + + @Override + protected void channelRead0(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf) throws Exception { + // can't do BytesValue.wrapBuffer since no control over BytesValue instance lifecycle + byte[] copy = new byte[byteBuf.readableBytes()]; + byteBuf.readBytes(copy); + inMessagesSink.next(BytesValue.wrap(copy)); + } + + @Override + public void close() { + ctx.channel().close(); + } + + public SocketAddress getLocalAddress() { + return ctx.channel().localAddress(); + } + + public SocketAddress getRemoteAddress() { + return ctx.channel().remoteAddress(); + } + + @Override + public String toString() { + return "NettyChannel[" + ctx.channel() + "]"; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/net/netty/NettyChannelInitializer.java b/wire/src/main/java/org/ethereum/beacon/wire/net/netty/NettyChannelInitializer.java new file mode 100644 index 000000000..3e7fe67da --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/net/netty/NettyChannelInitializer.java @@ -0,0 +1,41 @@ +package org.ethereum.beacon.wire.net.netty; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.FixedRecvByteBufAllocator; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.LengthFieldBasedFrameDecoder; +import io.netty.handler.codec.LengthFieldPrepender; +import io.netty.handler.timeout.ReadTimeoutHandler; +import java.util.function.Consumer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +class NettyChannelInitializer extends ChannelInitializer { + private static final int READ_TIMEOUT_SEC = 600; + private static final Logger logger = LogManager.getLogger(NettyChannelInitializer.class); + + private final Consumer activeChannelListener; + + public NettyChannelInitializer(Consumer activeChannelListener) { + this.activeChannelListener = activeChannelListener; + } + + @Override + protected void initChannel(NioSocketChannel ch) throws Exception { + ch.config().setRecvByteBufAllocator(new FixedRecvByteBufAllocator(256 * 1024)); + ch.config().setOption(ChannelOption.SO_RCVBUF, 256 * 1024); + ch.config().setOption(ChannelOption.SO_BACKLOG, 1024); + + ch.pipeline().addFirst(new ReadTimeoutHandler(READ_TIMEOUT_SEC)); + ch.pipeline().addLast(new LengthFieldPrepender(4)); + ch.pipeline().addLast(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4)); + ch.pipeline().addLast(new NettyChannel(activeChannelListener)); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + logger.error("Unexpected error during channel initialization: " + ctx.channel(), cause); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/net/netty/NettyClient.java b/wire/src/main/java/org/ethereum/beacon/wire/net/netty/NettyClient.java new file mode 100644 index 000000000..154b5555b --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/net/netty/NettyClient.java @@ -0,0 +1,57 @@ +package org.ethereum.beacon.wire.net.netty; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelOption; +import io.netty.channel.DefaultMessageSizeEstimator; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioSocketChannel; +import java.net.SocketAddress; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import org.ethereum.beacon.wire.net.Client; + +public class NettyClient implements Client { + private final NioEventLoopGroup workerGroup; + + public NettyClient(NioEventLoopGroup workerGroup) { + this.workerGroup = workerGroup; + } + + public NettyClient(Executor executor) { + this(new NioEventLoopGroup(2, executor)); + } + + public NettyClient() { + this(new NioEventLoopGroup(2, + new ThreadFactoryBuilder().setNameFormat("netty-client-worker-%d").build())); + } + + @Override + public CompletableFuture connect(SocketAddress address) { + Bootstrap b = new Bootstrap(); + b.group(workerGroup); + b.channel(NioSocketChannel.class); + + b.option(ChannelOption.SO_KEEPALIVE, true); + b.option(ChannelOption.MESSAGE_SIZE_ESTIMATOR, DefaultMessageSizeEstimator.DEFAULT); + b.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 15 * 1000); + b.remoteAddress(address); + + CompletableFuture ret = new CompletableFuture<>(); + + b.handler(new NettyChannelInitializer(ret::complete)); + + // Start the client. + b.connect().addListener((ChannelFutureListener) future -> { + try { + future.get(); + } catch (Exception e) { + ret.completeExceptionally(e); + } + }); + + return ret; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/net/netty/NettyServer.java b/wire/src/main/java/org/ethereum/beacon/wire/net/netty/NettyServer.java new file mode 100644 index 000000000..3bdace966 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/net/netty/NettyServer.java @@ -0,0 +1,103 @@ +package org.ethereum.beacon.wire.net.netty; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelOption; +import io.netty.channel.DefaultMessageSizeEstimator; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.logging.LoggingHandler; +import java.util.concurrent.Executor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.wire.net.Server; +import org.reactivestreams.Publisher; +import reactor.core.publisher.FluxSink; +import reactor.core.publisher.UnicastProcessor; + +public class NettyServer implements Server { + private static final Logger logger = LogManager.getLogger(NettyServer.class); + + private UnicastProcessor channels = UnicastProcessor.create(); + private FluxSink channelsSink = channels.sink(); + private final int port; + private ChannelFuture channelFuture; + private final NioEventLoopGroup workerGroup; + + public NettyServer(int port, NioEventLoopGroup workerGroup) { + this.port = port; + this.workerGroup = workerGroup; + } + + public NettyServer(int port, Executor executor) { + this(port, new NioEventLoopGroup(16, executor)); + } + + public NettyServer(int port) { + this(port, new NioEventLoopGroup(16, + new ThreadFactoryBuilder().setNameFormat("netty-service-worker-%d").build())); + } + + @Override + public Publisher channelsStream() { + return channels; + } + + private void onChannelActive(NettyChannel channel) { + channelsSink.next(channel); + } + + @Override + public ChannelFuture start() { + + try { + ServerBootstrap b = new ServerBootstrap(); + + b.group(workerGroup, workerGroup); + b.channel(NioServerSocketChannel.class); + + b.option(ChannelOption.MESSAGE_SIZE_ESTIMATOR, DefaultMessageSizeEstimator.DEFAULT); + b.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 10 * 1000); + + b.handler(new LoggingHandler()); + b.childHandler(new NettyChannelInitializer(this::onChannelActive)); + + channelFuture = b.bind(port); + + channelFuture.addListener((ChannelFutureListener) + future -> { + logger.info("Listening for incoming connections, port: " + port); + try { + future.get(); + } catch (Exception e) { + channelsSink.error(e); + channelsSink.complete(); + } + }); + + channelFuture.channel().closeFuture().addListener(aa -> { + logger.debug("Incoming port is closed: " + port); + workerGroup.shutdownGracefully(); + channelsSink.complete(); + }); + + return channelFuture; + } catch (Exception e) { + logger.debug("Exception: {} ({})", e.getMessage(), e.getClass().getName()); + throw new RuntimeException("Can't bind the port", e); + } + } + + @Override + public void stop() { + if (channelFuture == null) { + throw new IllegalStateException("Not started"); + } + channelFuture.addListener((ChannelFutureListener) future -> { + logger.info("Stopping listening on port " + port + "..."); + future.channel().close(); + }); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/sync/AbstractBlockTree.java b/wire/src/main/java/org/ethereum/beacon/wire/sync/AbstractBlockTree.java new file mode 100644 index 000000000..cfcf05815 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/sync/AbstractBlockTree.java @@ -0,0 +1,107 @@ +package org.ethereum.beacon.wire.sync; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.wire.sync.AbstractBlockTree.BlockWrap; + +public abstract class AbstractBlockTree, TRawBlock> + implements BlockTree { + + interface BlockWrap extends Block { + TRawBlock get(); + } + + private static final Logger logger = LogManager.getLogger(AbstractBlockTree.class); + + private TBlock topBlock; + private final Map hashMap = new HashMap<>(); + private final Map> childrenMap = new HashMap<>(); + + protected abstract TBlock wrap(TRawBlock origBlock); + + public List addBlock(TRawBlock block) { + return addBlock(wrap(block)).stream().map(BlockWrap::get).collect(Collectors.toList()); + } + + public void setTopBlock(TRawBlock block) { + setTopBlock(wrap(block)); + } + + @Nonnull + @Override + public synchronized List addBlock(@Nonnull TBlock block) { + if (topBlock == null) { + throw new IllegalStateException("Top block should be set first"); + } + try { + if (hashMap.containsKey(block.getHash())) return Collections.emptyList(); + if (topBlock.getHeight() >= block.getHeight()) return Collections.emptyList(); + hashMap.put(block.getHash(), block); + childrenMap.computeIfAbsent(block.getParentHash(), r -> new ArrayList<>()).add(block.getHash()); + + List ret = new ArrayList<>(); + if (isRootSuccessor(block)) { + ret.add(block); + addChildrenRecursively(block.getHash(), ret); + } + logger.debug("Returning " + ret.size() + " ready blocks on added block " + block + " ~~> " + ret); + return ret; + } catch (Exception e) { + logger.error("Exception adding block: " + block, e); + throw new RuntimeException(e); + } + } + + private boolean isRootSuccessor(TBlock block) { + while (block != null) { + if (block.getParentHash().equals(topBlock.getHash())) { + return true; + } + block = hashMap.get(block.getParentHash()); + } + return false; + } + + private void addChildrenRecursively(THash blockHash, List successors) { + List blockChildren = childrenMap.getOrDefault(blockHash, Collections.emptyList()); + for (THash childHash : blockChildren) { + successors.add(hashMap.get(childHash)); + addChildrenRecursively(childHash, successors); + } + } + + @Override + public synchronized void setTopBlock(@Nonnull TBlock block) { + if (topBlock != null) { + if (!hashMap.containsKey(block.getHash())) { + throw new IllegalArgumentException( + "setTopBlock() should be called with existing block or to initialize"); + } + Iterator> iterator = hashMap.entrySet().iterator(); + while (iterator.hasNext()) { + Entry entry = iterator.next(); + if (entry.getValue().getHeight() <= block.getHeight() + && !entry.getKey().equals(block.getHash())) { + iterator.remove(); + childrenMap.remove(entry.getKey()); + } + } + } + topBlock = block; + } + + @Nonnull + @Override + public synchronized TBlock getTopBlock() { + return topBlock; + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/sync/BeaconBlockTree.java b/wire/src/main/java/org/ethereum/beacon/wire/sync/BeaconBlockTree.java new file mode 100644 index 000000000..43d1165e4 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/sync/BeaconBlockTree.java @@ -0,0 +1,49 @@ +package org.ethereum.beacon.wire.sync; + +import org.ethereum.beacon.consensus.hasher.ObjectHasher; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.wire.Feedback; +import org.ethereum.beacon.wire.sync.BeaconBlockTree.BlockWrapper; +import tech.pegasys.artemis.ethereum.core.Hash32; + +public class BeaconBlockTree extends AbstractBlockTree> { + + private final ObjectHasher hasher; + + protected class BlockWrapper implements AbstractBlockTree.BlockWrap> { + private final Feedback block; + + public BlockWrapper(Feedback block) { + this.block = block; + } + + @Override + public Hash32 getHash() { + return hasher.getHashTruncateLast(block.get()); + } + + @Override + public Hash32 getParentHash() { + return block.get().getPreviousBlockRoot(); + } + + @Override + public long getHeight() { + return block.get().getSlot().longValue(); + } + + @Override + public Feedback get() { + return block; + } + } + + public BeaconBlockTree(ObjectHasher hasher) { + this.hasher = hasher; + } + + @Override + protected BlockWrapper wrap(Feedback origBlock) { + return new BlockWrapper(origBlock); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/sync/BlockTree.java b/wire/src/main/java/org/ethereum/beacon/wire/sync/BlockTree.java new file mode 100644 index 000000000..3c3436624 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/sync/BlockTree.java @@ -0,0 +1,57 @@ +package org.ethereum.beacon.wire.sync; + +import java.util.List; +import javax.annotation.Nonnull; +import org.ethereum.beacon.wire.sync.BlockTree.Block; + +/** + * Builds a tree of added blocks returning block chains linked to the top block + */ +public interface BlockTree> { + + /** + * Abstract blockchain Block which has a Hash, a Parent and a Height + */ + interface Block { + + THash getHash(); + + THash getParentHash(); + + long getHeight(); + } + + /** + * Adds a new block to the block tree and returns a list of blocks + * (in order of their inheritance - first parents then children) + * which became linked to the {@link #getTopBlock()} due to adding this + * block (including the block itself). + * All blocks returned across all calls to this method ar unique, i.e. no + * block returned twice. + * Any block returned from this method is connected to initial TopBlock + * with blocks already returned from this method before. + * E.g. + * - if the supplied block has no parents in the current tree, then block is stored + * but empty list is returned + * - if the supplied block has a parent but no existing children then only this block is + * returned + * - if the supplied block has a parent and a number of descendants then + * this block + all its descendants returned + * + * Blocks with height less than {@link #getTopBlock()} are dropped + * Blocks with height bigger than {@link #getTopBlock()} + threshold are dropped + * Duplicate blocks are ignored + */ + @Nonnull List addBlock(@Nonnull TBlock block); + + /** + * Sets the top final root block + * All blocks with height less than top block height are removed from the tree + */ + void setTopBlock(@Nonnull TBlock block); + + /** + * Returns current Top Block + */ + @Nonnull TBlock getTopBlock(); +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/sync/SyncManager.java b/wire/src/main/java/org/ethereum/beacon/wire/sync/SyncManager.java new file mode 100644 index 000000000..19534e72d --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/sync/SyncManager.java @@ -0,0 +1,19 @@ +package org.ethereum.beacon.wire.sync; + +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.wire.Feedback; +import org.ethereum.beacon.wire.WireApiSync; +import org.reactivestreams.Publisher; +import reactor.core.Disposable; + +// TODO: revisit and complete this interface +public interface SyncManager { + + Disposable subscribeToOnlineBlocks(Publisher> onlineBlocks); + + Disposable subscribeToFinalizedBlocks(Publisher finalBlocks); + + void setSyncApi(WireApiSync syncApi); + + Publisher> getBlocksReadyToImport(); +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/sync/SyncManagerImpl.java b/wire/src/main/java/org/ethereum/beacon/wire/sync/SyncManagerImpl.java new file mode 100644 index 000000000..68523ee77 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/sync/SyncManagerImpl.java @@ -0,0 +1,220 @@ +package org.ethereum.beacon.wire.sync; + +import static java.lang.Math.max; +import static org.ethereum.beacon.chain.MutableBeaconChain.ImportResult.ExistingBlock; +import static org.ethereum.beacon.chain.MutableBeaconChain.ImportResult.ExpiredBlock; +import static org.ethereum.beacon.chain.MutableBeaconChain.ImportResult.InvalidBlock; +import static org.ethereum.beacon.chain.MutableBeaconChain.ImportResult.NoParent; +import static org.ethereum.beacon.chain.MutableBeaconChain.ImportResult.OK; +import static org.ethereum.beacon.chain.MutableBeaconChain.ImportResult.StateMismatch; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.chain.BeaconTuple; +import org.ethereum.beacon.chain.BeaconTupleDetails; +import org.ethereum.beacon.chain.MutableBeaconChain; +import org.ethereum.beacon.chain.MutableBeaconChain.ImportResult; +import org.ethereum.beacon.chain.storage.BeaconChainStorage; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.wire.Feedback; +import org.ethereum.beacon.wire.WireApiSync; +import org.ethereum.beacon.wire.exceptions.WireInvalidConsensusDataException; +import org.ethereum.beacon.wire.message.payload.BlockHeadersRequestMessage; +import org.ethereum.beacon.wire.sync.SyncQueue.BlockRequest; +import org.reactivestreams.Publisher; +import reactor.core.Disposable; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Scheduler; +import tech.pegasys.artemis.ethereum.core.Hash32; + +public class SyncManagerImpl { + + public enum SyncMode { + Long, + Short + } + + private static final Logger logger = LogManager.getLogger(SyncManagerImpl.class); + + private final Publisher blockStatesStream; + private final BeaconChainSpec spec; + + private final WireApiSync syncApi; + private Publisher> newBlocks; + private final SyncQueue syncQueue; + private final ModeDetector modeDetector; + private final Flux syncModeFlux; + + FluxSink> requestsStreams; + Flux blockRequestFlux; + Scheduler delayScheduler; + Flux finalizedBlockStream; + + private Disposable wireBlocksStreamSub; + private Disposable finalizedBlockStreamSub; + private Disposable readyBlocksStreamSub; + + private Duration requestsDelayLongMode = Duration.ZERO; + private Duration requestsDelayShortMode = Duration.ofSeconds(1); + + // TODO: make this parameter dynamic depending on active peers number + int maxConcurrentBlockRequests = 2; + + public SyncManagerImpl( + MutableBeaconChain chain, + Publisher> newBlocks, + BeaconChainStorage storage, + BeaconChainSpec spec, + WireApiSync syncApi, + SyncQueue syncQueue, + int maxConcurrentBlockRequests, + Scheduler delayScheduler) { + + this.blockStatesStream = chain.getBlockStatesStream(); + this.newBlocks = newBlocks; + this.spec = spec; + this.syncApi = syncApi; + this.syncQueue = syncQueue; + this.maxConcurrentBlockRequests = maxConcurrentBlockRequests; + this.delayScheduler = delayScheduler; + + modeDetector = new ModeDetector( + Flux.from(chain.getBlockStatesStream()).map(BeaconTuple::getBlock), + Flux.from(newBlocks).map(Feedback::get)); + syncModeFlux = Flux.from(modeDetector.getSyncModeStream()).replay(1).autoConnect(); + blockRequestFlux = syncModeFlux + .doOnNext(mode -> logger.info("Switch sync to mode " + mode)) + .switchMap( + mode -> { + switch (mode) { + case Long: + Flux blockRequestFlux = Flux.from(syncQueue.getBlockRequestsStream()); + return requestsDelayLongMode.toMillis() == 0 ? blockRequestFlux + : blockRequestFlux.delayElements(requestsDelayLongMode, delayScheduler); + case Short: + return Flux.from(syncQueue.getBlockRequestsStream()) + .delayElements(requestsDelayShortMode, delayScheduler); + default: + throw new IllegalStateException(); + } + }, 1); + + Hash32 genesisBlockRoot = + storage.getBlockStorage().getSlotBlocks(spec.getConstants().getGenesisSlot()).get(0); + + Flux finalizedBlockRootStream = + Flux.from(blockStatesStream) + .map(bs -> bs.getFinalState().getFinalizedRoot()) + .distinct() + .map(br -> Hash32.ZERO.equals(br) ? genesisBlockRoot : br); + + finalizedBlockStream = + finalizedBlockRootStream.map( + root -> + storage.getBlockStorage().get(root).orElseThrow(() -> new IllegalStateException())); + + readyBlocksStreamSub = + Flux.from(syncQueue.getBlocksStream()) + .subscribe( + block -> { + ImportResult result = chain.insert(block.get()); + if (result == InvalidBlock || result == StateMismatch || result == ExpiredBlock) { + block.feedbackError( + new WireInvalidConsensusDataException( + "Couldn't insert block: " + block.get())); + } else { + block.feedbackSuccess(); + if (result == NoParent) { + logger.warn("No parent for block: " + block.get()); + } else if (result == ExistingBlock) { + logger.info("Trying to import existing block: " + block.get()); + } else if (result != OK) { + logger.info("Other error importing block: " + block.get()); + } + } + }); + } + + public Publisher> getBlocksReadyToImport() { + return syncQueue.getBlocksStream(); + } + + public void setRequestsDelay(Duration longMode, Duration shortMode) { + this.requestsDelayLongMode = longMode; + this.requestsDelayShortMode = shortMode; + } + + public void start() { + + finalizedBlockStreamSub = syncQueue.subscribeToFinalBlocks(finalizedBlockStream); + + Flux>> wireBlocksStream = blockRequestFlux + .map(req -> new BlockHeadersRequestMessage( + req.getStartRoot().orElse(BlockHeadersRequestMessage.NULL_START_ROOT), + req.getStartSlot().orElse(BlockHeadersRequestMessage.NULL_START_SLOT), + req.getMaxCount(), + req.getStep())) + .flatMap(req -> Mono.fromFuture(syncApi.requestBlocks(req, spec.getObjectHasher())), + maxConcurrentBlockRequests) + .onErrorContinue((t, o) -> logger.warn("SyncApi exception: " + t + ", " + o)); + + if (newBlocks != null) { + wireBlocksStream = wireBlocksStream.mergeWith( + Flux.from(newBlocks).map(blockF -> blockF.map(Collections::singletonList))); + } + + wireBlocksStreamSub = syncQueue.subscribeToNewBlocks(wireBlocksStream); + } + + public void stop() { + wireBlocksStreamSub.dispose(); + finalizedBlockStreamSub.dispose(); + readyBlocksStreamSub.dispose(); + } + + public Publisher getSyncModeStream() { + return syncModeFlux; + } + + class ModeDetector { + Publisher syncModeStream; + + public ModeDetector( + Publisher importedBlocks, + Publisher onlineBlocks) { + + syncModeStream = + Flux.combineLatest( + Flux.from(importedBlocks) + .scan(new ArrayList<>(), (arr, block) -> listAddLimited(arr, block, 8)), + Flux.from(onlineBlocks) + .scan(new ArrayList<>(), (arr, block) -> listAddLimited(arr, block, 8)), + (latestImported, latestOnline) -> { + HashSet s1 = new HashSet<>(latestImported); + HashSet s2 = new HashSet<>(latestOnline); + s1.retainAll(s2); + return s1.isEmpty() ? SyncMode.Long : SyncMode.Short; + }) + .distinctUntilChanged() + .onErrorContinue((t, o) -> logger.error("Unexpected error: ", t)); + } + + private ArrayList listAddLimited(ArrayList list, A elem, int maxSize) { + ArrayList ret = new ArrayList<>(list.subList(max(0, list.size() + 1 - maxSize), list.size())); + ret.add(elem); + return ret; + } + + public Publisher getSyncModeStream() { + return syncModeStream; + } + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/sync/SyncQueue.java b/wire/src/main/java/org/ethereum/beacon/wire/sync/SyncQueue.java new file mode 100644 index 000000000..7e84445b0 --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/sync/SyncQueue.java @@ -0,0 +1,90 @@ +package org.ethereum.beacon.wire.sync; + +import java.util.List; +import java.util.Optional; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.wire.Feedback; +import org.reactivestreams.Publisher; +import reactor.core.Disposable; +import reactor.core.publisher.Flux; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.uint.UInt64; + +/** + * The class which declares what blocks are wanted to be downloaded, consumes + * downloaded blocks, builds chains of blocks linked to the finalized block and + * streams them for importing. + */ +public interface SyncQueue { + + /** + * Potentially unbounded stream of blocks wanted to be downloaded. + * The stream may be unbounded because the queue tries to retrieve any + * new blocks above the final block again and again with the hope that + * something new can be discovered. Thus the consumer should have a mechanism + * of limiting these requests to prevent traffic overhead. + */ + Publisher getBlockRequestsStream(); + + /** + * The stream of blocks ready to be imported to the blockchain. + * Any issued block must be a child of some block issued before. + * Blocks are wrapped to a {@link Feedback} instance so + * block verification and importing result should be reported via this {@link Feedback} + */ + Publisher> getBlocksStream(); + + /** + * finalBlockRootStream notifies the {@link SyncQueue} on finalized blocks + * so the queue may stick to those blocks and perform necessary cleanup + * of outdated blocks + */ + Disposable subscribeToFinalBlocks(Flux finalBlockRootStream); + + /** + * All new blocks are streamed via blocksStream. + * Those blocks may include: + * - downloaded per {@link SyncQueue} requests blocks + * - new fresh blocks broadcasted from remote parties + * - new blocks proposed by local validators + */ + Disposable subscribeToNewBlocks(Publisher>> blocksStream); + + class BlockRequest { + private final SlotNumber startSlot; + private final Hash32 startRoot; + private final UInt64 maxCount; + private final boolean reverse; + private final UInt64 step; + + public BlockRequest(SlotNumber startSlot, Hash32 startRoot, + int maxCount, boolean reverse, int step) { + this.startSlot = startSlot; + this.startRoot = startRoot; + this.maxCount = UInt64.valueOf(maxCount); + this.reverse = reverse; + this.step = UInt64.valueOf(step); + } + + public Optional getStartSlot() { + return Optional.ofNullable(startSlot); + } + + public Optional getStartRoot() { + return Optional.ofNullable(startRoot); + } + + public UInt64 getMaxCount() { + return maxCount; + } + + public boolean isReverse() { + return reverse; + } + + public UInt64 getStep() { + return step; + } + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/sync/SyncQueueImpl.java b/wire/src/main/java/org/ethereum/beacon/wire/sync/SyncQueueImpl.java new file mode 100644 index 000000000..f07fef6dc --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/sync/SyncQueueImpl.java @@ -0,0 +1,88 @@ +package org.ethereum.beacon.wire.sync; + +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.wire.Feedback; +import org.reactivestreams.Publisher; +import reactor.core.Disposable; +import reactor.core.publisher.Flux; +import reactor.core.publisher.ReplayProcessor; + +public class SyncQueueImpl implements SyncQueue { + private static final Logger logger = LogManager.getLogger(SyncQueueImpl.class); + + private final BeaconBlockTree blockTree; + private final int maxBlocksRequest; + private final int maxHeightFromFinal; + + private final ReplayProcessor> readyBlocks = ReplayProcessor.cacheLast(); + private final ReplayProcessor> blockRequests = ReplayProcessor.cacheLast(); + private BeaconBlock finalBlock; + + public SyncQueueImpl(BeaconBlockTree blockTree, int maxBlocksRequest, int maxHeightFromFinal) { + this.blockTree = blockTree; + this.maxBlocksRequest = maxBlocksRequest; + this.maxHeightFromFinal = maxHeightFromFinal; + } + + public SyncQueueImpl(BeaconBlockTree blockTree) { + this(blockTree, 128, 4096); + } + + @Override + public Publisher getBlockRequestsStream() { + return Flux.switchOnNext(blockRequests, 1); + } + + @Override + public Publisher> getBlocksStream() { + return readyBlocks; + } + + protected Flux createBlockRequests() { + return Flux.generate( + () -> finalBlock.getSlot(), + (slot, sink) -> { + if (slot.greater(finalBlock.getSlot().plus(maxHeightFromFinal))) { + slot = finalBlock.getSlot(); + } + sink.next(new BlockRequest(slot, null, maxBlocksRequest, false, 0)); + return slot.plus(maxBlocksRequest); + }); + } + + protected void onNewFinalBlock(BeaconBlock finalBlock) { + logger.debug(() -> "New final bock: " + finalBlock); + blockTree.setTopBlock(Feedback.of(finalBlock)); + this.finalBlock = finalBlock; + blockRequests.onNext(createBlockRequests()); + } + + protected void onInvalidBlock(BeaconBlock block) { + logger.warn("Invalid block received: " + block); + } + + protected void onNewBlock(Feedback block) { + block.getFeedback().whenComplete((v,t) -> { + if (t != null) { + onInvalidBlock(block.get()); + } + }); + + blockTree.addBlock(block).forEach(readyBlocks::onNext); + } + + @Override + public Disposable subscribeToFinalBlocks(Flux finalBlockRootStream) { + return Flux.from(finalBlockRootStream).subscribe(this::onNewFinalBlock); + } + + @Override + public Disposable subscribeToNewBlocks(Publisher>> blocksStream) { + return Flux.from(blocksStream) + .flatMap(resp -> Flux.fromStream(resp.get().stream().map(resp::delegate))) + .subscribe(this::onNewBlock); + } +} diff --git a/wire/src/main/java/org/ethereum/beacon/wire/sync/WireApiSyncRouter.java b/wire/src/main/java/org/ethereum/beacon/wire/sync/WireApiSyncRouter.java new file mode 100644 index 000000000..35f595d5b --- /dev/null +++ b/wire/src/main/java/org/ethereum/beacon/wire/sync/WireApiSyncRouter.java @@ -0,0 +1,78 @@ +package org.ethereum.beacon.wire.sync; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Function; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.stream.RxUtil; +import org.ethereum.beacon.util.Utils; +import org.ethereum.beacon.wire.Feedback; +import org.ethereum.beacon.wire.WireApiSync; +import org.ethereum.beacon.wire.message.payload.BlockBodiesRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockBodiesResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockHeadersRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockHeadersResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsResponseMessage; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; +import reactor.core.publisher.ReplayProcessor; + +/** + * Tracks and aggregates {@link WireApiSync} instances from separate peers + * This is a pretty simple implementation which just delegates API calls in a round robin fashion + * When no single delegate api available all calls are queued until any api arises + */ +public class WireApiSyncRouter implements WireApiSync { + private static final Logger logger = LogManager.getLogger(WireApiSyncRouter.class); + + private final ReplayProcessor> tasks = ReplayProcessor.create(64); + private final FluxSink> tasksSink = tasks.sink(); + private final AtomicInteger pendingTasks = new AtomicInteger(); + + public WireApiSyncRouter( + Publisher addedPeersStream, + Publisher removedPeersStream) { + + Flux freePeersStream = + RxUtil.collect(addedPeersStream, removedPeersStream) + .doOnNext(activePeers -> logger.info("Active APIs count: " + activePeers.size())) + .switchMap( + activePeers -> + activePeers.isEmpty() ? Flux.never() : Flux.fromIterable(activePeers).repeat(), + 1); + + freePeersStream.zipWith(tasks, 1) + .doOnNext(p -> pendingTasks.decrementAndGet()) + .subscribe(p -> p.getT2().accept(p.getT1())); + } + + private CompletableFuture submitAsyncTask(Function> task) { + CompletableFuture ret = new CompletableFuture<>(); + tasksSink.next(api -> Utils.futureForward(task.apply(api), ret)); + int cnt = pendingTasks.incrementAndGet(); + logger.debug("New task submitted. Pending tasks: " + cnt); + return ret; + } + + @Override + public CompletableFuture requestBlockRoots( + BlockRootsRequestMessage requestMessage) { + return submitAsyncTask(api -> api.requestBlockRoots(requestMessage)); + } + + @Override + public CompletableFuture requestBlockHeaders( + BlockHeadersRequestMessage requestMessage) { + return submitAsyncTask(api -> api.requestBlockHeaders(requestMessage)); + } + + @Override + public CompletableFuture> requestBlockBodies( + BlockBodiesRequestMessage requestMessage) { + return submitAsyncTask(api -> api.requestBlockBodies(requestMessage)); + } +} diff --git a/wire/src/test/java/org/ethereum/beacon/wire/NodeTest.java b/wire/src/test/java/org/ethereum/beacon/wire/NodeTest.java new file mode 100644 index 000000000..81b9d8136 --- /dev/null +++ b/wire/src/test/java/org/ethereum/beacon/wire/NodeTest.java @@ -0,0 +1,220 @@ +package org.ethereum.beacon.wire; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.time.Duration; +import java.util.List; +import java.util.Random; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import org.ethereum.beacon.start.common.NodeLauncher; +import org.ethereum.beacon.chain.storage.impl.MemBeaconChainStorageFactory; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.core.operations.Deposit; +import org.ethereum.beacon.core.state.Eth1Data; +import org.ethereum.beacon.core.types.Time; +import org.ethereum.beacon.crypto.BLS381.KeyPair; +import org.ethereum.beacon.emulator.config.ConfigBuilder; +import org.ethereum.beacon.emulator.config.chainspec.SpecBuilder; +import org.ethereum.beacon.emulator.config.chainspec.SpecData; +import org.ethereum.beacon.pow.DepositContract; +import org.ethereum.beacon.schedulers.ControlledSchedulers; +import org.ethereum.beacon.start.common.util.MDCControlledSchedulers; +import org.ethereum.beacon.start.common.util.SimpleDepositContract; +import org.ethereum.beacon.start.common.util.SimulateUtils; +import org.ethereum.beacon.validator.crypto.BLS381Credentials; +import org.ethereum.beacon.wire.channel.Channel; +import org.ethereum.beacon.wire.net.ConnectionManager; +import org.ethereum.beacon.wire.net.netty.NettyClient; +import org.ethereum.beacon.wire.net.netty.NettyServer; +import org.ethereum.beacon.wire.sync.SyncManager; +import org.ethereum.beacon.wire.sync.SyncManagerImpl; +import org.ethereum.beacon.wire.sync.SyncManagerImpl.SyncMode; +import org.javatuples.Pair; +import org.junit.Assert; +import org.junit.Test; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +public class NodeTest { + + @Test + public void test1() throws Exception { + Random rnd = new Random(); + + ConfigBuilder specConfigBuilder = + new ConfigBuilder<>(SpecData.class) + .addYamlConfigFromResources("/config/spec-constants.yml") + .addYamlConfigFromResources("/test-spec-config.yml"); + SpecData specData = specConfigBuilder.build(); + SpecBuilder specBuilder = new SpecBuilder() + .withSpec(specData); + BeaconChainSpec spec = specBuilder.buildSpec(); + + int depositCount = 16; + Pair, List> depositPairs = + SimulateUtils.getAnyDeposits(rnd, spec, depositCount, false); + + Time genesisTime = Time.of(60000); + + MDCControlledSchedulers controlledSchedulers = new MDCControlledSchedulers(); + controlledSchedulers.setCurrentTime(genesisTime.getMillis().getValue() + 1000); + + Eth1Data eth1Data = new Eth1Data(Hash32.random(rnd), UInt64.valueOf(depositCount), Hash32.random(rnd)); + + DepositContract.ChainStart chainStart = + new DepositContract.ChainStart(genesisTime, eth1Data, depositPairs.getValue0()); + SimpleDepositContract depositContract = new SimpleDepositContract(chainStart); + + try (NettyServer nettyServer = new NettyServer(41001)) { + // master node with all validators + NodeLauncher masterNode; + { + ControlledSchedulers schedulers = controlledSchedulers.createNew("master"); + nettyServer.start(); + ConnectionManager connectionManager = new ConnectionManager<>( + nettyServer, null, schedulers.reactorEvents()); + masterNode = new NodeLauncher( + specBuilder.buildSpec(), + depositContract, + depositPairs + .getValue1() + .stream() + .map(BLS381Credentials::createWithDummySigner) + .collect(Collectors.toList()), + connectionManager, + new MemBeaconChainStorageFactory(spec.getObjectHasher()), + schedulers, + false); + } + + // generate some blocks + controlledSchedulers.addTime(Duration.ofSeconds(64 * 10)); + + // slave node + ConnectionManager slaveConnectionManager; + CompletableFuture> connectFut; + NodeLauncher slaveNode; + { + ControlledSchedulers schedulers = controlledSchedulers.createNew("slave"); + NettyClient nettyClient = new NettyClient(); + slaveConnectionManager = new ConnectionManager<>( + null, nettyClient, schedulers.reactorEvents()); + slaveNode = new NodeLauncher( + specBuilder.buildSpec(), + depositContract, + null, + slaveConnectionManager, + new MemBeaconChainStorageFactory(spec.getObjectHasher()), + schedulers, + true); + connectFut = slaveConnectionManager + .connect(InetSocketAddress.createUnresolved("localhost", 41001)); + System.out.println("Connected! " + connectFut.get()); + } + + Assert.assertEquals( + SyncMode.Long, + Mono.from(slaveNode.getSyncManager().getSyncModeStream()).block(Duration.ZERO)); + + // generate some new blocks + System.out.println("Generating online blocks"); + for (int i = 0; i < 10; i++) { + controlledSchedulers.addTime(Duration.ofSeconds(1)); + Thread.sleep(100); + } + + Flux.from(slaveNode.getSyncManager().getSyncModeStream()) + .filter(mode -> mode == SyncMode.Short) + .blockFirst(Duration.ofSeconds(30)); + + // 'realtime' mode + System.out.println("Some time in 'realtime' mode..."); + for (int i = 0; i < 50; i++) { + controlledSchedulers.addTime(Duration.ofSeconds(1)); + Thread.sleep(50); + } + + // disconnecting slave + System.out.println("Disconnecting slave"); + connectFut.get().close(); + + // generate new blocks on master + System.out.println("Generate new blocks on master"); + controlledSchedulers.addTime(Duration.ofSeconds(32 * 10)); + + // connect the slave again + System.out.println("Connect the slave again"); + CompletableFuture> connectFut1 = slaveConnectionManager + .connect(InetSocketAddress.createUnresolved("localhost", 41001)); + connectFut1.get(); + System.out.println("Slave connected"); + + System.out.println("Generating online blocks"); + controlledSchedulers.addTime(Duration.ofSeconds(10 * 10)); + + Flux.from(slaveNode.getSyncManager().getSyncModeStream()) + .filter(mode -> mode == SyncMode.Long) + .blockFirst(Duration.ofSeconds(30)); + + System.out.println("Some time in 'realtime' mode..."); + for (int i = 0; i < 50; i++) { + controlledSchedulers.addTime(Duration.ofSeconds(1)); + Thread.sleep(50); + } + + Flux.from(slaveNode.getSyncManager().getSyncModeStream()) + .filter(mode -> mode == SyncMode.Short) + .blockFirst(Duration.ofSeconds(30)); + } + } + + public static Integer getValue() { + System.out.println("I am called"); + // Simulating a long network call of 1 second in the worst case + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return 10; + } + + @Test + public void main() { + ScheduledExecutorService schedulerService = Executors + .newSingleThreadScheduledExecutor(); + ExecutorService executor = new ThreadPoolExecutor(10, 10, + 0L, TimeUnit.MILLISECONDS, + // This is an unbounded Queue. This should never be used + // in real life. That is the first step to failure. + new LinkedBlockingQueue()); + // We want to call the dummy service 10 times + CompletableFuture[] allFutures = new CompletableFuture[10]; + for (int i = 0; i < 10; ++i) { + CompletableFuture dependencyFuture = CompletableFuture + .supplyAsync(() -> getValue(), executor); + CompletableFuture futureTimeout = new CompletableFuture(); + schedulerService.schedule(() -> + futureTimeout.completeExceptionally(new TimeoutException()), 3000, TimeUnit.MILLISECONDS); + CompletableFuture result = CompletableFuture.anyOf(dependencyFuture, futureTimeout); + allFutures[i] = result; + } + // Finally wait for all futures to join + CompletableFuture.allOf(allFutures).join(); + System.out.println("All futures completed"); + System.out.println(executor.toString()); + + } +} \ No newline at end of file diff --git a/wire/src/test/java/org/ethereum/beacon/wire/PeersTest.java b/wire/src/test/java/org/ethereum/beacon/wire/PeersTest.java new file mode 100644 index 000000000..a48995437 --- /dev/null +++ b/wire/src/test/java/org/ethereum/beacon/wire/PeersTest.java @@ -0,0 +1,183 @@ +package org.ethereum.beacon.wire; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.ethereum.beacon.start.common.Launcher; +import org.ethereum.beacon.simulator.SimulatorLauncher; +import org.ethereum.beacon.simulator.SimulatorLauncher.Builder; +import org.ethereum.beacon.ssz.SSZBuilder; +import org.ethereum.beacon.ssz.SSZSerializer; +import org.ethereum.beacon.stream.SimpleProcessor; +import org.ethereum.beacon.wire.channel.Channel; +import org.ethereum.beacon.wire.message.SSZMessageSerializer; +import org.ethereum.beacon.wire.net.ConnectionManager; +import org.ethereum.beacon.wire.net.netty.NettyClient; +import org.ethereum.beacon.wire.net.netty.NettyServer; +import org.ethereum.beacon.wire.net.Server; +import org.ethereum.beacon.wire.sync.BeaconBlockTree; +import org.ethereum.beacon.wire.sync.SyncManagerImpl; +import org.ethereum.beacon.wire.sync.SyncQueue; +import org.ethereum.beacon.wire.sync.SyncQueueImpl; +import org.junit.Assert; +import org.junit.Test; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; +import reactor.core.publisher.Mono; +import reactor.core.publisher.ReplayProcessor; +import reactor.core.scheduler.Schedulers; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +public class PeersTest { + + @Test + public void test0() { + SimpleProcessor processor = new SimpleProcessor<>(Schedulers.single(), "aaa"); + processor.onNext(1); + Integer i = Mono.from(processor).block(Duration.ofMillis(300)); + System.out.println(i); + } + + @Test + public void test01() { + ReplayProcessor proc = ReplayProcessor.create(); + FluxSink sink = proc.sink(); + + sink.next(1); + sink.next(2); + + Flux.from(proc) + .onErrorContinue((t, o) -> System.out.println("Continue on error: " + t)) + .subscribe( + i -> System.out.println("Next: " + i) + ,t -> System.out.println("Err: " + t) + ,() -> System.out.println("Complete") + ); + + sink.error(new RuntimeException("Test")); + sink.next(3); + sink.complete(); + } + + @Test + public void test1() throws Exception { + int slotCount = 32; + SimulatorLauncher simulatorLauncher = new Builder() + .withConfigFromResource("/sync-simulation-config.yml") + .build(); + simulatorLauncher.run(slotCount); + Launcher peer0 = simulatorLauncher.getPeers().get(0); + System.out.println(peer0); + + Launcher peer1 = simulatorLauncher.createPeer("test"); + + try (Server server = new NettyServer(41001)) { + { + // peer 0 + server.start().await(); + System.out.println("Peer 0 listening on port 40001"); + ConnectionManager connectionManager = new ConnectionManager<>( + server, null, Schedulers.single()); + + SSZSerializer ssz = new SSZBuilder().buildSerializer(); + MessageSerializer messageSerializer = new SSZMessageSerializer(ssz); + WireApiSyncServer syncServer = new WireApiSyncServer(peer0.getBeaconChainStorage()); + SimplePeerManagerImpl peerManager = new SimplePeerManagerImpl( + (byte) 1, + UInt64.valueOf(1), + connectionManager.channelsStream(), + ssz, + peer0.getSpec(), + messageSerializer, + peer0.getSchedulers(), + syncServer, + peer0.getBeaconChain().getBlockStatesStream()); + + Flux.from(peerManager.connectedPeerStream()) + .subscribe( + peer -> { + System.out.println("Remote peer connected: " + peer); + Flux.from(peer.getRawChannel().inboundMessageStream()) + .doOnError(e -> System.out.println("#### Error: " + e)) + .doOnComplete(() -> System.out.println("#### Complete")) + .doOnNext(msg -> System.out.println("#### on message")) + .subscribe(); + }); + Flux.from(peerManager.activatedPeerStream()) + .subscribe(peer -> System.out.println("Remote peer active: " + peer)); + Flux.from(peerManager.disconnectedPeerStream()) + .subscribe(peer -> System.out.println("Remote peer disconnected: " + peer)); + System.out.println("Peer 0 is ready."); + } + + { + // peer 1 + ConnectionManager connectionManager = new ConnectionManager<>( + null, new NettyClient(), Schedulers.single()); + + SSZSerializer ssz = new SSZBuilder().buildSerializer(); + MessageSerializer messageSerializer = new SSZMessageSerializer(ssz); + SimplePeerManagerImpl peerManager = new SimplePeerManagerImpl( + (byte) 1, + UInt64.valueOf(1), + connectionManager.channelsStream(), + ssz, + peer1.getSpec(), + messageSerializer, + peer0.getSchedulers(), + null, + peer1.getBeaconChain().getBlockStatesStream()); + + Flux.from(peerManager.connectedPeerStream()) + .subscribe(peer -> System.out.println("Peer 1 connected: " + peer)); + Flux.from(peerManager.activatedPeerStream()) + .subscribe(peer -> System.out.println("Peer 1 active: " + peer)); + Flux.from(peerManager.disconnectedPeerStream()) + .subscribe(peer -> System.out.println("Peer 1 disconnected: " + peer)); + + BeaconBlockTree blockTree = new BeaconBlockTree( + simulatorLauncher.getSpec().getObjectHasher()); + SyncQueue syncQueue = new SyncQueueImpl(blockTree, 4, 20); + + SyncManagerImpl syncManager = new SyncManagerImpl( + peer1.getBeaconChain(), + Flux.from(peerManager.getWireApiSub().inboundBlocksStream()).map(Feedback::of), + peer1.getBeaconChainStorage(), + peer1.getSpec(), + peerManager.getWireApiSync(), + syncQueue, + 1, + peer1.getSchedulers().reactorEvents()); + + CountDownLatch syncLatch = new CountDownLatch(1); + Flux.from(peer1.getBeaconChain().getBlockStatesStream()) + .subscribe(s -> { + System.out.println(s); + if (s.getFinalState().getSlot().equals( + simulatorLauncher.getSpec().getConstants().getGenesisSlot().plus(slotCount))) { + syncManager.stop(); + syncLatch.countDown(); + } + }); + + System.out.println("Peer 1: starting sync manager"); + syncManager.start(); + + // simulatorLauncher.getControlledSchedulers().addTime(3000); + + System.out.println("Peer 1: connecting to peer 0 for syncing..."); + CompletableFuture> localhost = connectionManager + .connect(InetSocketAddress.createUnresolved("localhost", 41001)); + localhost.get(); + System.out.println("Peer 1: connected to peer 0"); + + Assert.assertTrue(syncLatch.await(1, TimeUnit.MINUTES)); + System.out.println("Done"); + } + } + } +} diff --git a/wire/src/test/java/org/ethereum/beacon/wire/RxTest.java b/wire/src/test/java/org/ethereum/beacon/wire/RxTest.java new file mode 100644 index 000000000..bdcdef3fe --- /dev/null +++ b/wire/src/test/java/org/ethereum/beacon/wire/RxTest.java @@ -0,0 +1,46 @@ +package org.ethereum.beacon.wire; + +import static java.util.Arrays.asList; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.ethereum.beacon.stream.RxUtil; +import org.javatuples.Pair; +import org.junit.Test; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; +import reactor.test.StepVerifier; +import reactor.test.StepVerifier.FirstStep; + +public class RxTest { + + FluxSink addSink; + FluxSink removeSink; + + @Test + public void test1() { + Publisher addedPeersStream = Flux.create(e -> addSink = e); + Publisher removedPeersStream = Flux.create(e -> removeSink = e); + + Flux> activeList = RxUtil.collect(addedPeersStream, removedPeersStream); + + activeList.subscribe(l -> System.out.println(l)); + FirstStep> test = StepVerifier.create(activeList); + + addSink.next(1); + + test.expectNext(new ArrayList<>(asList(1))); + + addSink.next(2); + addSink.next(3); + + test.expectNext(new ArrayList<>(asList(1, 2, 3))); + + removeSink.next(2); + + test.expectNext(new ArrayList<>(asList(1, 3))); + } +} diff --git a/wire/src/test/java/org/ethereum/beacon/wire/WireApiSubRouterTest.java b/wire/src/test/java/org/ethereum/beacon/wire/WireApiSubRouterTest.java new file mode 100644 index 000000000..769cc5b4c --- /dev/null +++ b/wire/src/test/java/org/ethereum/beacon/wire/WireApiSubRouterTest.java @@ -0,0 +1,133 @@ +package org.ethereum.beacon.wire; + +import static tech.pegasys.artemis.util.bytes.BytesValue.fromHexString; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import org.ethereum.beacon.core.BeaconBlock; +import org.ethereum.beacon.core.operations.Attestation; +import org.ethereum.beacon.core.util.TestDataFactory; +import org.ethereum.beacon.wire.WireApiSubRouterTest.TestRouter.Connection; +import org.ethereum.beacon.wire.channel.beacon.WireApiSubAdapter; +import org.junit.Assert; +import org.junit.Test; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; + +public class WireApiSubRouterTest { + + static class TestRouter { + static class Connection { + TestRouter router1; + TestRouter router2; + WireApiSub outerApi1; + WireApiSub outerApi2; + + public Connection(TestRouter router1, + TestRouter router2, WireApiSub outerApi1, WireApiSub outerApi2) { + this.router1 = router1; + this.router2 = router2; + this.outerApi1 = outerApi1; + this.outerApi2 = outerApi2; + } + + public void disconnect() { + router1.removeSink.next(outerApi2); + router2.removeSink.next(outerApi1); + } + } + + FluxSink addSink; + FluxSink removeSink; + WireApiSubRouter router; + + List receivedBlocks = new ArrayList<>(); + List receivedAttestations = new ArrayList<>(); + + public TestRouter() { + router = new WireApiSubRouter( + Flux.create(s -> addSink = s).publish(1).autoConnect(), + Flux.create(s -> removeSink = s).publish(1).autoConnect()); + Flux.from(router.inboundBlocksStream()).subscribe(receivedBlocks::add); + Flux.from(router.inboundAttestationsStream()).subscribe(receivedAttestations::add); + } + + public Connection connect(TestRouter other) { + WireApiSubAdapter thisApi = new WireApiSubAdapter(); + WireApiSubAdapter otherApi = new WireApiSubAdapter(); + thisApi.setSubClient(otherApi); + otherApi.setSubClient(thisApi); + this.addSink.next(otherApi); + other.addSink.next(thisApi); + + return new Connection(this, other, thisApi, otherApi); + } + + void clear() { + receivedBlocks.clear(); + receivedAttestations.clear(); + } + } + + @Test + public void test1() { + TestDataFactory dataFactory = new TestDataFactory(); + TestRouter router1 = new TestRouter(); + router1.router.sendProposedBlock(dataFactory.createBeaconBlock()); + Assert.assertTrue(router1.receivedBlocks.isEmpty()); + + TestRouter router2 = new TestRouter(); + Connection connection1 = router1.connect(router2); + + router1.router.sendProposedBlock(dataFactory.createBeaconBlock(fromHexString("01"))); + Assert.assertTrue(router1.receivedBlocks.isEmpty()); + Assert.assertEquals(1, router2.receivedBlocks.size()); + + router1.router.sendProposedBlock(dataFactory.createBeaconBlock(fromHexString("02"))); + Assert.assertTrue(router1.receivedBlocks.isEmpty()); + Assert.assertEquals(2, router2.receivedBlocks.size()); + + router1.router.sendProposedBlock(dataFactory.createBeaconBlock(fromHexString("01"))); + Assert.assertTrue(router1.receivedBlocks.isEmpty()); + Assert.assertEquals(2, router2.receivedBlocks.size()); + + connection1.outerApi2.sendProposedBlock(dataFactory.createBeaconBlock(fromHexString("01"))); + Assert.assertEquals(2, router2.receivedBlocks.size()); + + connection1.disconnect(); + router2.clear(); + + router1.router.sendProposedBlock(dataFactory.createBeaconBlock(fromHexString("03"))); + Assert.assertTrue(router1.receivedBlocks.isEmpty()); + Assert.assertTrue(router2.receivedBlocks.isEmpty()); + + Connection connection2 = router1.connect(router2); + + router1.router.sendProposedBlock(dataFactory.createBeaconBlock(fromHexString("04"))); + Assert.assertTrue(router1.receivedBlocks.isEmpty()); + Assert.assertEquals(1, router2.receivedBlocks.size()); + + router2.clear(); + + TestRouter router3 = new TestRouter(); + Connection connection3 = router2.connect(router3); + + router1.router.sendProposedBlock(dataFactory.createBeaconBlock(fromHexString("05"))); + Assert.assertTrue(router1.receivedBlocks.isEmpty()); + Assert.assertEquals(1, router2.receivedBlocks.size()); + Assert.assertEquals(1, router3.receivedBlocks.size()); + } + + @Test + public void testMisc() { + TestDataFactory dataFactory = new TestDataFactory(); + BeaconBlock b1 = dataFactory.createBeaconBlock(fromHexString("01")); + BeaconBlock b2 = dataFactory.createBeaconBlock(fromHexString("01")); + Assert.assertTrue(b1.equals(b2)); + Assert.assertEquals(b1.hashCode(), b2.hashCode()); + HashSet set = new HashSet<>(); + Assert.assertTrue(set.add(b1)); + Assert.assertFalse(set.add(b2)); + } +} diff --git a/wire/src/test/java/org/ethereum/beacon/wire/channel/BeaconPipelineChannelTest.java b/wire/src/test/java/org/ethereum/beacon/wire/channel/BeaconPipelineChannelTest.java new file mode 100644 index 000000000..e543987ec --- /dev/null +++ b/wire/src/test/java/org/ethereum/beacon/wire/channel/BeaconPipelineChannelTest.java @@ -0,0 +1,234 @@ +package org.ethereum.beacon.wire.channel; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.ethereum.beacon.core.types.SlotNumber; +import org.ethereum.beacon.schedulers.ControlledSchedulers; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.ssz.SSZBuilder; +import org.ethereum.beacon.ssz.SSZSerializer; +import org.ethereum.beacon.wire.Feedback; +import org.ethereum.beacon.wire.WireApiSync; +import org.ethereum.beacon.wire.channel.beacon.BeaconPipeline; +import org.ethereum.beacon.wire.message.Message; +import org.ethereum.beacon.wire.message.payload.BlockBodiesRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockBodiesResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockHeadersRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockHeadersResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsResponseMessage.BlockRootSlot; +import org.junit.Assert; +import org.junit.Test; +import org.reactivestreams.Processor; +import org.reactivestreams.Publisher; +import reactor.core.publisher.DirectProcessor; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxSink; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.uint.UInt64; + +public class BeaconPipelineChannelTest { + + + static class SimpleChannel implements Channel { + DirectProcessor in; + DirectProcessor out; + FluxSink inSink; + FluxSink outSink; + + public SimpleChannel(DirectProcessor in, DirectProcessor out) { + this.in = in; + this.out = out; + inSink = in.sink(); + outSink = out.sink(); + } + + @Override + public Publisher inboundMessageStream() { + return Flux.from(in).doOnError(Throwable::printStackTrace); + } + + @Override + public void subscribeToOutbound(Publisher outboundMessageStream) { + Flux.from(outboundMessageStream).doOnError(Throwable::printStackTrace).subscribe(out); + } + + public void close() { + inSink.complete(); + outSink.complete(); + } + } + + static class DummyWireApiSync implements WireApiSync { + + private final Scheduler scheduler; + private final Duration responseDuration; + + public DummyWireApiSync(Scheduler scheduler, Duration responseDuration) { + this.scheduler = scheduler; + this.responseDuration = responseDuration; + } + + @Override + public CompletableFuture requestBlockRoots( + BlockRootsRequestMessage requestMessage) { + + CompletableFuture ret = new CompletableFuture<>(); + scheduler.executeWithDelay( + responseDuration, + () -> + ret.complete( + new BlockRootsResponseMessage( + Collections.singletonList( + new BlockRootSlot(Hash32.ZERO, SlotNumber.of(666)))))); + return ret; + } + + @Override + public CompletableFuture requestBlockHeaders( + BlockHeadersRequestMessage requestMessage) { + return null; + } + + @Override + public CompletableFuture> requestBlockBodies( + BlockBodiesRequestMessage requestMessage) { + return null; + } + }; + + ; + + @Test + public void simpleTest1() throws Exception { + Schedulers schedulers = Schedulers.createDefault(); + DummyWireApiSync dummyServer = new DummyWireApiSync(schedulers.blocking(), Duration.ZERO); + + DirectProcessor _1to2 = DirectProcessor.create(); + DirectProcessor _2to1 = DirectProcessor.create(); + + SSZSerializer sszSerializer = new SSZBuilder().buildSerializer(); + + BeaconPipeline peer1Pipeline = + new BeaconPipeline(sszSerializer, null, null, dummyServer, schedulers); + SimpleChannel peer1Channel = new SimpleChannel<>(_2to1, _1to2); + peer1Pipeline.initFromMessageChannel(peer1Channel); + + BeaconPipeline peer2Pipeline = + new BeaconPipeline(sszSerializer, null, null, dummyServer, schedulers); + SimpleChannel peer2Channel = new SimpleChannel<>(_1to2, _2to1); + peer2Pipeline.initFromMessageChannel(peer2Channel); + + WireApiSync peer2SyncClient = peer2Pipeline.getSyncClient(); + + CompletableFuture resp = peer2SyncClient + .requestBlockRoots(new BlockRootsRequestMessage(SlotNumber.ZERO, UInt64.ZERO)); + BlockRootsResponseMessage responseMessage = resp.get(1, TimeUnit.SECONDS); + System.out.println(responseMessage); + Assert.assertEquals(SlotNumber.of(666), responseMessage.getRoots().get(0).getSlot()); + } + + @Test + public void closeTest1() throws Exception { + + DummyWireApiSync dummyServer = new DummyWireApiSync(Schedulers.createDefault().blocking(), Duration.ofMillis(50)); + + DirectProcessor _1to2 = DirectProcessor.create(); + DirectProcessor _2to1 = DirectProcessor.create(); + + SSZSerializer sszSerializer = new SSZBuilder().buildSerializer(); + ControlledSchedulers schedulers = Schedulers.createControlled(); + + BeaconPipeline peer1Pipeline = + new BeaconPipeline(sszSerializer, null, null, dummyServer, schedulers); + SimpleChannel peer1Channel = new SimpleChannel<>(_2to1, _1to2); + peer1Pipeline.initFromMessageChannel(peer1Channel); + + BeaconPipeline peer2Pipeline = + new BeaconPipeline(sszSerializer, null, null, dummyServer, schedulers); + SimpleChannel peer2Channel = new SimpleChannel<>(_1to2, _2to1); + peer2Pipeline.initFromMessageChannel(peer2Channel); + + WireApiSync peer2SyncClient = peer2Pipeline.getSyncClient(); + + CountDownLatch closeLatch = new CountDownLatch(10); + new Thread(() -> { + try { + closeLatch.await(); + System.out.println("Closing the channel"); + peer1Channel.close(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }).start(); + + List> futs = new ArrayList<>(); + + for (int i = 0; i < 20; i++) { + System.out.println("Sending request #" + i); + CompletableFuture resp = peer2SyncClient + .requestBlockRoots(new BlockRootsRequestMessage(SlotNumber.ZERO, UInt64.ZERO)); + futs.add(resp); + int finalI = i; + resp.whenComplete( + (r, t) -> { + System.out.println("Call #" + finalI + " complete with " + r + ", " + t); + closeLatch.countDown(); + }); + Thread.sleep(10); + } + + schedulers.addTime(Duration.ofMinutes(10)); + + Assert.assertTrue(futs.stream().allMatch(fut -> fut.isDone())); + Assert.assertTrue(futs.stream().anyMatch(fut -> fut.isCompletedExceptionally())); + Assert.assertTrue(futs.stream().anyMatch(fut -> !fut.isCompletedExceptionally())); + } + + @Test + public void timeoutTest1() throws Exception { + + DummyWireApiSync dummyServer = new DummyWireApiSync(Schedulers.createDefault().blocking(), Duration.ofDays(100500)); + + DirectProcessor _1to2 = DirectProcessor.create(); + DirectProcessor _2to1 = DirectProcessor.create(); + + SSZSerializer sszSerializer = new SSZBuilder().buildSerializer(); + ControlledSchedulers schedulers = Schedulers.createControlled(); + + BeaconPipeline peer1Pipeline = + new BeaconPipeline(sszSerializer, null, null, dummyServer, schedulers); + SimpleChannel peer1Channel = new SimpleChannel<>(_2to1, _1to2); + peer1Pipeline.initFromMessageChannel(peer1Channel); + + BeaconPipeline peer2Pipeline = + new BeaconPipeline(sszSerializer, null, null, dummyServer, schedulers); + SimpleChannel peer2Channel = new SimpleChannel<>(_1to2, _2to1); + peer2Pipeline.initFromMessageChannel(peer2Channel); + + WireApiSync peer2SyncClient = peer2Pipeline.getSyncClient(); + + System.out.println("Sending request..."); + CompletableFuture resp = peer2SyncClient + .requestBlockRoots(new BlockRootsRequestMessage(SlotNumber.ZERO, UInt64.ZERO)); + resp.whenComplete((r, t) -> System.out.println("Call complete with " + r + ", " + t)); + + schedulers.addTime(Duration.ofMillis(500)); + + Assert.assertFalse(resp.isDone()); + + schedulers.addTime(Duration.ofMinutes(10)); + + Assert.assertTrue(resp.isDone()); + Assert.assertTrue(resp.isCompletedExceptionally()); + } +} diff --git a/wire/src/test/java/org/ethereum/beacon/wire/net/ConnectionManagerTest.java b/wire/src/test/java/org/ethereum/beacon/wire/net/ConnectionManagerTest.java new file mode 100644 index 000000000..8cb61be64 --- /dev/null +++ b/wire/src/test/java/org/ethereum/beacon/wire/net/ConnectionManagerTest.java @@ -0,0 +1,93 @@ +package org.ethereum.beacon.wire.net; + +import io.netty.channel.ConnectTimeoutException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import org.ethereum.beacon.schedulers.ControlledSchedulers; +import org.ethereum.beacon.schedulers.Schedulers; +import org.ethereum.beacon.wire.channel.Channel; +import org.junit.Assert; +import org.junit.Test; +import org.reactivestreams.Publisher; +import reactor.core.publisher.DirectProcessor; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public class ConnectionManagerTest { + + class TestClient implements Client { + List>> connections = new ArrayList<>(); + + @Override + public CompletableFuture> connect(String s) { + CompletableFuture> ret = new CompletableFuture<>(); + connections.add(ret); + return ret; + } + } + + class TestChannel implements Channel { + DirectProcessor data = DirectProcessor.create(); + + @Override + public Publisher inboundMessageStream() { + return data; + } + + @Override + public void subscribeToOutbound(Publisher outboundMessageStream) { + } + + public void close() { + data.onComplete(); + } + } + + @Test + public void test1() { + List> channels = new ArrayList<>(); + + TestClient client = new TestClient(); + ControlledSchedulers schedulers = Schedulers.createControlled(); + ConnectionManager manager = new ConnectionManager<>(null, client, + schedulers.reactorEvents()); + Flux.from(manager.channelsStream()).subscribe(channels::add); + + manager.addActivePeer("1"); + Assert.assertEquals(1, client.connections.size()); + Assert.assertEquals(0, channels.size()); + schedulers.addTime(Duration.ofSeconds(10)); + Assert.assertEquals(1, client.connections.size()); + Assert.assertEquals(0, channels.size()); + Channel testChannel1 = new TestChannel(); + client.connections.get(0).complete(testChannel1); + Assert.assertEquals(1, channels.size()); + schedulers.addTime(Duration.ofSeconds(10)); + Assert.assertEquals(1, client.connections.size()); + testChannel1.close(); + Assert.assertEquals(1, client.connections.size()); + schedulers.addTime(Duration.ofMillis(500)); + Assert.assertEquals(1, client.connections.size()); + schedulers.addTime(Duration.ofSeconds(10)); + Assert.assertEquals(2, client.connections.size()); + Assert.assertEquals(1, channels.size()); + + client.connections.get(1).completeExceptionally(new ConnectTimeoutException()); + Assert.assertEquals(1, channels.size()); + Assert.assertEquals(2, client.connections.size()); + schedulers.addTime(Duration.ofSeconds(10)); + Assert.assertEquals(3, client.connections.size()); + TestChannel testChannel2 = new TestChannel(); + client.connections.get(2).complete(testChannel2); + Assert.assertEquals(2, channels.size()); + testChannel2.close(); + + Assert.assertEquals(3, client.connections.size()); + manager.removeActivePeer("1", true); + schedulers.addTime(Duration.ofSeconds(10)); + Assert.assertEquals(3, client.connections.size()); + } +} diff --git a/wire/src/test/java/org/ethereum/beacon/wire/net/NettyChannelTest.java b/wire/src/test/java/org/ethereum/beacon/wire/net/NettyChannelTest.java new file mode 100644 index 000000000..c227702df --- /dev/null +++ b/wire/src/test/java/org/ethereum/beacon/wire/net/NettyChannelTest.java @@ -0,0 +1,99 @@ +package org.ethereum.beacon.wire.net; + +import java.net.InetSocketAddress; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.ethereum.beacon.wire.net.netty.NettyChannel; +import org.ethereum.beacon.wire.net.netty.NettyClient; +import org.ethereum.beacon.wire.net.netty.NettyServer; +import org.junit.Assert; +import org.junit.Test; +import reactor.core.publisher.Flux; +import tech.pegasys.artemis.util.bytes.BytesValue; + +public class NettyChannelTest { + + @Test + public void test1() throws Exception { + NettyServer nettyServer = new NettyServer(26666); + NettyClient nettyClient = new NettyClient(); + + System.out.println("Starting server..."); + nettyServer.start().await(); + System.out.println("Server started"); + + System.out.println("Connecting 1..."); + CompletableFuture chFut1 = + nettyClient.connect(InetSocketAddress.createUnresolved("localhost", 26666)); + NettyChannel ch1 = chFut1.get(5, TimeUnit.SECONDS); + System.out.println("Client channel 1 created"); + + Flux.from(ch1.inboundMessageStream()) + .subscribe( + msg -> System.out.println("Client channel 1 message: " + msg), + err -> System.out.println("Client channel 1 error: " + err), + () -> System.out.println("Client channel 1 closed")); + ch1.subscribeToOutbound( + Flux.just(BytesValue.fromHexString("0x1111")) + .repeat(5) + .delayElements(Duration.ofMillis(20)) + .doOnNext(bb -> System.out.println("Sending msg from client 1..."))); + + Thread.sleep(200); + + CountDownLatch msgLatch = new CountDownLatch(10); + Flux.from(nettyServer.channelsStream()) + .subscribe( + nettyChannel -> { + System.out.println("Server channel created."); + Flux.from(nettyChannel.inboundMessageStream()) + .subscribe( + msg -> { + System.out.println("Server channel message: " + msg); + msgLatch.countDown(); + }, + err -> System.out.println("Server channel error: " + err), + () -> System.out.println("Server channel closed")); + }, + err -> System.out.println("Server error: " + err), + () -> System.out.println("Server socket closed")); + + System.out.println("Connecting 2..."); + CompletableFuture chFut2 = + nettyClient.connect(InetSocketAddress.createUnresolved("localhost", 26666)); + NettyChannel ch2 = chFut2.get(5, TimeUnit.SECONDS); + System.out.println("Client channel 2 created"); + + Flux.from(ch2.inboundMessageStream()) + .subscribe( + msg -> System.out.println("Client channel 2 message: " + msg), + err -> System.out.println("Client channel 2 error: " + err), + () -> System.out.println("Client channel 2 closed")); + ch2.subscribeToOutbound( + Flux.just(BytesValue.fromHexString("0x2222")) + .repeat(5) + .delayElements(Duration.ofMillis(20)) + .doOnNext(bb -> System.out.println("Sending msg from client 2..."))); + + System.out.println("Waiting for all messages on the server..."); + Assert.assertTrue(msgLatch.await(2, TimeUnit.SECONDS)); + System.out.println("Stopping server..."); + nettyServer.stop(); + + Thread.sleep(1000); + System.out.println("Complete"); + } + + @Test(expected = ExecutionException.class) + public void test2() throws Exception { + NettyClient nettyClient = new NettyClient(); + + System.out.println("Connecting 1..."); + CompletableFuture chFut1 = + nettyClient.connect(InetSocketAddress.createUnresolved("localhost", 26667)); + NettyChannel ch1 = chFut1.get(5, TimeUnit.SECONDS); + } +} diff --git a/wire/src/test/java/org/ethereum/beacon/wire/sync/SyncTest.java b/wire/src/test/java/org/ethereum/beacon/wire/sync/SyncTest.java new file mode 100644 index 000000000..22128f373 --- /dev/null +++ b/wire/src/test/java/org/ethereum/beacon/wire/sync/SyncTest.java @@ -0,0 +1,110 @@ +package org.ethereum.beacon.wire.sync; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ethereum.beacon.start.common.Launcher; +import org.ethereum.beacon.schedulers.Scheduler; +import org.ethereum.beacon.simulator.SimulatorLauncher; +import org.ethereum.beacon.simulator.SimulatorLauncher.Builder; +import org.ethereum.beacon.wire.Feedback; +import org.ethereum.beacon.wire.WireApiSync; +import org.ethereum.beacon.wire.WireApiSyncServer; +import org.ethereum.beacon.wire.message.payload.BlockBodiesRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockBodiesResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockHeadersRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockHeadersResponseMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsRequestMessage; +import org.ethereum.beacon.wire.message.payload.BlockRootsResponseMessage; +import org.junit.Assert; +import org.junit.Test; +import reactor.core.publisher.Flux; + +public class SyncTest { + private static final Logger logger = LogManager.getLogger(SyncTest.class); + + static class AsyncWireApiSync implements WireApiSync { + WireApiSync delegate; + Scheduler scheduler; + Duration delay; + + public AsyncWireApiSync(WireApiSync delegate, Scheduler scheduler, Duration delay) { + this.delegate = delegate; + this.scheduler = scheduler; + this.delay = delay; + } + + @Override + public CompletableFuture requestBlockRoots( + BlockRootsRequestMessage requestMessage) { + return scheduler.executeWithDelay(delay, () -> delegate.requestBlockRoots(requestMessage).get()); + } + + @Override + public CompletableFuture requestBlockHeaders( + BlockHeadersRequestMessage requestMessage) { + System.out.println("Headers requested: " + requestMessage); + return scheduler.executeWithDelay(delay, () -> delegate.requestBlockHeaders(requestMessage).get()); + } + + @Override + public CompletableFuture> requestBlockBodies( + BlockBodiesRequestMessage requestMessage) { + return scheduler.executeWithDelay(delay, () -> delegate.requestBlockBodies(requestMessage).get()); + } + } + + @Test(timeout = 30000) + public void test1() throws Exception { + int slotCount = 64; + SimulatorLauncher simulatorLauncher = new Builder() + .withConfigFromResource("/sync-simulation-config.yml") + .withLogLevel(null) + .build(); + simulatorLauncher.run(slotCount); + Launcher peer0 = simulatorLauncher.getPeers().get(0); + System.out.println(peer0); + + Launcher testPeer = simulatorLauncher.createPeer("test"); + + WireApiSyncServer syncServer = new WireApiSyncServer(peer0.getBeaconChainStorage()); + AsyncWireApiSync asyncSyncServer = new AsyncWireApiSync(syncServer, + testPeer.getSchedulers().blocking(), Duration.ofMillis(10)); + + BeaconBlockTree blockTree = new BeaconBlockTree(simulatorLauncher.getSpec().getObjectHasher()); + SyncQueue syncQueue = new SyncQueueImpl(blockTree, 4, 20); + + SyncManagerImpl syncManager = new SyncManagerImpl( + testPeer.getBeaconChain(), + Flux.never(), + testPeer.getBeaconChainStorage(), + testPeer.getSpec(), + asyncSyncServer, + syncQueue, + 1, + testPeer.getSchedulers().reactorEvents()); + + AtomicBoolean synced = new AtomicBoolean(); + Flux.from(testPeer.getBeaconChain().getBlockStatesStream()) + .subscribe(s -> { + System.out.println(s); + if (s.getFinalState().getSlot().equals( + simulatorLauncher.getSpec().getConstants().getGenesisSlot().plus(slotCount))) { + syncManager.stop(); + synced.set(true); + } + }); + + System.out.println("Starting sync manager..."); + syncManager.start(); + + System.out.println("Adding 3 seconds..."); + simulatorLauncher.getControlledSchedulers().addTime(5000); + + Assert.assertTrue(synced.get()); + System.out.println("Done"); + } + +} diff --git a/wire/src/test/resources/log4j2.xml b/wire/src/test/resources/log4j2.xml new file mode 100644 index 000000000..b8d50f0b5 --- /dev/null +++ b/wire/src/test/resources/log4j2.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + %d{HH:mm:ss.SSS} [%X{validatorTime}] %p %c{1.} [%t] %m%n + + + + + + + + + %d{HH:mm:ss.SSS} [%X{validatorTime}] %p %c{1.} [%t] #%X{validatorIndex} %m%n + + + + + + %d{HH:mm:ss.SSS} #%X{validatorIndex} %X{validatorTime} %-5level - %msg%n + + + + + + + + + + + + + + + + + diff --git a/wire/src/test/resources/sync-simulation-config.yml b/wire/src/test/resources/sync-simulation-config.yml new file mode 100644 index 000000000..737fe5e13 --- /dev/null +++ b/wire/src/test/resources/sync-simulation-config.yml @@ -0,0 +1,37 @@ +plan: !simulation + seed: 1 + genesisTime: 600 + peers: + - count: 8 + validator: true + systemTimeShift: 0 + wireInboundDelay: 0 + wireOutboundDelay: 0 + - count: 1 + validator: false + +chainSpec: + specConstants: + initialValues: + GENESIS_SLOT: 1000000 + miscParameters: + SHARD_COUNT: 4 + TARGET_COMMITTEE_SIZE: 2 + timeParameters: + SECONDS_PER_SLOT: 10 + MIN_ATTESTATION_INCLUSION_DELAY: 1 + SLOTS_PER_EPOCH: 4 + SLOTS_PER_HISTORICAL_ROOT: 64 + + honestValidatorParameters: + ETH1_FOLLOW_DISTANCE: 1 + stateListLengths: + LATEST_RANDAO_MIXES_LENGTH: 64 + LATEST_ACTIVE_INDEX_ROOTS_LENGTH: 64 + LATEST_SLASHED_EXIT_LENGTH: 64 + + specHelpersOptions: + blsVerify: false + blsVerifyProofOfPossession: false + blsSign: false + enableCache: false diff --git a/wire/src/test/resources/test-spec-config.yml b/wire/src/test/resources/test-spec-config.yml new file mode 100644 index 000000000..4a0f1d189 --- /dev/null +++ b/wire/src/test/resources/test-spec-config.yml @@ -0,0 +1,25 @@ +specConstants: + initialValues: + GENESIS_SLOT: 1000000 + miscParameters: + SHARD_COUNT: 4 + TARGET_COMMITTEE_SIZE: 2 + timeParameters: + SECONDS_PER_SLOT: 10 + MIN_ATTESTATION_INCLUSION_DELAY: 1 + SLOTS_PER_EPOCH: 4 + SLOTS_PER_HISTORICAL_ROOT: 64 + + honestValidatorParameters: + ETH1_FOLLOW_DISTANCE: 1 + stateListLengths: + LATEST_RANDAO_MIXES_LENGTH: 64 + LATEST_ACTIVE_INDEX_ROOTS_LENGTH: 64 + LATEST_SLASHED_EXIT_LENGTH: 64 + +specHelpersOptions: + blsVerify: false + blsVerifyProofOfPossession: false + blsSign: false + enableCache: false +