From 7d47ef9c1e3676694bec46fe438edd24c0ff9f81 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sat, 11 May 2019 15:00:00 +0300 Subject: [PATCH 01/14] Added merkle tree realization for deposit contract --- .../ethereum/beacon/pow/MinimalMerkle.java | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java new file mode 100644 index 000000000..370cd1155 --- /dev/null +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java @@ -0,0 +1,158 @@ +package org.ethereum.beacon.pow; + +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static tech.pegasys.artemis.util.bytes.BytesValue.concat; + +/** + * Minimal merkle tree https://en.wikipedia.org/wiki/Merkle_tree + * implementation + */ +public class MinimalMerkle { + + private final int treeDepth; + private final Hash32[] zeroHashes; + private final Function hashFunction; + + public MinimalMerkle(Function hashFunction, int treeDepth) { + this.hashFunction = hashFunction; + this.treeDepth = treeDepth; + this.zeroHashes = new Hash32[treeDepth]; + } + + public Hash32 getZeroHash(int distanceFromBottom) { + if (zeroHashes[distanceFromBottom] == null) { + if (distanceFromBottom == 0) { + zeroHashes[0] = Hash32.ZERO; + } else { + Hash32 lowerZeroHash = getZeroHash(distanceFromBottom - 1); + zeroHashes[distanceFromBottom] = hashFunction.apply(concat(lowerZeroHash, lowerZeroHash)); + } + } + return zeroHashes[distanceFromBottom]; + } + + // # Compute a Merkle root of a right-zerobyte-padded 2**32 sized tree + // def calc_merkle_tree_from_leaves(values): + // values = list(values) + // tree = [values[::]] + // for h in range(32): + // if len(values) % 2 == 1: + // values.append(zerohashes[h]) + // values = [hash(values[i] + values[i+1]) for i in range(0, len(values), 2)] + // tree.append(values[::]) + // return tree + public List> calc_merkle_tree_from_leaves(List valueList) { + List values = new ArrayList<>(valueList); + List> tree = new ArrayList<>(); + tree.add(values); + for (int h = 0; h < treeDepth; ++h) { + if (values.size() % 2 == 1) { + values.add(getZeroHash(h)); + } + List valuesTemp = new ArrayList<>(); + for (int i = 0; i < values.size(); i += 2) { + valuesTemp.add(hashFunction.apply(values.get(i).concat(values.get(i + 1)))); + } + values = valuesTemp; + tree.add(values); + } + + return tree; + } + + // def get_merkle_root(values): + // return calc_merkle_tree_from_leaves(values)[-1][0] + public BytesValue get_merkle_root(List values) { + List> tree = calc_merkle_tree_from_leaves(values); + try { + return tree.get(tree.size() - 1).get(0); + } catch (Exception ex) { + throw ex; + } + } + + // def get_merkle_proof(tree, item_index): + // proof = [] + // for i in range(32): + // subindex = (item_index//2**i)^1 + // proof.append(tree[i][subindex] if subindex < len(tree[i]) else zerohashes[i]) + // return proof + public List get_merkle_proof(List> tree, int item_index) { + List proof = new ArrayList<>(); + for (int i = 0; i < treeDepth; ++i) { + int subIndex = (item_index / (1 << i)) ^ 1; + if (subIndex < tree.get(i).size()) { + proof.add(Hash32.wrap(Bytes32.leftPad(tree.get(i).get(subIndex)))); + } else { + proof.add(getZeroHash(i)); + } + } + + return proof; + } + + // Progressive Merkle + public List getZeroHashes() { + return IntStream.range(0, 32) + .mapToObj(this::getZeroHash) + .map(h -> h.slice(0)) + .collect(Collectors.toList()); + } + + // # Add a value to a Merkle tree by using the algo + // # that stores a branch of sub-roots + // def add_value(branch, index, value): + // i = 0 + // while (index+1) % 2**(i+1) == 0: + // i += 1 + // for j in range(0, i): + // value = hash(branch[j] + value) + // # branch[j] = zerohashes[j] + // branch[i] = value + + List add_value(List branch, int index, BytesValue value) { + int i = 0; + while ((index + 1) % (1 << (i + 1)) == 0) { + ++i; + } + + BytesValue valueCopy = value; + for (int j = 0; j < i; ++j) { + valueCopy = hashFunction.apply(branch.get(j).concat(valueCopy)); + } + branch.set(i, valueCopy); + + return branch; + } + + // def get_root_from_branch(branch, size): + // r = b'\x00' * 32 + // for h in range(32): + // if (size >> h) % 2 == 1: + // r = hash(branch[h] + r) + // else: + // r = hash(r + zerohashes[h]) + // return r + BytesValue get_root_from_branch(List branch, int size) { + BytesValue r = Bytes32.ZERO.slice(0); + for (int h = 0; h < treeDepth; ++h) { + if ((size >> h) % 2 == 1) { + r = hashFunction.apply(branch.get(h).concat(r)); + } else { + r = hashFunction.apply(r.concat(getZeroHash(h))); + } + } + + return r; + } +} From d31cade9b63be13f4d569b371ca851360bb7079e Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sat, 11 May 2019 15:00:42 +0300 Subject: [PATCH 02/14] Remove unused progressive methods from minimal merkle tree realization --- .../ethereum/beacon/pow/MinimalMerkle.java | 55 ------------------- 1 file changed, 55 deletions(-) diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java index 370cd1155..0f8489475 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java @@ -100,59 +100,4 @@ public List get_merkle_proof(List> tree, int item_index return proof; } - - // Progressive Merkle - public List getZeroHashes() { - return IntStream.range(0, 32) - .mapToObj(this::getZeroHash) - .map(h -> h.slice(0)) - .collect(Collectors.toList()); - } - - // # Add a value to a Merkle tree by using the algo - // # that stores a branch of sub-roots - // def add_value(branch, index, value): - // i = 0 - // while (index+1) % 2**(i+1) == 0: - // i += 1 - // for j in range(0, i): - // value = hash(branch[j] + value) - // # branch[j] = zerohashes[j] - // branch[i] = value - - List add_value(List branch, int index, BytesValue value) { - int i = 0; - while ((index + 1) % (1 << (i + 1)) == 0) { - ++i; - } - - BytesValue valueCopy = value; - for (int j = 0; j < i; ++j) { - valueCopy = hashFunction.apply(branch.get(j).concat(valueCopy)); - } - branch.set(i, valueCopy); - - return branch; - } - - // def get_root_from_branch(branch, size): - // r = b'\x00' * 32 - // for h in range(32): - // if (size >> h) % 2 == 1: - // r = hash(branch[h] + r) - // else: - // r = hash(r + zerohashes[h]) - // return r - BytesValue get_root_from_branch(List branch, int size) { - BytesValue r = Bytes32.ZERO.slice(0); - for (int h = 0; h < treeDepth; ++h) { - if ((size >> h) % 2 == 1) { - r = hashFunction.apply(branch.get(h).concat(r)); - } else { - r = hashFunction.apply(r.concat(getZeroHash(h))); - } - } - - return r; - } } From 151269a1881d2accb33a2aab76f9ce8174ffe137 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sat, 11 May 2019 15:01:09 +0300 Subject: [PATCH 03/14] Update deposit contract ABI --- .../org/ethereum/beacon/pow/ContractAbi.json | 97 +++++++++++++++++-- 1 file changed, 88 insertions(+), 9 deletions(-) diff --git a/pow/core/src/main/resources/org/ethereum/beacon/pow/ContractAbi.json b/pow/core/src/main/resources/org/ethereum/beacon/pow/ContractAbi.json index c4b866c65..886eb97b9 100644 --- a/pow/core/src/main/resources/org/ethereum/beacon/pow/ContractAbi.json +++ b/pow/core/src/main/resources/org/ethereum/beacon/pow/ContractAbi.json @@ -3,23 +3,28 @@ "name": "Deposit", "inputs": [ { - "type": "bytes32", - "name": "deposit_root", + "type": "bytes", + "name": "pubkey", "indexed": false }, { "type": "bytes", - "name": "data", + "name": "withdrawal_credentials", "indexed": false }, { "type": "bytes", - "name": "merkle_tree_index", + "name": "amount", + "indexed": false + }, + { + "type": "bytes", + "name": "signature", "indexed": false }, { - "type": "bytes32[32]", - "name": "branch", + "type": "bytes", + "name": "merkle_tree_index", "indexed": false } ], @@ -27,13 +32,18 @@ "type": "event" }, { - "name": "ChainStart", + "name": "Eth2Genesis", "inputs": [ { "type": "bytes32", "name": "deposit_root", "indexed": false }, + { + "type": "bytes", + "name": "deposit_count", + "indexed": false + }, { "type": "bytes", "name": "time", @@ -44,13 +54,48 @@ "type": "event" }, { - "name": "__init__", "outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor" }, + { + "name": "to_little_endian_64", + "outputs": [ + { + "type": "bytes", + "name": "out" + } + ], + "inputs": [ + { + "type": "uint256", + "name": "value" + } + ], + "constant": true, + "payable": false, + "type": "function" + }, + { + "name": "from_little_endian_64", + "outputs": [ + { + "type": "uint256", + "name": "out" + } + ], + "inputs": [ + { + "type": "bytes", + "name": "value" + } + ], + "constant": true, + "payable": false, + "type": "function" + }, { "name": "get_deposit_root", "outputs": [ @@ -64,17 +109,51 @@ "payable": false, "type": "function" }, + { + "name": "get_deposit_count", + "outputs": [ + { + "type": "bytes", + "name": "out" + } + ], + "inputs": [], + "constant": true, + "payable": false, + "type": "function" + }, { "name": "deposit", "outputs": [], "inputs": [ { "type": "bytes", - "name": "deposit_input" + "name": "pubkey" + }, + { + "type": "bytes", + "name": "withdrawal_credentials" + }, + { + "type": "bytes", + "name": "signature" } ], "constant": false, "payable": true, "type": "function" + }, + { + "name": "chainStarted", + "outputs": [ + { + "type": "bool", + "name": "out" + } + ], + "inputs": [], + "constant": true, + "payable": false, + "type": "function" } ] \ No newline at end of file From 42f66bb0185e2a9c49acb84c40388d6d9ac09892 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sat, 11 May 2019 15:16:44 +0300 Subject: [PATCH 04/14] Update deposit contract usage logic according spec v0.6.1 --- .../beacon/pow/AbstractDepositContract.java | 297 ++++++++++----- .../beacon/pow/EthereumJDepositContract.java | 171 +++++---- .../pow/StandaloneDepositContractTest.java | 346 ++++++++++++++---- 3 files changed, 590 insertions(+), 224 deletions(-) diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java b/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java index b7fad35ae..f83667de0 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java @@ -1,11 +1,5 @@ package org.ethereum.beacon.pow; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; import org.ethereum.beacon.core.operations.Deposit; import org.ethereum.beacon.core.operations.deposit.DepositData; import org.ethereum.beacon.core.state.Eth1Data; @@ -18,6 +12,7 @@ import org.ethereum.beacon.ssz.SSZSerializer; import org.ethereum.beacon.stream.SimpleProcessor; import org.javatuples.Pair; +import org.javatuples.Triplet; import org.reactivestreams.Publisher; import reactor.core.publisher.MonoProcessor; import tech.pegasys.artemis.ethereum.core.Hash32; @@ -25,63 +20,120 @@ import tech.pegasys.artemis.util.bytes.Bytes48; import tech.pegasys.artemis.util.bytes.Bytes8; import tech.pegasys.artemis.util.bytes.Bytes96; +import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.collections.ReadVector; import tech.pegasys.artemis.util.uint.UInt64; -public abstract class AbstractDepositContract implements DepositContract { - protected class DepositEventData { - public final byte[] deposit_root; - public final byte[] data; - public final byte[] merkle_tree_index; - public final byte[][] merkle_branch; - - public DepositEventData(byte[] deposit_root, byte[] data, byte[] merkle_tree_index, - byte[][] merkle_branch) { - this.deposit_root = deposit_root; - this.data = data; - this.merkle_tree_index = merkle_tree_index; - this.merkle_branch = merkle_branch; - } - } - - private final SSZSerializer ssz = new SSZBuilder().buildSerializer(); - - private long distanceFromHead; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +public abstract class AbstractDepositContract implements DepositContract { protected final Schedulers schedulers; + private final SSZSerializer ssz = new SSZBuilder().buildSerializer(); private final MonoProcessor chainStartSink = MonoProcessor.create(); private final Publisher chainStartStream; - private final SimpleProcessor depositStream; - + private final MinimalMerkle minimalMerkle; + private final Function hashFunction; + private long distanceFromHead; private List initialDeposits = new ArrayList<>(); private boolean startChainSubscribed; - - public AbstractDepositContract(Schedulers schedulers) { + private List deposits = new ArrayList<>(); + public AbstractDepositContract( + Schedulers schedulers, Function hashFunction, int treeDepth) { this.schedulers = schedulers; - chainStartStream = chainStartSink - .publishOn(this.schedulers.reactorEvents()) - .doOnSubscribe(s -> chainStartSubscribedPriv()) - .name("PowClient.chainStart"); + chainStartStream = + chainStartSink + .publishOn(this.schedulers.reactorEvents()) + .doOnSubscribe(s -> chainStartSubscribedPriv()) + .name("PowClient.chainStart"); depositStream = new SimpleProcessor<>(this.schedulers.reactorEvents(), "PowClient.deposit"); + this.hashFunction = hashFunction; + this.minimalMerkle = new MinimalMerkle(hashFunction, treeDepth); + } + + /** + * Stores deposits data from invocation list eventDataList + * @param eventDataList All deposit events in blockHash + * @param blockHash Block hash + */ + protected synchronized void newDeposits(List eventDataList, byte[] blockHash) { + List deposits = + eventDataList.stream().map(this::newDeposit).collect(Collectors.toList()); + if (deposits.isEmpty()) { + return; + } + int size = deposits.get(deposits.size() - 1).getIndex().increment().intValue(); + for (Deposit deposit : deposits) { + Deposit depositProofed = + new Deposit( + getProof(deposit.getIndex().intValue(), size), deposit.getIndex(), deposit.getData()); + if (startChainSubscribed && !chainStartSink.isTerminated()) { + initialDeposits.add(depositProofed); + } + depositStream.onNext(depositProofed); + } } - protected synchronized void newDeposit(DepositEventData eventData, byte[] blockHash) { - if (startChainSubscribed && !chainStartSink.isTerminated()) { - DepositInfo depositInfo = createDepositInfo(eventData, blockHash); - initialDeposits.add(depositInfo.getDeposit()); - depositStream.onNext(depositInfo.getDeposit()); + /** + * Same as {@link #newDeposits(List, byte[])} but doesn't store deposits data, instead expects its + * already stored + */ + private List restoreDeposits(List eventData, byte[] blockHash) { + List deposits = + eventData.stream().map(this::createUnProofedDeposit).collect(Collectors.toList()); + if (deposits.isEmpty()) { + return Collections.emptyList(); } + int size = deposits.get(0).getIndex().plus(deposits.size()).intValue(); + return deposits.stream() + .map( + deposit -> + new Deposit( + getProof(deposit.getIndex().intValue(), size), + deposit.getIndex(), + deposit.getData())) + .map( + d -> + new DepositInfo( + d, + new Eth1Data( + getDepositRoot(d.getIndex()), + d.getIndex().decrement(), + Hash32.wrap(Bytes32.wrap(blockHash))))) + .collect(Collectors.toList()); + } + + /** + * Inserts deposit in storage and returns it + * NOTE: returns Deposit with empty proof, proof should be filled by someone else + * @param eventData Deposit event + * @return Deposit + */ + private Deposit newDeposit(DepositEventData eventData) { + Deposit deposit = createUnProofedDeposit(eventData); + insertDepositData(createDepositDataValue(deposit.getData()).extractArray()); + return deposit; } - protected synchronized void chainStart(byte[] deposit_root, byte[] time, byte[] blockHash) { - ChainStart chainStart = new ChainStart( - Time.castFrom(UInt64.fromBytesBigEndian(Bytes8.wrap(time))), - new Eth1Data(Hash32.wrap(Bytes32.wrap(deposit_root)), - UInt64.valueOf(initialDeposits.size()), - Hash32.wrap(Bytes32.wrap(blockHash))), - initialDeposits); + protected synchronized void chainStart( + byte[] deposit_root, byte[] deposit_count, byte[] time, byte[] blockHash) { + assert UInt64.fromBytesLittleEndian(Bytes8.wrap(deposit_count)).intValue() + == initialDeposits.size(); + ChainStart chainStart = + new ChainStart( + Time.castFrom(UInt64.fromBytesLittleEndian(Bytes8.wrap(time))), + new Eth1Data( + Hash32.wrap(Bytes32.wrap(deposit_root)), + UInt64.valueOf(initialDeposits.size()), + Hash32.wrap(Bytes32.wrap(blockHash))), + initialDeposits); chainStartSink.onNext(chainStart); chainStartSink.onComplete(); chainStartDone(); @@ -98,6 +150,66 @@ private void chainStartSubscribedPriv() { protected abstract void chainStartDone(); + protected ReadVector getProof(int index, int size) { + return ReadVector.wrap( + minimalMerkle.get_merkle_proof( + minimalMerkle.calc_merkle_tree_from_leaves(deposits.subList(0, size)), index), + Integer::new); + } + + private Hash32 getDepositRoot(UInt64 index) { + return Hash32.wrap( + Bytes32.leftPad(minimalMerkle.get_merkle_root(deposits.subList(0, index.intValue() + 1)))); + } + + protected Hash32 getDepositRoot(byte[] merkleTreeIndex) { + UInt64 index = UInt64.fromBytesLittleEndian(Bytes8.wrap(merkleTreeIndex)); + return getDepositRoot(index); + } + + private void insertDepositData(byte[] depositData) { + deposits.add(BytesValue.wrap(depositData)); + } + + // zero_bytes_32: bytes32 + // pubkey_root: bytes32 = sha256(concat(pubkey, slice(zero_bytes_32, start=0, len=16))) + // signature_root: bytes32 = sha256(concat( + // sha256(slice(signature, start=0, len=64)), + // sha256(concat(slice(signature, start=64, len=32), zero_bytes_32)) + // )) + // value: bytes32 = sha256(concat( + // sha256(concat(pubkey_root, withdrawal_credentials)), + // sha256(concat( + // amount, + // slice(zero_bytes_32, start=0, len=24), + // signature_root, + // )) + // )) + private Hash32 createDepositDataValue(DepositData depositData) { + BytesValue zero_bytes_32 = Bytes32.ZERO.slice(0); + Hash32 pubkey_root = + hashFunction.apply(depositData.getPubKey().concat(zero_bytes_32.slice(0, 16))); + Hash32 signature_root = + hashFunction.apply( + hashFunction + .apply(depositData.getSignature().slice(0, 64)) + .concat( + hashFunction.apply( + depositData.getSignature().slice(64, 32).concat(zero_bytes_32)))); + Hash32 value = + hashFunction.apply( + hashFunction + .apply(pubkey_root.concat(depositData.getWithdrawalCredentials())) + .concat( + hashFunction.apply( + depositData + .getAmount() + .toBytes8LittleEndian() + .concat(zero_bytes_32.slice(0, 24).concat(signature_root))))); + + return value; + } + @Override public Publisher getChainStartMono() { return chainStartStream; @@ -108,27 +220,15 @@ public Publisher getDepositStream() { return depositStream; } - private DepositInfo createDepositInfo(DepositEventData eventData, byte[] blockHash) { - List merkleBranch = Arrays.stream(eventData.merkle_branch) - .map(bytes -> Hash32.wrap(Bytes32.wrap(bytes))) - .collect(Collectors.toList()); - Deposit deposit = new Deposit(ReadVector.wrap(merkleBranch, Function.identity()), - UInt64.fromBytesBigEndian(Bytes8.wrap(eventData.merkle_tree_index)), - parseDepositData(eventData.data)); - return new DepositInfo(deposit, - new Eth1Data(Hash32.wrap(Bytes32.wrap(eventData.deposit_root)), - UInt64.ZERO, - Hash32.wrap(Bytes32.wrap(blockHash)))); - } - - private DepositData parseDepositData(byte[] data) { - BLSPubkey pubkey = BLSPubkey.wrap(Bytes48.wrap(data, 0)); - Hash32 withdrawalCredentials = Hash32.wrap(Bytes32.wrap(data, 48)); - Gwei amount = - Gwei.castFrom(UInt64.fromBytesLittleEndian(Bytes8.wrap(data, Bytes48.SIZE + Bytes32.SIZE))); - BLSSignature signature = - BLSSignature.wrap(Bytes96.wrap(data, Bytes48.SIZE + Bytes32.SIZE + Bytes8.SIZE)); - return new DepositData(pubkey, withdrawalCredentials, amount, signature); + private Deposit createUnProofedDeposit(DepositEventData eventData) { + UInt64 index = UInt64.fromBytesLittleEndian(Bytes8.wrap(eventData.merkleTreeIndex)); + DepositData depositData = + new DepositData( + BLSPubkey.wrap(Bytes48.wrap(eventData.pubkey)), + Hash32.wrap(Bytes32.wrap(eventData.withdrawalCredentials)), + Gwei.castFrom(UInt64.fromBytesLittleEndian(Bytes8.wrap(eventData.amount))), + BLSSignature.wrap(Bytes96.wrap(eventData.signature))); + return new Deposit(ReadVector.wrap(Collections.emptyList(), Integer::new), index, depositData); } @Override @@ -140,39 +240,62 @@ public boolean hasDepositRoot(Hash32 blockHash, Hash32 depositRoot) { @Override public Optional getLatestEth1Data() { - return getLatestBlockHashDepositRoot().map( - r -> new Eth1Data( - Hash32.wrap(Bytes32.wrap(r.getValue1())), - UInt64.ZERO, - Hash32.wrap(Bytes32.wrap(r.getValue0())))); + return getLatestBlockHashDepositRoot() + .map( + r -> + new Eth1Data( + Hash32.wrap(Bytes32.wrap(r.getValue0())), + UInt64.valueOf(r.getValue1()), + Hash32.wrap(Bytes32.wrap(r.getValue2())))); } - protected abstract Optional> getLatestBlockHashDepositRoot(); + protected abstract Optional> getLatestBlockHashDepositRoot(); @Override - public List peekDeposits(int count, Eth1Data fromDepositExclusive, - Eth1Data tillDepositInclusive) { - return peekDepositsImpl(count, - fromDepositExclusive.getBlockHash().extractArray(), - fromDepositExclusive.getDepositRoot().extractArray(), - tillDepositInclusive.getBlockHash().extractArray(), - tillDepositInclusive.getDepositRoot().extractArray()) + public List peekDeposits( + int count, Eth1Data fromDepositExclusive, Eth1Data tillDepositInclusive) { + return peekDepositsImpl( + count, + fromDepositExclusive.getBlockHash().extractArray(), + tillDepositInclusive.getBlockHash().extractArray()) .stream() - .map(blockDepositPair -> createDepositInfo(blockDepositPair.getValue1(), blockDepositPair.getValue0())) + .map( + blockDepositPair -> + restoreDeposits(blockDepositPair.getValue1(), blockDepositPair.getValue0())) + .flatMap(Collection::stream) .collect(Collectors.toList()); } - protected abstract List> peekDepositsImpl( - int count, - byte[] startBlockHash, byte[] startDepositRoot, - byte[] endBlockHash, byte[] endDepositRoot); + protected abstract List>> peekDepositsImpl( + int count, byte[] startBlockHash, byte[] endBlockHash); + + protected long getDistanceFromHead() { + return distanceFromHead; + } @Override public void setDistanceFromHead(long distanceFromHead) { this.distanceFromHead = distanceFromHead; } - protected long getDistanceFromHead() { - return distanceFromHead; + protected class DepositEventData { + public final byte[] pubkey; + public final byte[] withdrawalCredentials; + public final byte[] amount; + public final byte[] signature; + public final byte[] merkleTreeIndex; + + public DepositEventData( + byte[] pubkey, + byte[] withdrawalCredentials, + byte[] amount, + byte[] signature, + byte[] merkleTreeIndex) { + this.pubkey = pubkey; + this.withdrawalCredentials = withdrawalCredentials; + this.amount = amount; + this.signature = signature; + this.merkleTreeIndex = merkleTreeIndex; + } } -} \ No newline at end of file +} diff --git a/pow/ethereumj/src/main/java/org/ethereum/beacon/pow/EthereumJDepositContract.java b/pow/ethereumj/src/main/java/org/ethereum/beacon/pow/EthereumJDepositContract.java index aee64cd8a..e54c034ff 100644 --- a/pow/ethereumj/src/main/java/org/ethereum/beacon/pow/EthereumJDepositContract.java +++ b/pow/ethereumj/src/main/java/org/ethereum/beacon/pow/EthereumJDepositContract.java @@ -1,12 +1,5 @@ package org.ethereum.beacon.pow; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; import org.ethereum.beacon.schedulers.LatestExecutor; import org.ethereum.beacon.schedulers.Schedulers; import org.ethereum.core.Block; @@ -21,17 +14,30 @@ import org.ethereum.listener.EthereumListenerAdapter; import org.ethereum.vm.LogInfo; import org.javatuples.Pair; +import org.javatuples.Triplet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import tech.pegasys.artemis.ethereum.core.Address; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes8; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; public class EthereumJDepositContract extends AbstractDepositContract { private static final Logger logger = LoggerFactory.getLogger(EthereumJDepositContract.class); private static final String DEPOSIT_EVENT_NAME = "Deposit"; - private static final String CHAIN_START_EVENT_NAME = "ChainStart"; + private static final String CHAIN_START_EVENT_NAME = "Eth2Genesis"; private final LatestExecutor blockExecutor; @@ -45,9 +51,14 @@ public class EthereumJDepositContract extends AbstractDepositContract { private volatile long processedUpToBlock; private boolean chainStartComplete; - public EthereumJDepositContract(Ethereum ethereum, long contractDeployBlock, - String contractDeployAddress, Schedulers schedulers) { - super(schedulers); + public EthereumJDepositContract( + Ethereum ethereum, + long contractDeployBlock, + String contractDeployAddress, + Schedulers schedulers, + Function hashFunction, + int merkleTreeDepth) { + super(schedulers, hashFunction, merkleTreeDepth); this.ethereum = ethereum; this.contractDeployAddress = Address.fromHexString(contractDeployAddress); contractDeployAddressHash = @@ -61,19 +72,20 @@ public EthereumJDepositContract(Ethereum ethereum, long contractDeployBlock, @Override protected void chainStartSubscribed() { - ethereum.addListener(new EthereumListenerAdapter() { - @Override - public void onSyncDone(SyncState state) { - if (state == SyncState.COMPLETE) { - onEthereumUpdated(); - } - } + ethereum.addListener( + new EthereumListenerAdapter() { + @Override + public void onSyncDone(SyncState state) { + if (state == SyncState.COMPLETE) { + onEthereumUpdated(); + } + } - @Override - public void onBlock(Block block, List receipts) { - onEthereumUpdated(); - } - }); + @Override + public void onBlock(Block block, List receipts) { + onEthereumUpdated(); + } + }); if (ethereum.getSyncStatus().getStage() == SyncStage.Complete) { processConfirmedBlocks(); @@ -86,20 +98,21 @@ protected void chainStartDone() { } private void onEthereumUpdated() { - if (!chainStartComplete) { - processConfirmedBlocks(); - } + processConfirmedBlocks(); + } + + private long getBestConfirmedBlock() { + return ethereum.getBlockchain().getBestBlock().getNumber() - getDistanceFromHead(); } private void processConfirmedBlocks() { - long bestConfirmedBlock = - ethereum.getBlockchain().getBestBlock().getNumber() - getDistanceFromHead(); + long bestConfirmedBlock = getBestConfirmedBlock(); blockExecutor.newEvent(bestConfirmedBlock); } private void processBlocksUpTo(long bestConfirmedBlock) { try { - for (long number = processedUpToBlock; number < bestConfirmedBlock; number++) { + for (long number = processedUpToBlock; number <= bestConfirmedBlock; number++) { Block block = ethereum.getBlockchain().getBlockByNumber(number); onConfirmedBlock(block); processedUpToBlock = number + 1; @@ -116,30 +129,41 @@ private List getBlockTransactionReceipts(Block block) { .collect(Collectors.toList()); } - private void onConfirmedBlock(Block block) { + private synchronized void onConfirmedBlock(Block block) { + List depositEventDataList = new ArrayList<>(); for (Invocation invocation : getContractEvents(block)) { if (DEPOSIT_EVENT_NAME.equals(invocation.function.name)) { - newDeposit(createDepositEventData(invocation), block.getHash()); + depositEventDataList.add(createDepositEventData(invocation)); } else if (CHAIN_START_EVENT_NAME.equals(invocation.function.name)) { - chainStart((byte[]) invocation.args[0], (byte[]) invocation.args[1], block.getHash()); + if (!depositEventDataList.isEmpty()) { + newDeposits(depositEventDataList, block.getHash()); + } + depositEventDataList.clear(); + chainStart( + (byte[]) invocation.args[0], + (byte[]) invocation.args[1], + (byte[]) invocation.args[2], + block.getHash()); } else { throw new IllegalStateException("Invalid event from the contract: " + invocation); } } + if (!depositEventDataList.isEmpty()) { + newDeposits(depositEventDataList, block.getHash()); + } } + /** + * @param depositEvent Deposit: event({ pubkey: bytes[48], withdrawal_credentials: bytes[32], + * amount: bytes[8], signature: bytes[96], merkle_tree_index: bytes[8], }) + */ private DepositEventData createDepositEventData(Invocation depositEvent) { - Object[] merkle_branch_obj = (Object[]) depositEvent.args[3]; - byte[][] merkle_branch_arr = new byte[merkle_branch_obj.length][]; - for (int i = 0; i < merkle_branch_obj.length; i++) { - merkle_branch_arr[i] = (byte[]) merkle_branch_obj[i]; - } - return new DepositEventData( (byte[]) depositEvent.args[0], (byte[]) depositEvent.args[1], (byte[]) depositEvent.args[2], - merkle_branch_arr); + (byte[]) depositEvent.args[3], + (byte[]) depositEvent.args[4]); } private List getContractEvents(Block block) { @@ -166,7 +190,7 @@ protected boolean hasDepositRootImpl(byte[] blockHash, byte[] depositRoot) { if (block == null) { return false; } - if (ethereum.getBlockchain().getBestBlock().getNumber() - block.getNumber() < getDistanceFromHead()) { + if (block.getNumber() > getBestConfirmedBlock()) { return false; } @@ -176,17 +200,28 @@ protected boolean hasDepositRootImpl(byte[] blockHash, byte[] depositRoot) { } @Override - protected Optional> getLatestBlockHashDepositRoot() { - long bestBlock = ethereum.getBlockchain().getBestBlock().getNumber() - getDistanceFromHead(); - for(long blockNum = bestBlock; blockNum >= contractDeployBlock; blockNum--) { + protected synchronized Optional> + getLatestBlockHashDepositRoot() { + long bestBlock = getBestConfirmedBlock(); + for (long blockNum = bestBlock; blockNum >= contractDeployBlock; blockNum--) { Block block = ethereum.getBlockchain().getBlockByNumber(blockNum); List contractEvents = getContractEvents(block); Collections.reverse(contractEvents); for (Invocation contractEvent : contractEvents) { if (CHAIN_START_EVENT_NAME.equals(contractEvent.function.name)) { - return Optional.of(Pair.with(block.getHash(), (byte[]) contractEvent.args[0])); + return Optional.of( + Triplet.with( + (byte[]) contractEvent.args[0], + UInt64.fromBytesLittleEndian(Bytes8.wrap((byte[]) contractEvent.args[1])) + .intValue(), + block.getHash())); } else { - return Optional.of(Pair.with(block.getHash(), (byte[]) contractEvent.args[0])); + byte[] merkleTreeIndex = (byte[]) contractEvent.args[4]; + return Optional.of( + Triplet.with( + getDepositRoot(merkleTreeIndex).extractArray(), + UInt64.fromBytesLittleEndian(Bytes8.wrap(merkleTreeIndex)).increment().intValue(), + block.getHash())); } } } @@ -194,16 +229,17 @@ protected Optional> getLatestBlockHashDepositRoot() { } @Override - protected List> peekDepositsImpl(int count, byte[] startBlockHash, - byte[] startDepositRoot, byte[] endBlockHash, byte[] endDepositRoot) { - List> ret = new ArrayList<>(); + protected List>> peekDepositsImpl( + int count, byte[] startBlockHash, byte[] endBlockHash) { + List>> ret = new ArrayList<>(); Block startBlock = ethereum.getBlockchain().getBlockByHash(startBlockHash); Block endBlock = ethereum.getBlockchain().getBlockByHash(endBlockHash); - Iterator> iterator = iterateDepositEvents(startBlock, endBlock); + Iterator>> iterator = + iterateDepositEvents(startBlock, endBlock); boolean started = false; while (iterator.hasNext()) { - if (Arrays.equals(startDepositRoot, iterator.next().getValue1().deposit_root)) { + if (Arrays.equals(startBlockHash, iterator.next().getValue0().getHash())) { started = true; break; } @@ -214,10 +250,10 @@ protected List> peekDepositsImpl(int count, byte[ } while (iterator.hasNext() && count > 0) { - Pair event = iterator.next(); + Pair> event = iterator.next(); ret.add(Pair.with(event.getValue0().getHash(), event.getValue1())); count--; - if (Arrays.equals(endDepositRoot, event.getValue1().deposit_root)){ + if (Arrays.equals(endBlockHash, event.getValue0().getHash())) { break; } } @@ -225,10 +261,10 @@ protected List> peekDepositsImpl(int count, byte[ return ret; } - private Iterator> iterateDepositEvents(Block fromInclusive, - Block tillInclusive) { - return new Iterator>() { - Iterator iterator = Collections.emptyIterator(); + private Iterator>> iterateDepositEvents( + Block fromInclusive, Block tillInclusive) { + return new Iterator>>() { + Iterator> iterator = Collections.emptyIterator(); Block curBlock; @Override @@ -240,24 +276,31 @@ public boolean hasNext() { if (curBlock.getNumber() >= tillInclusive.getNumber()) { return false; } - if (ethereum.getBlockchain().getBestBlock().getNumber() - getDistanceFromHead() - <= curBlock.getNumber()) { + if (getBestConfirmedBlock() <= curBlock.getNumber()) { return false; } curBlock = ethereum.getBlockchain().getBlockByNumber(curBlock.getNumber() + 1); } - iterator = - getContractEvents(curBlock) - .stream() + List cur = + getContractEvents(curBlock).stream() .filter(invocation -> DEPOSIT_EVENT_NAME.equals(invocation.function.name)) - .iterator(); + .collect(Collectors.toList()); + if (!cur.isEmpty()) { + List> iteratorList = new ArrayList<>(); + iteratorList.add(cur); + iterator = iteratorList.iterator(); + } } return true; } @Override - public Pair next() { - return Pair.with(curBlock, createDepositEventData(iterator.next())); + public Pair> next() { + return Pair.with( + curBlock, + iterator.next().stream() + .map(i -> createDepositEventData(i)) + .collect(Collectors.toList())); } }; } diff --git a/pow/ethereumj/src/test/java/org/ethereum/beacon/pow/StandaloneDepositContractTest.java b/pow/ethereumj/src/test/java/org/ethereum/beacon/pow/StandaloneDepositContractTest.java index 2c80b91dd..71c7ff56b 100644 --- a/pow/ethereumj/src/test/java/org/ethereum/beacon/pow/StandaloneDepositContractTest.java +++ b/pow/ethereumj/src/test/java/org/ethereum/beacon/pow/StandaloneDepositContractTest.java @@ -1,13 +1,23 @@ package org.ethereum.beacon.pow; -import java.math.BigInteger; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; import org.apache.commons.codec.binary.Hex; +import org.ethereum.beacon.consensus.BeaconChainSpec; +import org.ethereum.beacon.consensus.hasher.ObjectHasher; +import org.ethereum.beacon.consensus.util.CachingBeaconChainSpec; +import org.ethereum.beacon.core.BeaconState; +import org.ethereum.beacon.core.MutableBeaconState; +import org.ethereum.beacon.core.operations.Deposit; +import org.ethereum.beacon.core.operations.deposit.DepositData; +import org.ethereum.beacon.core.spec.SignatureDomains; +import org.ethereum.beacon.core.spec.SpecConstants; import org.ethereum.beacon.core.state.Eth1Data; +import org.ethereum.beacon.core.types.BLSPubkey; +import org.ethereum.beacon.core.types.BLSSignature; +import org.ethereum.beacon.core.types.Gwei; +import org.ethereum.beacon.crypto.BLS381; +import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.crypto.MessageParameters; +import org.ethereum.beacon.crypto.util.BlsKeyPairGenerator; import org.ethereum.beacon.pow.DepositContract.ChainStart; import org.ethereum.beacon.pow.DepositContract.DepositInfo; import org.ethereum.beacon.schedulers.Schedulers; @@ -26,11 +36,20 @@ import reactor.core.publisher.Mono; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.Bytes48; import tech.pegasys.artemis.util.bytes.Bytes96; import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.bytes.MutableBytes48; import tech.pegasys.artemis.util.uint.UInt64; +import java.math.BigInteger; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; + @Ignore public class StandaloneDepositContractTest { @@ -38,19 +57,20 @@ public class StandaloneDepositContractTest { // CHAIN_START_FULL_DEPOSIT_THRESHOLD: constant(uint256) = 16 # 2**14 // SECONDS_PER_DAY: constant(uint256) = 5 - String depositBin = "600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600060c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600060c052602060c020015460208261016001015260208101905080610160526101609050805160208201209050606051600161014051018060405190131561013157600080fd5b809190121561013f57600080fd5b6020811061014c57600080fd5b600060c052602060c0200155606051600161014051018060405190131561017257600080fd5b809190121561018057600080fd5b6020811061018d57600080fd5b600060c052602060c020015460605160016101405101806040519013156101b357600080fd5b80919012156101c157600080fd5b602081106101ce57600080fd5b600160c052602060c02001555b81516001018083528114156100aa575b5050610f2f56600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a05263c5f2892f60005114156101cc5734156100ac57600080fd5b6000610140526002546101605261018060006020818352015b600160026100d257600080fd5b60026101605106141561013c57600061018051602081106100f257600080fd5b600160c052602060c0200154602082610220010152602081019050610140516020826102200101526020810190508061022052610220905080516020820120905061014052610195565b6000610140516020826101a0010152602081019050610180516020811061016257600080fd5b600060c052602060c02001546020826101a0010152602081019050806101a0526101a09050805160208201209050610140525b61016060026101a357600080fd5b60028151048152505b81516001018083528114156100c5575b50506101405160005260206000f3005b6398b1e06a6000511415610d375760206004610140376108206004356004016101603761080060043560040135111561020457600080fd5b670de0b6b3a764000034101561021957600080fd5b6801bc16d674ec80000034111561022f57600080fd5b6002546109a0526018600860208206610ac0016000633b9aca0061025257600080fd5b633b9aca003404602082610a6001015260208101905080610a6052610a60905051828401111561028157600080fd5b602080610ae0826020602088068803016000633b9aca006102a157600080fd5b633b9aca003404602082610a6001015260208101905080610a6052610a60905001600060046015f15050818152809050905090508051602001806109c0828460006004600a8704601201f16102f557600080fd5b50506018600860208206610c4001600042602082610be001015260208101905080610be052610be0905051828401111561032e57600080fd5b602080610c6082602060208806880301600042602082610be001015260208101905080610be052610be0905001600060046015f1505081815280905090509050805160200180610b40828460006004600a8704601201f161038e57600080fd5b505060006109c060088060208461152001018260208501600060046012f1505080518201915050610b4060088060208461152001018260208501600060046012f150508051820191505061016061080080602084611520010182602085016000600460def150508051820191505080611520526115209050805160200180610cc0828460006004600a8704601201f161042657600080fd5b50506018600860208206611e800160006109a051602082611e2001015260208101905080611e2052611e20905051828401111561046257600080fd5b602080611ea08260206020880688030160006109a051602082611e2001015260208101905080611e2052611e20905001600060046015f1505081815280905090509050805160200180611d80828460006004600a8704601201f16104c557600080fd5b50506000611f00526002611f2052611f4060006020818352015b6000611f20516104ee57600080fd5b611f20516109a05160016109a05101101561050857600080fd5b60016109a051010614151561051c57610588565b611f0060605160018251018060405190131561053757600080fd5b809190121561054557600080fd5b815250611f208051151561055a576000610574565b600281516002835102041461056e57600080fd5b60028151025b8152505b81516001018083528114156104df575b5050610cc0805160208201209050611f6052611f8060006020818352015b611f0051611f8051121561060d576000611f8051602081106105c757600080fd5b600160c052602060c0200154602082611fa0010152602081019050611f6051602082611fa001015260208101905080611fa052611fa09050805160208201209050611f60525b5b81516001018083528114156105a6575b5050611f6051611f00516020811061063557600080fd5b600160c052602060c0200155600280546001825401101561065557600080fd5b60018154018155506020612080600463c5f2892f6120205261203c6000305af161067e57600080fd5b612080516120a0526120a05161212052600160c052602060c02054612180526001600160c052602060c02001546121a0526002600160c052602060c02001546121c0526003600160c052602060c02001546121e0526004600160c052602060c0200154612200526005600160c052602060c0200154612220526006600160c052602060c0200154612240526007600160c052602060c0200154612260526008600160c052602060c0200154612280526009600160c052602060c02001546122a052600a600160c052602060c02001546122c052600b600160c052602060c02001546122e052600c600160c052602060c020015461230052600d600160c052602060c020015461232052600e600160c052602060c020015461234052600f600160c052602060c0200154612360526010600160c052602060c0200154612380526011600160c052602060c02001546123a0526012600160c052602060c02001546123c0526013600160c052602060c02001546123e0526014600160c052602060c0200154612400526015600160c052602060c0200154612420526016600160c052602060c0200154612440526017600160c052602060c0200154612460526018600160c052602060c0200154612480526019600160c052602060c02001546124a052601a600160c052602060c02001546124c052601b600160c052602060c02001546124e052601c600160c052602060c020015461250052601d600160c052602060c020015461252052601e600160c052602060c020015461254052601f600160c052602060c0200154612560526104606120e0526120e05161214052610cc08051602001806120e05161212001828460006004600a8704601201f161090257600080fd5b50506120e051612120015160206001820306601f82010390506120e051612120016120c08151610820818352015b836120c0511015156109415761095e565b60006120c0516020850101535b8151600101808352811415610930575b5050505060206120e051612120015160206001820306601f82010390506120e05101016120e0526120e05161216052611d808051602001806120e05161212001828460006004600a8704601201f16109b557600080fd5b50506120e051612120015160206001820306601f82010390506120e051612120016120c081516020818352015b836120c0511015156109f357610a10565b60006120c0516020850101535b81516001018083528114156109e2575b5050505060206120e051612120015160206001820306601f82010390506120e05101016120e0527fce7a77a358682d6c81f71216fb7fb108b03bc8badbf67f5d131ba5363cbefb426120e051612120a16801bc16d674ec800000341415610d35576003805460018254011015610a8557600080fd5b600181540181555060106003541415610d3457426125a052426125c0526005610aad57600080fd5b60056125c051066125a0511015610ac357600080fd5b426125c0526005610ad357600080fd5b60056125c051066125a051036005426125a052426125c0526005610af657600080fd5b60056125c051066125a0511015610b0c57600080fd5b426125c0526005610b1c57600080fd5b60056125c051066125a05103011015610b3457600080fd5b6005426125a052426125c0526005610b4b57600080fd5b60056125c051066125a0511015610b6157600080fd5b426125c0526005610b7157600080fd5b60056125c051066125a05103016125805260186008602082066126e00160006125805160208261268001015260208101905080612680526126809050518284011115610bbc57600080fd5b602080612700826020602088068803016000612580516020826126800101526020810190508061268052612680905001600060046015f15050818152809050905090508051602001806125e0828460006004600a8704601201f1610c1f57600080fd5b505060206127c0600463c5f2892f6127605261277c6000305af1610c4257600080fd5b6127c0516127e0526127e0516128605260406128205261282051612880526125e08051602001806128205161286001828460006004600a8704601201f1610c8857600080fd5b505061282051612860015160206001820306601f8201039050612820516128600161280081516020818352015b8361280051101515610cc657610ce3565b6000612800516020850101535b8151600101808352811415610cb5575b50505050602061282051612860015160206001820306601f8201039050612820510101612820527fd1faa3f9bca1d698df559716fe6d1c9999155b38d3158fffbc98d76d568091fc61282051612860a15b5b005b60006000fd5b6101f2610f2f036101f26000396101f2610f2f036000f3"; - String abiTestBin = "0x6101db56600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a05263d45754f860005114156101d157602060046101403760606004356004016101603760406004356004013511156100c957600080fd5b7f11111111111111111111111111111111111111111111111111111111111111116102405260406102005261020051610260526101608051602001806102005161024001828460006004600a8704601201f161012457600080fd5b505061020051610240015160206001820306601f820103905061020051610240016101e081516040818352015b836101e0511015156101625761017f565b60006101e0516020850101535b8151600101808352811415610151575b50505050602061020051610240015160206001820306601f8201039050610200510101610200527f68ab17451419beb01e059af9ee2a11c36d17b75ed25144e5cf78a0a469883ed161020051610240a1005b60006000fd5b6100046101db036100046000396100046101db036000f3"; - + final int MERKLE_TREE_DEPTH = SpecConstants.DEPOSIT_CONTRACT_TREE_DEPTH.intValue(); + final Function HASH_FUNCTION = Hashes::sha256; + String depositBin = + "600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600060c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600060c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600060c052602060c0200155606051600161014051018060405190131561018757600080fd5b809190121561019557600080fd5b602081106101a257600080fd5b600060c052602060c020015460605160016101405101806040519013156101c857600080fd5b80919012156101d657600080fd5b602081106101e357600080fd5b600160c052602060c02001555b81516001018083528114156100aa575b50506115ea56600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a0526380673289600051141561026b57602060046101403734156100b457600080fd5b67ffffffffffffffff6101405111156100cc57600080fd5b60006101605261014051610180526101a060006008818352015b6101605160086000811215610103578060000360020a820461010a565b8060020a82025b905090506101605260ff61018051166101c052610160516101c0516101605101101561013557600080fd5b6101c051610160510161016052610180517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8600081121561017e578060000360020a8204610185565b8060020a82025b90509050610180525b81516001018083528114156100e6575b505060186008602082066101e001602082840111156101bc57600080fd5b60208061020082610160600060046015f15050818152809050905090508051602001806102a0828460006004600a8704601201f16101f957600080fd5b50506102a05160206001820306601f82010390506103006102a0516008818352015b8261030051111561022b57610247565b6000610300516102c001535b815160010180835281141561021b575b50505060206102805260406102a0510160206001820306601f8201039050610280f3005b639d70e8066000511415610405576020600461014037341561028c57600080fd5b60286004356004016101603760086004356004013511156102ac57600080fd5b60006101c0526101608060200151600082518060209013156102cd57600080fd5b80919012156102db57600080fd5b806020036101000a82049050905090506101e05261020060006008818352015b60ff6101e05116606051606051610200516007038060405190131561031f57600080fd5b809190121561032d57600080fd5b6008028060405190131561034057600080fd5b809190121561034e57600080fd5b6000811215610365578060000360020a820461036c565b8060020a82025b90509050610220526101c051610220516101c05101101561038c57600080fd5b610220516101c051016101c0526101e0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff860008112156103d5578060000360020a82046103dc565b8060020a82025b905090506101e0525b81516001018083528114156102fb575b50506101c05160005260206000f3005b63c5f2892f600051141561055d57341561041e57600080fd5b6000610140526002546101605261018060006020818352015b60016001610160511614156104b8576000610180516020811061045957600080fd5b600160c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af16104aa57600080fd5b60c051905061014052610526565b6000610140516020826101a001015260208101905061018051602081106104de57600080fd5b600060c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161051c57600080fd5b60c0519050610140525b610160600261053457600080fd5b60028151048152505b8151600101808352811415610437575b50506101405160005260206000f3005b63621fd130600051141561063357341561057657600080fd5b60606101c060246380673289610140526002546101605261015c6000305af161059e57600080fd5b6101e0805160200180610260828460006004600a8704601201f16105c157600080fd5b50506102605160206001820306601f82010390506102c0610260516008818352015b826102c05111156105f35761060f565b60006102c05161028001535b81516001018083528114156105e3575b5050506020610240526040610260510160206001820306601f8201039050610240f3005b63c47e300d60005114156113b757606060046101403760506004356004016101a037603060043560040135111561066957600080fd5b604060243560040161022037602060243560040135111561068957600080fd5b60806044356004016102803760606044356004013511156106a957600080fd5b633b9aca0061034052610340516106bf57600080fd5b61034051340461032052633b9aca006103205110156106dd57600080fd5b6060610440602463806732896103c052610320516103e0526103dc6000305af161070657600080fd5b610460805160200180610360828460006004600a8704601201f161072957600080fd5b50506002546104a05260006104c05260026104e05261050060006020818352015b60006104e05161075957600080fd5b6104e0516104a05160016104a05101101561077357600080fd5b60016104a0510106141515610787576107f3565b6104c06060516001825101806040519013156107a257600080fd5b80919012156107b057600080fd5b8152506104e0805115156107c55760006107df565b60028151600283510204146107d957600080fd5b60028151025b8152505b815160010180835281141561074a575b505060006101a06030806020846105e001018260208501600060046016f15050805182019150506000601060208206610560016020828401111561083657600080fd5b60208061058082610520600060046015f15050818152809050905090506010806020846105e001018260208501600060046013f1505080518201915050806105e0526105e09050602060c0825160208401600060025af161089657600080fd5b60c05190506105405260006000604060208206610680016102805182840111156108bf57600080fd5b6060806106a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af16108ff57600080fd5b60c0519050602082610880010152602081019050600060406020602082066107400161028051828401111561093357600080fd5b606080610760826020602088068803016102800160006004601bf150508181528090509050905060208060208461080001018260208501600060046015f15050805182019150506105205160208261080001015260208101905080610800526108009050602060c0825160208401600060025af16109b057600080fd5b60c051905060208261088001015260208101905080610880526108809050602060c0825160208401600060025af16109e757600080fd5b60c051905061066052600060006105405160208261092001015260208101905061022060208060208461092001018260208501600060046015f150508051820191505080610920526109209050602060c0825160208401600060025af1610a4d57600080fd5b60c0519050602082610aa00101526020810190506000610360600880602084610a2001018260208501600060046012f150508051820191505060006018602082066109a00160208284011115610aa257600080fd5b6020806109c082610520600060046015f1505081815280905090509050601880602084610a2001018260208501600060046014f150508051820191505061066051602082610a2001015260208101905080610a2052610a209050602060c0825160208401600060025af1610b1557600080fd5b60c0519050602082610aa001015260208101905080610aa052610aa09050602060c0825160208401600060025af1610b4c57600080fd5b60c051905061090052610b2060006020818352015b6104c051610b20511215610be1576000610b205160208110610b8257600080fd5b600160c052602060c0200154602082610b4001015260208101905061090051602082610b4001015260208101905080610b4052610b409050602060c0825160208401600060025af1610bd357600080fd5b60c051905061090052610be6565b610bf7565b5b8151600101808352811415610b61575b5050610900516104c05160208110610c0e57600080fd5b600160c052602060c02001556002805460018254011015610c2e57600080fd5b60018154018155506020610c40600463c5f2892f610be052610bfc6000305af1610c5757600080fd5b610c4051610bc0526060610ce060246380673289610c60526104a051610c8052610c7c6000305af1610c8857600080fd5b610d00805160200180610d40828460006004600a8704601201f1610cab57600080fd5b505060a0610dc052610dc051610e00526101a0805160200180610dc051610e0001828460006004600a8704601201f1610ce357600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516040818352015b83610da051101515610d2157610d3e565b6000610da0516020850101535b8151600101808352811415610d10575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc052610dc051610e2052610220805160200180610dc051610e0001828460006004600a8704601201f1610d9557600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516020818352015b83610da051101515610dd357610df0565b6000610da0516020850101535b8151600101808352811415610dc2575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc052610dc051610e4052610360805160200180610dc051610e0001828460006004600a8704601201f1610e4757600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516020818352015b83610da051101515610e8557610ea2565b6000610da0516020850101535b8151600101808352811415610e74575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc052610dc051610e6052610280805160200180610dc051610e0001828460006004600a8704601201f1610ef957600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516060818352015b83610da051101515610f3757610f54565b6000610da0516020850101535b8151600101808352811415610f26575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc052610dc051610e8052610d40805160200180610dc051610e0001828460006004600a8704601201f1610fab57600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516020818352015b83610da051101515610fe957611006565b6000610da0516020850101535b8151600101808352811415610fd8575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc0527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce610dc051610e00a1640773594000610320511015156113b557600380546001825401101561107b57600080fd5b6001815401815550601060035414156113b45742610ec05242610ee05260056110a357600080fd5b6005610ee05106610ec05110156110b957600080fd5b42610ee05260056110c957600080fd5b6005610ee05106610ec05103600a42610ec05242610ee05260056110ec57600080fd5b6005610ee05106610ec051101561110257600080fd5b42610ee052600561111257600080fd5b6005610ee05106610ec0510301101561112a57600080fd5b600a42610ec05242610ee052600561114157600080fd5b6005610ee05106610ec051101561115757600080fd5b42610ee052600561116757600080fd5b6005610ee05106610ec0510301610ea0526060610f8060246380673289610f0052600254610f2052610f1c6000305af16111a057600080fd5b610fa0805160200180610fe0828460006004600a8704601201f16111c357600080fd5b505060606110c06024638067328961104052610ea0516110605261105c6000305af16111ee57600080fd5b6110e0805160200180611120828460006004600a8704601201f161121157600080fd5b5050610bc0516111e05260606111a0526111a05161120052610fe08051602001806111a0516111e001828460006004600a8704601201f161125157600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b836111805110151561128f576112ac565b6000611180516020850101535b815160010180835281141561127e575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0526111a051611220526111208051602001806111a0516111e001828460006004600a8704601201f161130357600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b83611180511015156113415761135e565b6000611180516020850101535b8151600101808352811415611330575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0527f08b71ef3f1b58f7a23ffb82e27f12f0888c8403f1ceb0ea7ea26b274e2189d4c6111a0516111e0a160016004555b5b005b63845980e860005114156113dd5734156113d057600080fd5b60045460005260206000f3005b60006000fd5b6102076115ea036102076000396102076115ea036000f3"; + String abiTestBin = + "0x6101db56600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a05263d45754f860005114156101d157602060046101403760606004356004016101603760406004356004013511156100c957600080fd5b7f11111111111111111111111111111111111111111111111111111111111111116102405260406102005261020051610260526101608051602001806102005161024001828460006004600a8704601201f161012457600080fd5b505061020051610240015160206001820306601f820103905061020051610240016101e081516040818352015b836101e0511015156101625761017f565b60006101e0516020850101535b8151600101808352811415610151575b50505050602061020051610240015160206001820306601f8201039050610200510101610200527f68ab17451419beb01e059af9ee2a11c36d17b75ed25144e5cf78a0a469883ed161020051610240a1005b60006000fd5b6100046101db036100046000396100046101db036000f3"; String abiTestAbi = "[{\"name\": \"Deposit\", \"inputs\": [{\"type\": \"bytes32\", \"name\": \"deposit_root\", \"indexed\": false}, {\"type\": \"bytes\", \"name\": \"data\", \"indexed\": false}, {\"type\": \"bytes\", \"name\": \"merkle_tree_index\", \"indexed\": false}, {\"type\": \"bytes32[32]\", \"name\": \"branch\", \"indexed\": false}], \"anonymous\": false, \"type\": \"event\"}, {\"name\": \"ChainStart\", \"inputs\": [{\"type\": \"bytes32\", \"name\": \"deposit_root\", \"indexed\": false}, {\"type\": \"bytes\", \"name\": \"time\", \"indexed\": false}], \"anonymous\": false, \"type\": \"event\"}, {\"name\": \"Test\", \"inputs\": [{\"type\": \"bytes32\", \"name\": \"a\", \"indexed\": false}, {\"type\": \"bytes\", \"name\": \"data\", \"indexed\": false}], \"anonymous\": false, \"type\": \"event\"}, {\"name\": \"__init__\", \"outputs\": [], \"inputs\": [], \"constant\": false, \"payable\": false, \"type\": \"constructor\"}, {\"name\": \"get_deposit_root\", \"outputs\": [{\"type\": \"bytes32\", \"name\": \"out\"}], \"inputs\": [], \"constant\": true, \"payable\": false, \"type\": \"function\", \"gas\": 30775}, {\"name\": \"f\", \"outputs\": [], \"inputs\": [{\"type\": \"bytes\", \"name\": \"a\"}], \"constant\": false, \"payable\": true, \"type\": \"function\", \"gas\": 49719}, {\"name\": \"deposit\", \"outputs\": [], \"inputs\": [{\"type\": \"bytes\", \"name\": \"deposit_input\"}], \"constant\": false, \"payable\": true, \"type\": \"function\", \"gas\": 637708}]\n"; - - BigInteger depositAmount = - BigInteger.valueOf(32L * 1_000_000_000L).multiply(BigInteger.valueOf(1_000_000_000L)); + BigInteger gweiAmount = BigInteger.valueOf(32L * 1_000_000_000L); + BigInteger depositAmount = gweiAmount.multiply(BigInteger.valueOf(1_000_000_000L)); @Test public void test1() { - StandaloneBlockchain sb = new StandaloneBlockchain() - .withAutoblock(true); + StandaloneBlockchain sb = new StandaloneBlockchain().withAutoblock(true); ContractMetadata contractMetadata = new ContractMetadata(); contractMetadata.abi = ContractAbi.getContractAbi(); contractMetadata.bin = depositBin; @@ -60,16 +80,17 @@ public void test1() { SSZSerializer sszSerializer = new SSZBuilder().buildSerializer(); - for(int i = 0; i < 20; i++) { + for (int i = 0; i < 20; i++) { MutableBytes48 pubKey = MutableBytes48.create(); pubKey.set(0, (byte) i); - SolidityCallResult result = contract.callFunction( - depositAmount, - "deposit", - pubKey.extractArray(), - Hash32.ZERO.extractArray(), - Bytes96.ZERO.extractArray()); + SolidityCallResult result = + contract.callFunction( + depositAmount, + "deposit", + pubKey.extractArray(), + Hash32.ZERO.extractArray(), + Bytes96.ZERO.extractArray()); Assert.assertTrue(result.isSuccessful()); Assert.assertEquals(i == 15 ? 2 : 1, result.getEvents().size()); @@ -85,49 +106,51 @@ public void test1() { ethereum, 0, BytesValue.wrap(contract.getAddress()).toString(), - Schedulers.createDefault()); + Schedulers.createDefault(), + HASH_FUNCTION, + MERKLE_TREE_DEPTH); depositContract.setDistanceFromHead(3); - ChainStart chainStart = Mono.from(depositContract.getChainStartMono()) - .block(Duration.ofSeconds(2)); + ChainStart chainStart = + Mono.from(depositContract.getChainStartMono()).block(Duration.ofSeconds(2)); Assert.assertEquals(16, chainStart.getInitialDeposits().size()); - Assert.assertEquals(17, sb.getBlockchain() - .getBlockByHash(chainStart.getEth1Data().getBlockHash().extractArray()).getNumber()); + Assert.assertEquals( + 17, + sb.getBlockchain() + .getBlockByHash(chainStart.getEth1Data().getBlockHash().extractArray()) + .getNumber()); for (int i = 0; i < 16; i++) { Assert.assertEquals(UInt64.valueOf(i), chainStart.getInitialDeposits().get(i).getIndex()); - Assert.assertEquals((byte) i, chainStart.getInitialDeposits().get(i).getData() - .getPubKey().get(0)); + Assert.assertEquals( + (byte) i, chainStart.getInitialDeposits().get(i).getData().getPubKey().get(0)); } depositRoot = contract.callConstFunction("get_deposit_root"); System.out.println(Hex.encodeHexString((byte[]) depositRoot[0])); - Eth1Data lastDepositEthData = new Eth1Data( - Hash32.wrap(Bytes32.wrap((byte[]) depositRoot[0])), - UInt64.ZERO, - Hash32.wrap(Bytes32.wrap(sb.getBlockchain().getBlockByNumber(21).getHash()))); + Eth1Data lastDepositEthData = + new Eth1Data( + Hash32.wrap(Bytes32.wrap((byte[]) depositRoot[0])), + UInt64.ZERO, + Hash32.wrap(Bytes32.wrap(sb.getBlockchain().getBlockByNumber(21).getHash()))); - List depositInfos1 = depositContract.peekDeposits(2, - chainStart.getEth1Data(), lastDepositEthData); + List depositInfos1 = + depositContract.peekDeposits(2, chainStart.getEth1Data(), lastDepositEthData); Assert.assertEquals(2, depositInfos1.size()); - Assert.assertEquals((byte) 16, - depositInfos1.get(0).getDeposit().getData().getPubKey().get(0)); - Assert.assertEquals((byte) 17, - depositInfos1.get(1).getDeposit().getData().getPubKey().get(0)); + Assert.assertEquals((byte) 16, depositInfos1.get(0).getDeposit().getData().getPubKey().get(0)); + Assert.assertEquals((byte) 17, depositInfos1.get(1).getDeposit().getData().getPubKey().get(0)); - List depositInfos2 = depositContract.peekDeposits(200, - depositInfos1.get(1).getEth1Data(), lastDepositEthData); + List depositInfos2 = + depositContract.peekDeposits(200, depositInfos1.get(1).getEth1Data(), lastDepositEthData); Assert.assertEquals(2, depositInfos2.size()); - Assert.assertEquals((byte) 18, - depositInfos2.get(0).getDeposit().getData().getPubKey().get(0)); - Assert.assertEquals((byte) 19, - depositInfos2.get(1).getDeposit().getData().getPubKey().get(0)); + Assert.assertEquals((byte) 18, depositInfos2.get(0).getDeposit().getData().getPubKey().get(0)); + Assert.assertEquals((byte) 19, depositInfos2.get(1).getDeposit().getData().getPubKey().get(0)); - List depositInfos3 = depositContract.peekDeposits(200, - lastDepositEthData, lastDepositEthData); + List depositInfos3 = + depositContract.peekDeposits(200, lastDepositEthData, lastDepositEthData); Assert.assertEquals(0, depositInfos3.size()); } @@ -146,27 +169,30 @@ public void testOnline() { ethereum, 0, BytesValue.wrap(contract.getAddress()).toString(), - Schedulers.createDefault()); + Schedulers.createDefault(), + HASH_FUNCTION, + MERKLE_TREE_DEPTH); depositContract.setDistanceFromHead(3); Mono chainStartMono = Mono.from(depositContract.getChainStartMono()); chainStartMono.subscribe(); SSZSerializer sszSerializer = new SSZBuilder().buildSerializer(); - for(int i = 0; i < 16; i++) { + for (int i = 0; i < 16; i++) { sb.createBlock(); } - for(int i = 0; i < 16; i++) { + for (int i = 0; i < 16; i++) { MutableBytes48 pubKey = MutableBytes48.create(); pubKey.set(0, (byte) i); - SolidityCallResult result = contract.callFunction( - depositAmount, - "deposit", - pubKey.extractArray(), - Hash32.ZERO.extractArray(), - Bytes96.ZERO.extractArray()); + SolidityCallResult result = + contract.callFunction( + depositAmount, + "deposit", + pubKey.extractArray(), + Hash32.ZERO.extractArray(), + Bytes96.ZERO.extractArray()); sb.createBlock(); sb.createBlock(); @@ -183,12 +209,15 @@ public void testOnline() { ChainStart chainStart = chainStartMono.block(Duration.ofSeconds(1)); Assert.assertEquals(16, chainStart.getInitialDeposits().size()); - Assert.assertEquals(1 + 16 + 31, sb.getBlockchain() - .getBlockByHash(chainStart.getEth1Data().getBlockHash().extractArray()).getNumber()); + Assert.assertEquals( + 1 + 16 + 31, + sb.getBlockchain() + .getBlockByHash(chainStart.getEth1Data().getBlockHash().extractArray()) + .getNumber()); for (int i = 0; i < 16; i++) { Assert.assertEquals(UInt64.valueOf(i), chainStart.getInitialDeposits().get(i).getIndex()); - Assert.assertEquals((byte) i, chainStart.getInitialDeposits().get(i).getData() - .getPubKey().get(0)); + Assert.assertEquals( + (byte) i, chainStart.getInitialDeposits().get(i).getData().getPubKey().get(0)); } Optional latestEth1Data1 = depositContract.getLatestEth1Data(); @@ -200,8 +229,12 @@ public void testOnline() { MutableBytes48 pubKey = MutableBytes48.create(); pubKey.set(0, (byte) (0x20 + i * 4 + j)); - contract.callFunction(depositAmount,"deposit", - pubKey.extractArray(), Hash32.ZERO.extractArray(), Bytes96.ZERO.extractArray()); + contract.callFunction( + depositAmount, + "deposit", + pubKey.extractArray(), + Hash32.ZERO.extractArray(), + Bytes96.ZERO.extractArray()); } sb.createBlock(); sb.createBlock(); @@ -209,8 +242,12 @@ public void testOnline() { Optional latestEth1Data2 = depositContract.getLatestEth1Data(); Assert.assertTrue(latestEth1Data2.isPresent()); - Assert.assertEquals(ethereum.getBlockchain().getBestBlock().getNumber() - 3, - ethereum.getBlockchain().getBlockByHash(latestEth1Data2.get().getBlockHash().extractArray()).getNumber()); + Assert.assertEquals( + ethereum.getBlockchain().getBestBlock().getNumber() - 3, + ethereum + .getBlockchain() + .getBlockByHash(latestEth1Data2.get().getBlockHash().extractArray()) + .getNumber()); sb.createBlock(); sb.createBlock(); @@ -231,27 +268,190 @@ public void testOnline() { } Assert.assertEquals(16, allDepos.size()); for (int i = 0; i < 16; i++) { - Assert.assertEquals(0x20 + i, allDepos.get(i).getDeposit().getData() - .getPubKey().get(0)); + Assert.assertEquals(0x20 + i, allDepos.get(i).getDeposit().getData().getPubKey().get(0)); + } + } + + @Test + public void testVerifyDepositRoot() { + StandaloneBlockchain sb = new StandaloneBlockchain(); + ContractMetadata contractMetadata = new ContractMetadata(); + contractMetadata.abi = ContractAbi.getContractAbi(); + contractMetadata.bin = depositBin; + SolidityContract contract = sb.submitNewContract(contractMetadata); + sb.createBlock(); + + Ethereum ethereum = new StandaloneEthereum(new SystemProperties(), sb); + byte[] latestCalculatedDepositRoot = new byte[32]; + EthereumJDepositContract depositContract = + new EthereumJDepositContract( + ethereum, + 0, + BytesValue.wrap(contract.getAddress()).toString(), + Schedulers.createDefault(), + HASH_FUNCTION, + MERKLE_TREE_DEPTH) { + + @Override + protected synchronized void newDeposits( + List eventDataList, byte[] blockHash) { + super.newDeposits(eventDataList, blockHash); + for (DepositEventData eventData : eventDataList) { + System.arraycopy( + getDepositRoot(eventData.merkleTreeIndex).extractArray(), + 0, + latestCalculatedDepositRoot, + 0, + 32); + } + } + }; + depositContract.setDistanceFromHead(3); + Mono chainStartMono = Mono.from(depositContract.getChainStartMono()); + chainStartMono.subscribe(); + + for (int i = 0; i < 20; i++) { + MutableBytes48 pubKey = MutableBytes48.create(); + pubKey.set(0, (byte) i); + + SolidityCallResult result = + contract.callFunction( + depositAmount, + "deposit", + pubKey.extractArray(), + Hash32.ZERO.extractArray(), + Bytes96.ZERO.extractArray()); + sb.createBlock(); + sb.createBlock(); + sb.createBlock(); + sb.createBlock(); + + Object[] depositRoot = contract.callConstFunction("get_deposit_root"); + Assert.assertArrayEquals((byte[]) depositRoot[0], latestCalculatedDepositRoot); + + Assert.assertTrue(result.isSuccessful()); + Assert.assertEquals(i == 15 ? 2 : 1, result.getEvents().size()); + } + } + + @Test + public void testVerifyProofs() { + StandaloneBlockchain sb = new StandaloneBlockchain().withAutoblock(true); + ContractMetadata contractMetadata = new ContractMetadata(); + contractMetadata.abi = ContractAbi.getContractAbi(); + contractMetadata.bin = depositBin; + SolidityContract contract = sb.submitNewContract(contractMetadata); + Object[] depositRoot = contract.callConstFunction("get_deposit_root"); + System.out.println(Hex.encodeHexString((byte[]) depositRoot[0])); + + BlsKeyPairGenerator generator = BlsKeyPairGenerator.createWithoutSeed(); + BeaconChainSpec spec = + new CachingBeaconChainSpec( + BeaconChainSpec.DEFAULT_CONSTANTS, + Hashes::sha256, + ObjectHasher.createSSZOverSHA256(BeaconChainSpec.DEFAULT_CONSTANTS), + true, + true); + List eth1DataList = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + BLS381.KeyPair keyPair = generator.next(); + Bytes48 pubKey = keyPair.getPublic().getEncodedBytes(); + Hash32 withdrawalCredentials = Hash32.ZERO; + DepositData depositData = + new DepositData( + BLSPubkey.wrap(pubKey), + withdrawalCredentials, + Gwei.castFrom(UInt64.valueOf(gweiAmount.longValue())), + BLSSignature.ZERO); + // Let signature be the result of bls_sign of the signing_root(deposit_data) with + // domain=DOMAIN_DEPOSIT. + MessageParameters messageParameters = + MessageParameters.create(spec.signing_root(depositData), SignatureDomains.DEPOSIT); + BLS381.Signature signature = BLS381.sign(messageParameters, keyPair); + + SolidityCallResult result = + contract.callFunction( + depositAmount, + "deposit", + pubKey.extractArray(), + withdrawalCredentials.extractArray(), + signature.getEncoded().extractArray()); + + Assert.assertTrue(result.isSuccessful()); + + depositRoot = contract.callConstFunction("get_deposit_root"); + Eth1Data lastDepositEthData = + new Eth1Data( + Hash32.wrap(Bytes32.wrap((byte[]) depositRoot[0])), UInt64.valueOf(i), Hash32.ZERO); + eth1DataList.add(lastDepositEthData); + + Assert.assertEquals(i == 15 ? 2 : 1, result.getEvents().size()); + } + + for (int i = 0; i < 16; i++) { + sb.createBlock(); + } + + Ethereum ethereum = new StandaloneEthereum(new SystemProperties(), sb); + EthereumJDepositContract depositContract = + new EthereumJDepositContract( + ethereum, + 0, + BytesValue.wrap(contract.getAddress()).toString(), + Schedulers.createDefault(), + HASH_FUNCTION, + MERKLE_TREE_DEPTH); + depositContract.setDistanceFromHead(3); + + ChainStart chainStart = + Mono.from(depositContract.getChainStartMono()).block(Duration.ofSeconds(2)); + + Assert.assertEquals(16, chainStart.getInitialDeposits().size()); + MutableBeaconState beaconState = BeaconState.getEmpty().createMutableCopy(); + for (Deposit deposit : chainStart.getInitialDeposits()) { + // The proof for each deposit must be constructed against the deposit root contained in + // state.latest_eth1_data rather than the deposit root at the time the deposit was initially + // logged from the 1.0 chain. This entails storing a full deposit merkle tree locally and + // computing updated proofs against the latest_eth1_data.deposit_root as needed. See + // minimal_merkle.py for a sample implementation. + beaconState.setLatestEth1Data(eth1DataList.get(deposit.getIndex().intValue())); + spec.verify_deposit(beaconState, deposit); + } + + depositRoot = contract.callConstFunction("get_deposit_root"); + System.out.println(Hex.encodeHexString((byte[]) depositRoot[0])); + + Eth1Data lastDepositEthData = + new Eth1Data( + Hash32.wrap(Bytes32.wrap((byte[]) depositRoot[0])), + UInt64.ZERO, + Hash32.wrap(Bytes32.wrap(sb.getBlockchain().getBlockByNumber(21).getHash()))); + + List depositInfos1 = + depositContract.peekDeposits(100, chainStart.getEth1Data(), lastDepositEthData); + + Assert.assertEquals(4, depositInfos1.size()); + for (DepositInfo depositInfo : depositInfos1) { + beaconState.setLatestEth1Data( + eth1DataList.get(depositInfo.getDeposit().getIndex().intValue())); + spec.verify_deposit(beaconState, depositInfo.getDeposit()); } } @Test public void testVyperAbi() { - StandaloneBlockchain sb = new StandaloneBlockchain() - .withAutoblock(true); + StandaloneBlockchain sb = new StandaloneBlockchain().withAutoblock(true); ContractMetadata contractMetadata = new ContractMetadata(); contractMetadata.abi = abiTestAbi.replaceAll(", *\"gas\": *[0-9]+", ""); contractMetadata.bin = abiTestBin.replace("0x", ""); SolidityContract contract = sb.submitNewContract(contractMetadata); - ((SolidityContractImpl) contract).addRelatedContract(ContractAbi.getContractAbi()); // TODO ethJ bug workaround + ((SolidityContractImpl) contract) + .addRelatedContract(ContractAbi.getContractAbi()); // TODO ethJ bug workaround - byte[] bytes = new byte[64]; - Arrays.fill(bytes, (byte) 0x33); + byte[] bytes = new byte[64]; + Arrays.fill(bytes, (byte) 0x33); - SolidityCallResult result = contract.callFunction( - "f", - (Object) bytes); + SolidityCallResult result = contract.callFunction("f", (Object) bytes); Object[] returnValues = result.getEvents().get(0).args; System.out.println(result); From 83029f9cb5c34a6a31df9fac97e7aa391ad39139 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Sat, 11 May 2019 15:33:13 +0300 Subject: [PATCH 05/14] add required dependencies for deposit contract test --- pow/ethereumj/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pow/ethereumj/build.gradle b/pow/ethereumj/build.gradle index c604a2c5e..de96557b4 100644 --- a/pow/ethereumj/build.gradle +++ b/pow/ethereumj/build.gradle @@ -21,4 +21,6 @@ dependencies { implementation 'io.projectreactor:reactor-core' testImplementation project(':ssz') + testImplementation project(':consensus') + testImplementation project(':crypto') } From f6f7429779a595f16806f41b649258fe0123553d Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 15 May 2019 11:22:10 +0300 Subject: [PATCH 06/14] pow: deposit contract clean up --- .../org/ethereum/beacon/pow/AbstractDepositContract.java | 3 --- .../main/java/org/ethereum/beacon/pow/MinimalMerkle.java | 7 +++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java b/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java index f83667de0..13e676929 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java @@ -8,8 +8,6 @@ import org.ethereum.beacon.core.types.Gwei; import org.ethereum.beacon.core.types.Time; import org.ethereum.beacon.schedulers.Schedulers; -import org.ethereum.beacon.ssz.SSZBuilder; -import org.ethereum.beacon.ssz.SSZSerializer; import org.ethereum.beacon.stream.SimpleProcessor; import org.javatuples.Pair; import org.javatuples.Triplet; @@ -34,7 +32,6 @@ public abstract class AbstractDepositContract implements DepositContract { protected final Schedulers schedulers; - private final SSZSerializer ssz = new SSZBuilder().buildSerializer(); private final MonoProcessor chainStartSink = MonoProcessor.create(); private final Publisher chainStartStream; private final SimpleProcessor depositStream; diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java index 0f8489475..f2cee6082 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java @@ -7,15 +7,14 @@ import java.util.ArrayList; import java.util.List; import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; import static tech.pegasys.artemis.util.bytes.BytesValue.concat; /** * Minimal merkle tree https://en.wikipedia.org/wiki/Merkle_tree - * implementation + * implementation, adoption from python code from https://github.com/ethereum/research/blob/master/spec_pythonizer/utils/merkle_minimal.py */ public class MinimalMerkle { @@ -29,7 +28,7 @@ public MinimalMerkle(Function hashFunction, int treeDepth) { this.zeroHashes = new Hash32[treeDepth]; } - public Hash32 getZeroHash(int distanceFromBottom) { + private Hash32 getZeroHash(int distanceFromBottom) { if (zeroHashes[distanceFromBottom] == null) { if (distanceFromBottom == 0) { zeroHashes[0] = Hash32.ZERO; From c0147b3f29749f714b16ef0c21150fdf737e9eaa Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 16 May 2019 15:56:21 +0300 Subject: [PATCH 07/14] pow: added incremental deposit contract (proofs are failed) --- .../beacon/pow/AbstractDepositContract.java | 95 +++-------- .../beacon/pow/DepositDataMerkle.java | 78 +++++++++ .../beacon/pow/DepositIncrementalMerkle.java | 158 ++++++++++++++++++ ...alMerkle.java => DepositSimpleMerkle.java} | 82 ++++++--- .../org/ethereum/beacon/pow/MerkleTree.java | 37 ++++ .../pow/util/DepositDataMerkleTest.java | 71 ++++++++ .../ethereum/beacon/util/ConsumerList.java | 125 ++++++++++++++ .../beacon/util/ConsumerListTest.java | 44 +++++ 8 files changed, 593 insertions(+), 97 deletions(-) create mode 100644 pow/core/src/main/java/org/ethereum/beacon/pow/DepositDataMerkle.java create mode 100644 pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java rename pow/core/src/main/java/org/ethereum/beacon/pow/{MinimalMerkle.java => DepositSimpleMerkle.java} (50%) create mode 100644 pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java create mode 100644 pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java create mode 100644 util/src/main/java/org/ethereum/beacon/util/ConsumerList.java create mode 100644 util/src/test/java/org/ethereum/beacon/util/ConsumerListTest.java diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java b/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java index 13e676929..0ef1571f5 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java @@ -35,12 +35,11 @@ public abstract class AbstractDepositContract implements DepositContract { private final MonoProcessor chainStartSink = MonoProcessor.create(); private final Publisher chainStartStream; private final SimpleProcessor depositStream; - private final MinimalMerkle minimalMerkle; - private final Function hashFunction; + private final MerkleTree tree; private long distanceFromHead; private List initialDeposits = new ArrayList<>(); private boolean startChainSubscribed; - private List deposits = new ArrayList<>(); + public AbstractDepositContract( Schedulers schedulers, Function hashFunction, int treeDepth) { this.schedulers = schedulers; @@ -51,14 +50,14 @@ public AbstractDepositContract( .doOnSubscribe(s -> chainStartSubscribedPriv()) .name("PowClient.chainStart"); depositStream = new SimpleProcessor<>(this.schedulers.reactorEvents(), "PowClient.deposit"); - this.hashFunction = hashFunction; - this.minimalMerkle = new MinimalMerkle(hashFunction, treeDepth); + this.tree = new DepositSimpleMerkle(hashFunction, treeDepth); } /** * Stores deposits data from invocation list eventDataList - * @param eventDataList All deposit events in blockHash - * @param blockHash Block hash + * + * @param eventDataList All deposit events in blockHash + * @param blockHash Block hash */ protected synchronized void newDeposits(List eventDataList, byte[] blockHash) { List deposits = @@ -70,7 +69,9 @@ protected synchronized void newDeposits(List eventDataList, by for (Deposit deposit : deposits) { Deposit depositProofed = new Deposit( - getProof(deposit.getIndex().intValue(), size), deposit.getIndex(), deposit.getData()); + tree.getProof(deposit.getIndex().intValue(), size), + deposit.getIndex(), + deposit.getData()); if (startChainSubscribed && !chainStartSink.isTerminated()) { initialDeposits.add(depositProofed); } @@ -93,7 +94,7 @@ private List restoreDeposits(List eventData, byte .map( deposit -> new Deposit( - getProof(deposit.getIndex().intValue(), size), + tree.getProof(deposit.getIndex().intValue(), size), deposit.getIndex(), deposit.getData())) .map( @@ -101,24 +102,30 @@ private List restoreDeposits(List eventData, byte new DepositInfo( d, new Eth1Data( - getDepositRoot(d.getIndex()), + tree.getDepositRoot(d.getIndex()), d.getIndex().decrement(), Hash32.wrap(Bytes32.wrap(blockHash))))) .collect(Collectors.toList()); } /** - * Inserts deposit in storage and returns it - * NOTE: returns Deposit with empty proof, proof should be filled by someone else - * @param eventData Deposit event + * Inserts deposit in storage and returns it NOTE: returns Deposit with empty proof, proof should + * be filled by someone else + * + * @param eventData Deposit event * @return Deposit */ private Deposit newDeposit(DepositEventData eventData) { Deposit deposit = createUnProofedDeposit(eventData); - insertDepositData(createDepositDataValue(deposit.getData()).extractArray()); + tree.insertValue(deposit.getData()); return deposit; } + public Hash32 getDepositRoot(byte[] merkleTreeIndex) { + UInt64 index = UInt64.fromBytesLittleEndian(Bytes8.wrap(merkleTreeIndex)); + return tree.getDepositRoot(index); + } + protected synchronized void chainStart( byte[] deposit_root, byte[] deposit_count, byte[] time, byte[] blockHash) { assert UInt64.fromBytesLittleEndian(Bytes8.wrap(deposit_count)).intValue() @@ -147,66 +154,6 @@ private void chainStartSubscribedPriv() { protected abstract void chainStartDone(); - protected ReadVector getProof(int index, int size) { - return ReadVector.wrap( - minimalMerkle.get_merkle_proof( - minimalMerkle.calc_merkle_tree_from_leaves(deposits.subList(0, size)), index), - Integer::new); - } - - private Hash32 getDepositRoot(UInt64 index) { - return Hash32.wrap( - Bytes32.leftPad(minimalMerkle.get_merkle_root(deposits.subList(0, index.intValue() + 1)))); - } - - protected Hash32 getDepositRoot(byte[] merkleTreeIndex) { - UInt64 index = UInt64.fromBytesLittleEndian(Bytes8.wrap(merkleTreeIndex)); - return getDepositRoot(index); - } - - private void insertDepositData(byte[] depositData) { - deposits.add(BytesValue.wrap(depositData)); - } - - // zero_bytes_32: bytes32 - // pubkey_root: bytes32 = sha256(concat(pubkey, slice(zero_bytes_32, start=0, len=16))) - // signature_root: bytes32 = sha256(concat( - // sha256(slice(signature, start=0, len=64)), - // sha256(concat(slice(signature, start=64, len=32), zero_bytes_32)) - // )) - // value: bytes32 = sha256(concat( - // sha256(concat(pubkey_root, withdrawal_credentials)), - // sha256(concat( - // amount, - // slice(zero_bytes_32, start=0, len=24), - // signature_root, - // )) - // )) - private Hash32 createDepositDataValue(DepositData depositData) { - BytesValue zero_bytes_32 = Bytes32.ZERO.slice(0); - Hash32 pubkey_root = - hashFunction.apply(depositData.getPubKey().concat(zero_bytes_32.slice(0, 16))); - Hash32 signature_root = - hashFunction.apply( - hashFunction - .apply(depositData.getSignature().slice(0, 64)) - .concat( - hashFunction.apply( - depositData.getSignature().slice(64, 32).concat(zero_bytes_32)))); - Hash32 value = - hashFunction.apply( - hashFunction - .apply(pubkey_root.concat(depositData.getWithdrawalCredentials())) - .concat( - hashFunction.apply( - depositData - .getAmount() - .toBytes8LittleEndian() - .concat(zero_bytes_32.slice(0, 24).concat(signature_root))))); - - return value; - } - @Override public Publisher getChainStartMono() { return chainStartStream; diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositDataMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositDataMerkle.java new file mode 100644 index 000000000..8d978c6a0 --- /dev/null +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositDataMerkle.java @@ -0,0 +1,78 @@ +package org.ethereum.beacon.pow; + +import org.ethereum.beacon.core.operations.deposit.DepositData; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; + +import java.util.function.Function; + +import static tech.pegasys.artemis.util.bytes.BytesValue.concat; + +/** Abstract implementation of {@link MerkleTree} for {@link DepositData} */ +abstract class DepositDataMerkle implements MerkleTree { + + private final Hash32[] zeroHashes; + private final Function hashFunction; + + DepositDataMerkle(Function hashFunction, int treeDepth) { + this.hashFunction = hashFunction; + this.zeroHashes = new Hash32[treeDepth]; + } + + // zero_bytes_32: bytes32 + // pubkey_root: bytes32 = sha256(concat(pubkey, slice(zero_bytes_32, start=0, len=16))) + // signature_root: bytes32 = sha256(concat( + // sha256(slice(signature, start=0, len=64)), + // sha256(concat(slice(signature, start=64, len=32), zero_bytes_32)) + // )) + // value: bytes32 = sha256(concat( + // sha256(concat(pubkey_root, withdrawal_credentials)), + // sha256(concat( + // amount, + // slice(zero_bytes_32, start=0, len=24), + // signature_root, + // )) + // )) + static Hash32 createDepositDataValue( + DepositData depositData, Function hashFunction) { + BytesValue zero_bytes_32 = Bytes32.ZERO.slice(0); + Hash32 pubkey_root = + hashFunction.apply(depositData.getPubKey().concat(zero_bytes_32.slice(0, 16))); + Hash32 signature_root = + hashFunction.apply( + hashFunction + .apply(depositData.getSignature().slice(0, 64)) + .concat( + hashFunction.apply( + depositData.getSignature().slice(64, 32).concat(zero_bytes_32)))); + Hash32 value = + hashFunction.apply( + hashFunction + .apply(pubkey_root.concat(depositData.getWithdrawalCredentials())) + .concat( + hashFunction.apply( + depositData + .getAmount() + .toBytes8LittleEndian() + .concat(zero_bytes_32.slice(0, 24).concat(signature_root))))); + + return value; + } + + Function getHashFunction() { + return hashFunction; + } + + Hash32 getZeroHash(int distanceFromBottom) { + if (zeroHashes[distanceFromBottom] == null) { + if (distanceFromBottom == 0) { + zeroHashes[0] = Hash32.ZERO; + } else { + Hash32 lowerZeroHash = getZeroHash(distanceFromBottom - 1); + zeroHashes[distanceFromBottom] = hashFunction.apply(concat(lowerZeroHash, lowerZeroHash)); + } + } + return zeroHashes[distanceFromBottom]; + } +} diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java new file mode 100644 index 000000000..714432aef --- /dev/null +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java @@ -0,0 +1,158 @@ +package org.ethereum.beacon.pow; + +import org.ethereum.beacon.core.operations.deposit.DepositData; +import org.ethereum.beacon.util.ConsumerList; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.collections.ReadVector; +import tech.pegasys.artemis.util.uint.UInt64; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +/** + * Progressive Merkle Adoption of https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py + */ +public class DepositIncrementalMerkle extends DepositDataMerkle { + + private final int treeDepth; + private final List lastElements; + private int branchDepositCount = 0; + private List branch; + + public DepositIncrementalMerkle( + Function hashFunction, int treeDepth, int bufferDeposits) { + super(hashFunction, treeDepth); + this.treeDepth = treeDepth; + Consumer consumer = + depositData -> { + branchDepositCount++; + add_value(branch, branchDepositCount - 1, depositData.slice(0)); + }; + this.lastElements = ConsumerList.create(bufferDeposits, consumer); + this.branch = + IntStream.range(0, treeDepth) + .mapToObj(this::getZeroHash) + .map(h -> h.slice(0)) + .collect(Collectors.toList()); + } + + // # Add a value to a Merkle tree by using the algo + // # that stores a branch of sub-roots + // def add_value(branch, index, value): + // i = 0 + // while (index+1) % 2**(i+1) == 0: + // i += 1 + // for j in range(0, i): + // value = hash(branch[j] + value) + // # branch[j] = zerohashes[j] + // branch[i] = value + void add_value(List branch, int index, BytesValue value) { + int i = 0; + while ((index + 1) % (1 << (i + 1)) == 0) { + ++i; + } + + BytesValue valueCopy = value; + for (int j = 0; j < i; ++j) { + valueCopy = getHashFunction().apply(branch.get(j).concat(valueCopy)); + } + branch.set(i, valueCopy); + } + + // def get_root_from_branch(branch, size): + // r = b'\x00' * 32 + // for h in range(32): + // if (size >> h) % 2 == 1: + // r = hash(branch[h] + r) + // else: + // r = hash(r + zerohashes[h]) + // return r + BytesValue get_root_from_branch(List branch, int size) { + BytesValue r = Bytes32.ZERO.slice(0); + for (int h = 0; h < treeDepth; ++h) { + if ((size >> h) % 2 == 1) { + r = getHashFunction().apply(branch.get(h).concat(r)); + } else { + r = getHashFunction().apply(r.concat(getZeroHash(h))); + } + } + + return r; + } + + @Override + public ReadVector getProof(int index, int size) { + verifyIndexNotTooBig(index); + verifyIndexNotTooOld(index); + if (size > (getLastIndex() + 1)) { + throw new RuntimeException( + String.format("Max size is %s, asked for size %s!", getLastIndex() + 1, size)); + } + List branchCopy = new ArrayList<>(branch); + for (int i = branchDepositCount; i < index + 1; ++i) { + add_value(branchCopy, i, lastElements.get(i - branchDepositCount)); + } + + return ReadVector.wrap( + branchCopy.stream().map(e -> Hash32.wrap(Bytes32.leftPad(e))).collect(Collectors.toList()), + Integer::new); + } + + @Override + public Hash32 getDepositRoot(UInt64 index) { + verifyIndexNotTooBig(index.intValue()); + verifyIndexNotTooOld(index.intValue()); + + List branchCopy = new ArrayList<>(branch); + // index of 1st == 0 + for (int i = branchDepositCount; i < index.intValue() + 1; ++i) { + add_value(branchCopy, i, lastElements.get(i - branchDepositCount)); + } + BytesValue root = get_root_from_branch(branchCopy, index.intValue() + 1); + return Hash32.wrap(Bytes32.leftPad(root)); + } + + private void verifyIndexNotTooOld(int index) { + if (index < branchDepositCount) { + throw new RuntimeException( + String.format("Too old element index queried, %s, minimum: %s!", index, branchDepositCount)); + } + } + + private void verifyIndexNotTooBig(int index) { + if (index > getLastIndex()) { + throw new RuntimeException( + String.format("Max element index is %s, asked for %s!", getLastIndex(), index)); + } + } + + @Override + public void insertValue(DepositData value) { + lastElements.add(createDepositDataValue(value, getHashFunction())); + } + + @Override + public int getLastIndex() { + return branchDepositCount + lastElements.size() - 1; + } + + public Stream getLastElementsStream() { + return lastElements.stream(); + } + + public int getBranchDepositCount() { + return branchDepositCount; + } + + public Stream getBranchStream() { + return branch.stream(); + } +} diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java similarity index 50% rename from pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java rename to pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java index f2cee6082..9cb26d2c4 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/MinimalMerkle.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java @@ -1,43 +1,41 @@ package org.ethereum.beacon.pow; +import org.ethereum.beacon.core.operations.deposit.DepositData; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.collections.ReadVector; +import tech.pegasys.artemis.util.uint.UInt64; import java.util.ArrayList; import java.util.List; import java.util.function.Function; - -import static tech.pegasys.artemis.util.bytes.BytesValue.concat; +import java.util.stream.Stream; /** - * Minimal merkle tree Minimal merkle tree https://en.wikipedia.org/wiki/Merkle_tree * implementation, adoption from python code from https://github.com/ethereum/research/blob/master/spec_pythonizer/utils/merkle_minimal.py */ -public class MinimalMerkle { +public class DepositSimpleMerkle extends DepositDataMerkle { private final int treeDepth; - private final Hash32[] zeroHashes; - private final Function hashFunction; + private List deposits = new ArrayList<>(); - public MinimalMerkle(Function hashFunction, int treeDepth) { - this.hashFunction = hashFunction; + public DepositSimpleMerkle(Function hashFunction, int treeDepth) { + super(hashFunction, treeDepth); this.treeDepth = treeDepth; - this.zeroHashes = new Hash32[treeDepth]; } - private Hash32 getZeroHash(int distanceFromBottom) { - if (zeroHashes[distanceFromBottom] == null) { - if (distanceFromBottom == 0) { - zeroHashes[0] = Hash32.ZERO; - } else { - Hash32 lowerZeroHash = getZeroHash(distanceFromBottom - 1); - zeroHashes[distanceFromBottom] = hashFunction.apply(concat(lowerZeroHash, lowerZeroHash)); - } - } - return zeroHashes[distanceFromBottom]; + public DepositSimpleMerkle( + Function hashFunction, int treeDepth, List deposits) { + this(hashFunction, treeDepth); + this.deposits = deposits; } // # Compute a Merkle root of a right-zerobyte-padded 2**32 sized tree @@ -50,7 +48,7 @@ private Hash32 getZeroHash(int distanceFromBottom) { // values = [hash(values[i] + values[i+1]) for i in range(0, len(values), 2)] // tree.append(values[::]) // return tree - public List> calc_merkle_tree_from_leaves(List valueList) { + private List> calc_merkle_tree_from_leaves(List valueList) { List values = new ArrayList<>(valueList); List> tree = new ArrayList<>(); tree.add(values); @@ -60,7 +58,7 @@ public List> calc_merkle_tree_from_leaves(List valu } List valuesTemp = new ArrayList<>(); for (int i = 0; i < values.size(); i += 2) { - valuesTemp.add(hashFunction.apply(values.get(i).concat(values.get(i + 1)))); + valuesTemp.add(getHashFunction().apply(values.get(i).concat(values.get(i + 1)))); } values = valuesTemp; tree.add(values); @@ -71,7 +69,7 @@ public List> calc_merkle_tree_from_leaves(List valu // def get_merkle_root(values): // return calc_merkle_tree_from_leaves(values)[-1][0] - public BytesValue get_merkle_root(List values) { + private BytesValue get_merkle_root(List values) { List> tree = calc_merkle_tree_from_leaves(values); try { return tree.get(tree.size() - 1).get(0); @@ -86,7 +84,7 @@ public BytesValue get_merkle_root(List values) { // subindex = (item_index//2**i)^1 // proof.append(tree[i][subindex] if subindex < len(tree[i]) else zerohashes[i]) // return proof - public List get_merkle_proof(List> tree, int item_index) { + private List get_merkle_proof(List> tree, int item_index) { List proof = new ArrayList<>(); for (int i = 0; i < treeDepth; ++i) { int subIndex = (item_index / (1 << i)) ^ 1; @@ -99,4 +97,42 @@ public List get_merkle_proof(List> tree, int item_index return proof; } + + @Override + public ReadVector getProof(int index, int size) { + if (index > getLastIndex() || size > (getLastIndex() + 1)) { + throw new RuntimeException( + String.format("Max element index is %s, asked for %s and size %s!", getLastIndex(), index, size)); + } + return ReadVector.wrap( + get_merkle_proof(calc_merkle_tree_from_leaves(deposits.subList(0, size)), index), + Integer::new); + } + + @Override + public Hash32 getDepositRoot(UInt64 index) { + if (index.intValue() > getLastIndex()) { + throw new RuntimeException( + String.format("Max element index is %s, asked for %s!", getLastIndex(), index)); + } + return Hash32.wrap(Bytes32.leftPad(get_merkle_root(deposits.subList(0, index.intValue() + 1)))); + } + + @Override + public void insertValue(DepositData depositData) { + insertDepositData(createDepositDataValue(depositData, getHashFunction()).extractArray()); + } + + public Stream getDepositStream() { + return deposits.stream(); + } + + private void insertDepositData(byte[] depositData) { + this.deposits.add(BytesValue.wrap(depositData)); + } + + @Override + public int getLastIndex() { + return deposits.size() - 1; + } } diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java b/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java new file mode 100644 index 000000000..1f55c27e6 --- /dev/null +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java @@ -0,0 +1,37 @@ +package org.ethereum.beacon.pow; + +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.collections.ReadVector; +import tech.pegasys.artemis.util.uint.UInt64; + +/** + * Merkle Tree https://en.wikipedia.org/wiki/Merkle_tree with proofs + * @param Element type + */ +public interface MerkleTree { + /** + * Proofs for element + * @param index at index + * @param size with all tree made of size elements + * @return proofs + */ + ReadVector getProof(int index, int size); + + /** + * Deposit Root of merkle tree with all elements up to index + * @param index last element index + * @return deposit root + */ + Hash32 getDepositRoot(UInt64 index); + + /** + * Inserts value in tree / storage + * @param value Element value + */ + void insertValue(V value); + + /** + * @return Index of last/highest element + */ + int getLastIndex(); +} diff --git a/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java b/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java new file mode 100644 index 000000000..033adb733 --- /dev/null +++ b/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java @@ -0,0 +1,71 @@ +package org.ethereum.beacon.pow.util; + +import org.ethereum.beacon.core.operations.deposit.DepositData; +import org.ethereum.beacon.core.types.BLSPubkey; +import org.ethereum.beacon.core.types.BLSSignature; +import org.ethereum.beacon.core.types.Gwei; +import org.ethereum.beacon.crypto.Hashes; +import org.ethereum.beacon.pow.DepositIncrementalMerkle; +import org.ethereum.beacon.pow.DepositSimpleMerkle; +import org.ethereum.beacon.pow.MerkleTree; +import org.junit.Test; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.Bytes48; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.uint.UInt64; + +import java.util.function.Consumer; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class DepositDataMerkleTest { + @Test + public void test() { + MerkleTree simple = new DepositSimpleMerkle(Hashes::sha256, 32); + MerkleTree incremental = new DepositIncrementalMerkle(Hashes::sha256, 32, 3); + Consumer insertInBoth = + integer -> { + simple.insertValue(createDepositData(integer)); + incremental.insertValue(createDepositData(integer)); + }; + for (int i = 1; i < 20; ++i) { + insertInBoth.accept(i); + assertEquals( + simple.getDepositRoot(UInt64.valueOf(i - 1)), + incremental.getDepositRoot(UInt64.valueOf(i - 1))); + } + int someIndex = simple.getLastIndex() + 1; + for (int i = 20; i < 22; ++i) { + insertInBoth.accept(i); + assertArrayEquals( + simple.getProof(someIndex, i).listCopy().toArray(), + incremental.getProof(someIndex, i).listCopy().toArray()); + } + insertInBoth.accept(22); + try { + simple.getProof(someIndex,22); + fail(); + } catch (RuntimeException e) { + + } + try { + incremental.getProof(someIndex,22); + fail(); + } catch (RuntimeException e) { + + } + + } + + private DepositData createDepositData(int num) { + return new DepositData( + BLSPubkey.wrap( + Bytes48.leftPad( + BytesValue.wrap(UInt64.valueOf(num).toBytes8LittleEndian().extractArray()))), + Hash32.ZERO, + Gwei.ZERO, + BLSSignature.ZERO); + } +} diff --git a/util/src/main/java/org/ethereum/beacon/util/ConsumerList.java b/util/src/main/java/org/ethereum/beacon/util/ConsumerList.java new file mode 100644 index 000000000..07e5d5e93 --- /dev/null +++ b/util/src/main/java/org/ethereum/beacon/util/ConsumerList.java @@ -0,0 +1,125 @@ +package org.ethereum.beacon.util; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; + +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.function.Consumer; +import java.util.stream.Stream; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +public final class ConsumerList extends LinkedList { + + private final LinkedList delegate; + private final Consumer spillOutConsumer; + + @VisibleForTesting + final int maxSize; + + private ConsumerList(int maxSize, Consumer spillOutConsumer) { + checkArgument(maxSize >= 0, "maxSize (%s) must >= 0", maxSize); + this.delegate = new LinkedList(); + this.maxSize = maxSize; + this.spillOutConsumer = spillOutConsumer; + } + + /** + * Creates and returns a new consumer list that will hold up to {@code maxSize} elements. + * + *

When {@code maxSize} is zero, elements will be consumed immediately after being added to the + * queue. + */ + public static ConsumerList create(int maxSize, Consumer spillOutConsumer) { + return new ConsumerList<>(maxSize, spillOutConsumer); + } + + + /** + * Returns the number of additional elements that this list can accept without consuming; zero if + * the list is currently full. + * + * @since 16.0 + */ + public int remainingCapacity() { + return maxSize - size(); + } + + protected List delegate() { + return delegate; + } + + /** + * Adds the given element to this queue. If the queue is currently full, the element at the head + * of the queue is consumed to make room. + * + * @return {@code true} always + */ + @Override + public boolean add(E e) { + checkNotNull(e); // check before removing + if (maxSize == 0) { + spillOutConsumer.accept(e); + return true; + } + if (size() == maxSize) { + spillOutConsumer.accept(delegate.remove()); + } + delegate.add(e); + return true; + } + + @Override + public boolean addAll(Collection collection) { + int size = collection.size(); + if (size >= maxSize) { + while(!delegate.isEmpty()) { + spillOutConsumer.accept(delegate.remove()); + } + int numberToSkip = size - maxSize; + Iterator iterator = collection.iterator(); + int i = 0; + while (iterator.hasNext() && i < numberToSkip) { + spillOutConsumer.accept(iterator.next()); + ++i; + } + return Iterables.addAll(this, Iterables.skip(collection, numberToSkip)); + } + return Iterators.addAll(this, collection.iterator()); + } + + @Override + public int size() { + return delegate.size(); + } + + @Override + public boolean contains(Object o) { + return delegate.contains(o); + } + + @Override + public E getFirst() { + return delegate.getFirst(); + } + + @Override + public E getLast() { + return delegate.getLast(); + } + + @Override + public E get(int index) { + return delegate.get(index); + } + + @Override + public Stream stream() { + return delegate.stream(); + } +} diff --git a/util/src/test/java/org/ethereum/beacon/util/ConsumerListTest.java b/util/src/test/java/org/ethereum/beacon/util/ConsumerListTest.java new file mode 100644 index 000000000..64210084c --- /dev/null +++ b/util/src/test/java/org/ethereum/beacon/util/ConsumerListTest.java @@ -0,0 +1,44 @@ +package org.ethereum.beacon.util; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ConsumerListTest { + @Test + public void test() { + final List list = new ArrayList<>(); + Consumer consumer = list::add; + ConsumerList consumerList = ConsumerList.create(5, consumer); + assertEquals(0, consumerList.size()); + consumerList.add(1); + consumerList.add(2); + consumerList.add(3); + consumerList.add(4); + consumerList.add(5); + consumerList.add(6); + assertEquals(1, list.size()); + assertEquals(Integer.valueOf(1), list.get(0)); + assertEquals(5, consumerList.size()); + assertEquals(5, consumerList.maxSize); + List three = new ArrayList<>(); + three.add(7); + three.add(8); + three.add(9); + consumerList.addAll(three); + assertEquals(4, list.size()); + assertEquals(Integer.valueOf(2), list.get(1)); + assertEquals(Integer.valueOf(3), list.get(2)); + assertEquals(Integer.valueOf(4), list.get(3)); + assertTrue(consumerList.contains(5)); + assertTrue(consumerList.contains(6)); + assertTrue(consumerList.contains(7)); + assertTrue(consumerList.contains(8)); + assertTrue(consumerList.contains(9)); + } +} From e75c52e27df54c30f2cb119ad432f9b719917c7d Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 16 May 2019 16:41:41 +0300 Subject: [PATCH 08/14] pow: added javadoc to incremental merkle initialization --- .../org/ethereum/beacon/pow/DepositIncrementalMerkle.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java index 714432aef..313612909 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java @@ -27,6 +27,14 @@ public class DepositIncrementalMerkle extends DepositDataMerkle { private int branchDepositCount = 0; private List branch; + /** + * Incremental Merkle using + * + * @param hashFunction hash function + * @param treeDepth tree with depth of + * @param bufferDeposits number of not committed deposits, we could easily roll to any state + * backwards if it doesn't involve skipping more than this number of deposits + */ public DepositIncrementalMerkle( Function hashFunction, int treeDepth, int bufferDeposits) { super(hashFunction, treeDepth); From a4f60b1c58ef01d61b2dd8d3f5dc57c9961fc152 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Thu, 16 May 2019 16:56:59 +0300 Subject: [PATCH 09/14] pow: clean up input verifiers for merkle trees implementations --- .../org/ethereum/beacon/pow/DepositDataMerkle.java | 7 +++++++ .../ethereum/beacon/pow/DepositIncrementalMerkle.java | 9 +-------- .../org/ethereum/beacon/pow/DepositSimpleMerkle.java | 10 ++++------ 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositDataMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositDataMerkle.java index 8d978c6a0..facb00342 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositDataMerkle.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositDataMerkle.java @@ -75,4 +75,11 @@ Hash32 getZeroHash(int distanceFromBottom) { } return zeroHashes[distanceFromBottom]; } + + void verifyIndexNotTooBig(int index) { + if (index > getLastIndex()) { + throw new RuntimeException( + String.format("Max element index is %s, asked for %s!", getLastIndex(), index)); + } + } } diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java index 313612909..2e85e07b9 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java @@ -105,7 +105,7 @@ public ReadVector getProof(int index, int size) { String.format("Max size is %s, asked for size %s!", getLastIndex() + 1, size)); } List branchCopy = new ArrayList<>(branch); - for (int i = branchDepositCount; i < index + 1; ++i) { + for (int i = branchDepositCount; i < size; ++i) { add_value(branchCopy, i, lastElements.get(i - branchDepositCount)); } @@ -135,13 +135,6 @@ private void verifyIndexNotTooOld(int index) { } } - private void verifyIndexNotTooBig(int index) { - if (index > getLastIndex()) { - throw new RuntimeException( - String.format("Max element index is %s, asked for %s!", getLastIndex(), index)); - } - } - @Override public void insertValue(DepositData value) { lastElements.add(createDepositDataValue(value, getHashFunction())); diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java index 9cb26d2c4..83edef2a3 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java @@ -100,9 +100,10 @@ private List get_merkle_proof(List> tree, int item_inde @Override public ReadVector getProof(int index, int size) { - if (index > getLastIndex() || size > (getLastIndex() + 1)) { + verifyIndexNotTooBig(index); + if (size > (getLastIndex() + 1)) { throw new RuntimeException( - String.format("Max element index is %s, asked for %s and size %s!", getLastIndex(), index, size)); + String.format("Max size is %s, asked for size %s!", getLastIndex() + 1, size)); } return ReadVector.wrap( get_merkle_proof(calc_merkle_tree_from_leaves(deposits.subList(0, size)), index), @@ -111,10 +112,7 @@ public ReadVector getProof(int index, int size) { @Override public Hash32 getDepositRoot(UInt64 index) { - if (index.intValue() > getLastIndex()) { - throw new RuntimeException( - String.format("Max element index is %s, asked for %s!", getLastIndex(), index)); - } + verifyIndexNotTooBig(index.intValue()); return Hash32.wrap(Bytes32.leftPad(get_merkle_root(deposits.subList(0, index.intValue() + 1)))); } From 96eb6196f3d207d6e6692a76ed5db5932b14199d Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 17 May 2019 20:25:52 +0300 Subject: [PATCH 10/14] pow: fixed incremental merkle implementation --- .../beacon/pow/AbstractDepositContract.java | 2 +- .../beacon/pow/DepositBufferedMerkle.java | 139 +++++++++++++++ .../beacon/pow/DepositIncrementalMerkle.java | 159 ------------------ .../beacon/pow/DepositSimpleMerkle.java | 19 +-- .../org/ethereum/beacon/pow/MerkleTree.java | 24 +-- .../pow/util/DepositDataMerkleTest.java | 22 +-- 6 files changed, 159 insertions(+), 206 deletions(-) create mode 100644 pow/core/src/main/java/org/ethereum/beacon/pow/DepositBufferedMerkle.java delete mode 100644 pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java b/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java index 0ef1571f5..79572608c 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java @@ -50,7 +50,7 @@ public AbstractDepositContract( .doOnSubscribe(s -> chainStartSubscribedPriv()) .name("PowClient.chainStart"); depositStream = new SimpleProcessor<>(this.schedulers.reactorEvents(), "PowClient.deposit"); - this.tree = new DepositSimpleMerkle(hashFunction, treeDepth); + this.tree = new DepositBufferedMerkle(hashFunction, treeDepth, 1000); } /** diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositBufferedMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositBufferedMerkle.java new file mode 100644 index 000000000..db15a8d93 --- /dev/null +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositBufferedMerkle.java @@ -0,0 +1,139 @@ +package org.ethereum.beacon.pow; + +import org.ethereum.beacon.core.operations.deposit.DepositData; +import org.ethereum.beacon.util.ConsumerList; +import tech.pegasys.artemis.ethereum.core.Hash32; +import tech.pegasys.artemis.util.bytes.Bytes32; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.collections.ReadVector; +import tech.pegasys.artemis.util.uint.UInt64; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.IntStream; + +/** + * Sames as {@link DepositSimpleMerkle} but it's not keeping all elements, so any sliced tree could + * be created. Instead, it keeps last N elements and all other are put in the tree, so only slices + * of trees with last N elements could be created. + */ +public class DepositBufferedMerkle extends DepositDataMerkle { + + private final int treeDepth; + private final List lastElements; + private int treeDepositCount = 0; + private List> committedTree; + + /** + * Buffered Merkle tree using + * + * @param hashFunction hash function + * @param treeDepth tree with depth of + * @param bufferDeposits number of not committed deposits, we could easily roll to any state + * backwards if it doesn't involve skipping more than this number of deposits + */ + public DepositBufferedMerkle( + Function hashFunction, int treeDepth, int bufferDeposits) { + super(hashFunction, treeDepth); + this.treeDepth = treeDepth; + Consumer consumer = + depositData -> { + treeDepositCount++; + addValue(committedTree, depositData); + }; + this.lastElements = ConsumerList.create(bufferDeposits, consumer); + this.committedTree = new ArrayList<>(); + IntStream.range(0, treeDepth + 1).forEach(x -> committedTree.add(new ArrayList<>())); + } + + private void addValue(List> tree, BytesValue value) { + if (!tree.get(0).isEmpty() && tree.get(0).get(tree.get(0).size() - 1) == getZeroHash(0)) { + tree.get(0).remove(tree.get(0).size() - 1); + } + int stageSize = tree.get(0).size(); + tree.get(0).add(value); + for (int h = 0; h < treeDepth + 1; ++h) { + List stage = tree.get(h); + if (h > 0) { + // Remove elements that should be modified + stageSize = stageSize / 2; + while (stage.size() != stageSize) { + stage.remove(stage.size() - 1); + } + + List previousStage = tree.get(h - 1); + int previousStageSize = previousStage.size(); + stage.add( + getHashFunction() + .apply( + previousStage + .get(previousStageSize - 2) + .concat(previousStage.get(previousStageSize - 1)))); + } + if (stage.size() % 2 == 1 && h != treeDepth) { + stage.add(getZeroHash(h)); + } + } + } + + @Override + public ReadVector getProof(int index, int size) { + List> tree = buildTreeForIndex(size - 1); + List proof = new ArrayList<>(); + for (int i = 0; i < treeDepth; ++i) { + int subIndex = (index / (1 << i)) ^ 1; + if (subIndex < tree.get(i).size()) { + proof.add(Hash32.wrap(Bytes32.leftPad(tree.get(i).get(subIndex)))); + } else { + proof.add(getZeroHash(i)); + } + } + + return ReadVector.wrap(proof, Integer::new); + } + + private BytesValue get_merkle_root(List> tree) { + return tree.get(tree.size() - 1).get(0); + } + + private List> buildTreeForIndex(int index) { + verifyIndexNotTooBig(index); + verifyIndexNotTooOld(index); + + List> treeCopy = new ArrayList<>(); + committedTree.forEach(s -> treeCopy.add(new ArrayList<>(s))); + // index of 1st == 0 + for (int i = treeDepositCount; i < index + 1; ++i) { + addValue(treeCopy, lastElements.get(i - treeDepositCount)); + } + + return treeCopy; + } + + @Override + public Hash32 getDepositRoot(UInt64 index) { + List> tree = buildTreeForIndex(index.intValue()); + BytesValue root = get_merkle_root(tree); + return Hash32.wrap(Bytes32.leftPad(root)); + } + + private void verifyIndexNotTooOld(int index) { + if (index < treeDepositCount) { + throw new RuntimeException( + String.format( + "Too old element index queried, %s, minimum: %s!", index, treeDepositCount)); + } + } + + @Override + public void insertValue(DepositData value) { + lastElements.add(createDepositDataValue(value, getHashFunction())); + } + + @Override + public int getLastIndex() { + return treeDepositCount + lastElements.size() - 1; + } +} diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java deleted file mode 100644 index 2e85e07b9..000000000 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositIncrementalMerkle.java +++ /dev/null @@ -1,159 +0,0 @@ -package org.ethereum.beacon.pow; - -import org.ethereum.beacon.core.operations.deposit.DepositData; -import org.ethereum.beacon.util.ConsumerList; -import tech.pegasys.artemis.ethereum.core.Hash32; -import tech.pegasys.artemis.util.bytes.Bytes32; -import tech.pegasys.artemis.util.bytes.BytesValue; -import tech.pegasys.artemis.util.collections.ReadVector; -import tech.pegasys.artemis.util.uint.UInt64; - -import java.util.ArrayList; -import java.util.List; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -/** - * Progressive Merkle Adoption of https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py - */ -public class DepositIncrementalMerkle extends DepositDataMerkle { - - private final int treeDepth; - private final List lastElements; - private int branchDepositCount = 0; - private List branch; - - /** - * Incremental Merkle using - * - * @param hashFunction hash function - * @param treeDepth tree with depth of - * @param bufferDeposits number of not committed deposits, we could easily roll to any state - * backwards if it doesn't involve skipping more than this number of deposits - */ - public DepositIncrementalMerkle( - Function hashFunction, int treeDepth, int bufferDeposits) { - super(hashFunction, treeDepth); - this.treeDepth = treeDepth; - Consumer consumer = - depositData -> { - branchDepositCount++; - add_value(branch, branchDepositCount - 1, depositData.slice(0)); - }; - this.lastElements = ConsumerList.create(bufferDeposits, consumer); - this.branch = - IntStream.range(0, treeDepth) - .mapToObj(this::getZeroHash) - .map(h -> h.slice(0)) - .collect(Collectors.toList()); - } - - // # Add a value to a Merkle tree by using the algo - // # that stores a branch of sub-roots - // def add_value(branch, index, value): - // i = 0 - // while (index+1) % 2**(i+1) == 0: - // i += 1 - // for j in range(0, i): - // value = hash(branch[j] + value) - // # branch[j] = zerohashes[j] - // branch[i] = value - void add_value(List branch, int index, BytesValue value) { - int i = 0; - while ((index + 1) % (1 << (i + 1)) == 0) { - ++i; - } - - BytesValue valueCopy = value; - for (int j = 0; j < i; ++j) { - valueCopy = getHashFunction().apply(branch.get(j).concat(valueCopy)); - } - branch.set(i, valueCopy); - } - - // def get_root_from_branch(branch, size): - // r = b'\x00' * 32 - // for h in range(32): - // if (size >> h) % 2 == 1: - // r = hash(branch[h] + r) - // else: - // r = hash(r + zerohashes[h]) - // return r - BytesValue get_root_from_branch(List branch, int size) { - BytesValue r = Bytes32.ZERO.slice(0); - for (int h = 0; h < treeDepth; ++h) { - if ((size >> h) % 2 == 1) { - r = getHashFunction().apply(branch.get(h).concat(r)); - } else { - r = getHashFunction().apply(r.concat(getZeroHash(h))); - } - } - - return r; - } - - @Override - public ReadVector getProof(int index, int size) { - verifyIndexNotTooBig(index); - verifyIndexNotTooOld(index); - if (size > (getLastIndex() + 1)) { - throw new RuntimeException( - String.format("Max size is %s, asked for size %s!", getLastIndex() + 1, size)); - } - List branchCopy = new ArrayList<>(branch); - for (int i = branchDepositCount; i < size; ++i) { - add_value(branchCopy, i, lastElements.get(i - branchDepositCount)); - } - - return ReadVector.wrap( - branchCopy.stream().map(e -> Hash32.wrap(Bytes32.leftPad(e))).collect(Collectors.toList()), - Integer::new); - } - - @Override - public Hash32 getDepositRoot(UInt64 index) { - verifyIndexNotTooBig(index.intValue()); - verifyIndexNotTooOld(index.intValue()); - - List branchCopy = new ArrayList<>(branch); - // index of 1st == 0 - for (int i = branchDepositCount; i < index.intValue() + 1; ++i) { - add_value(branchCopy, i, lastElements.get(i - branchDepositCount)); - } - BytesValue root = get_root_from_branch(branchCopy, index.intValue() + 1); - return Hash32.wrap(Bytes32.leftPad(root)); - } - - private void verifyIndexNotTooOld(int index) { - if (index < branchDepositCount) { - throw new RuntimeException( - String.format("Too old element index queried, %s, minimum: %s!", index, branchDepositCount)); - } - } - - @Override - public void insertValue(DepositData value) { - lastElements.add(createDepositDataValue(value, getHashFunction())); - } - - @Override - public int getLastIndex() { - return branchDepositCount + lastElements.size() - 1; - } - - public Stream getLastElementsStream() { - return lastElements.stream(); - } - - public int getBranchDepositCount() { - return branchDepositCount; - } - - public Stream getBranchStream() { - return branch.stream(); - } -} diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java index 83edef2a3..25c296bbd 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java @@ -10,12 +10,9 @@ import java.util.ArrayList; import java.util.List; import java.util.function.Function; -import java.util.stream.Stream; /** * Simplest implementation that keeps all input values in memory and recalculates tree on any query. - * Values could be backed up via {@link #getDepositStream()} and loaded using alternative - * constructor * *

Minimal merkle tree https://en.wikipedia.org/wiki/Merkle_tree @@ -32,12 +29,6 @@ public DepositSimpleMerkle(Function hashFunction, int treeDe this.treeDepth = treeDepth; } - public DepositSimpleMerkle( - Function hashFunction, int treeDepth, List deposits) { - this(hashFunction, treeDepth); - this.deposits = deposits; - } - // # Compute a Merkle root of a right-zerobyte-padded 2**32 sized tree // def calc_merkle_tree_from_leaves(values): // values = list(values) @@ -71,11 +62,7 @@ private List> calc_merkle_tree_from_leaves(List val // return calc_merkle_tree_from_leaves(values)[-1][0] private BytesValue get_merkle_root(List values) { List> tree = calc_merkle_tree_from_leaves(values); - try { - return tree.get(tree.size() - 1).get(0); - } catch (Exception ex) { - throw ex; - } + return tree.get(tree.size() - 1).get(0); } // def get_merkle_proof(tree, item_index): @@ -121,10 +108,6 @@ public void insertValue(DepositData depositData) { insertDepositData(createDepositDataValue(depositData, getHashFunction()).extractArray()); } - public Stream getDepositStream() { - return deposits.stream(); - } - private void insertDepositData(byte[] depositData) { this.deposits.add(BytesValue.wrap(depositData)); } diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java b/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java index 1f55c27e6..85d9c9b1f 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java @@ -5,33 +5,37 @@ import tech.pegasys.artemis.util.uint.UInt64; /** - * Merkle Tree https://en.wikipedia.org/wiki/Merkle_tree with proofs - * @param Element type + * Merkle Hash Tree https://en.wikipedia.org/wiki/Merkle_tree + * with proofs + * + * @param Element type */ public interface MerkleTree { /** * Proofs for element + * * @param index at index - * @param size with all tree made of size elements - * @return proofs + * @param size with all tree made of size elements + * @return proofs */ ReadVector getProof(int index, int size); /** * Deposit Root of merkle tree with all elements up to index - * @param index last element index - * @return deposit root + * + * @param index last element index + * @return deposit root */ Hash32 getDepositRoot(UInt64 index); /** * Inserts value in tree / storage - * @param value Element value + * + * @param value Element value */ void insertValue(V value); - /** - * @return Index of last/highest element - */ + /** @return Index of last/highest element */ int getLastIndex(); } diff --git a/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java b/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java index 033adb733..9aadf7771 100644 --- a/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java +++ b/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java @@ -5,7 +5,7 @@ import org.ethereum.beacon.core.types.BLSSignature; import org.ethereum.beacon.core.types.Gwei; import org.ethereum.beacon.crypto.Hashes; -import org.ethereum.beacon.pow.DepositIncrementalMerkle; +import org.ethereum.beacon.pow.DepositBufferedMerkle; import org.ethereum.beacon.pow.DepositSimpleMerkle; import org.ethereum.beacon.pow.MerkleTree; import org.junit.Test; @@ -18,45 +18,31 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; public class DepositDataMerkleTest { @Test public void test() { MerkleTree simple = new DepositSimpleMerkle(Hashes::sha256, 32); - MerkleTree incremental = new DepositIncrementalMerkle(Hashes::sha256, 32, 3); + MerkleTree incremental = new DepositBufferedMerkle(Hashes::sha256, 32, 3); Consumer insertInBoth = integer -> { simple.insertValue(createDepositData(integer)); incremental.insertValue(createDepositData(integer)); }; for (int i = 1; i < 20; ++i) { + System.out.println(i); insertInBoth.accept(i); assertEquals( simple.getDepositRoot(UInt64.valueOf(i - 1)), incremental.getDepositRoot(UInt64.valueOf(i - 1))); } int someIndex = simple.getLastIndex() + 1; - for (int i = 20; i < 22; ++i) { + for (int i = 20; i < 40; ++i) { insertInBoth.accept(i); assertArrayEquals( simple.getProof(someIndex, i).listCopy().toArray(), incremental.getProof(someIndex, i).listCopy().toArray()); } - insertInBoth.accept(22); - try { - simple.getProof(someIndex,22); - fail(); - } catch (RuntimeException e) { - - } - try { - incremental.getProof(someIndex,22); - fail(); - } catch (RuntimeException e) { - - } - } private DepositData createDepositData(int num) { From 8fa34f2dabe936602fd992f8f42fc90ebdcdba5d Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Fri, 17 May 2019 20:45:53 +0300 Subject: [PATCH 11/14] pow: fixed missed dependency --- pow/core/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/pow/core/build.gradle b/pow/core/build.gradle index 7c045776b..abdaa4b88 100644 --- a/pow/core/build.gradle +++ b/pow/core/build.gradle @@ -3,6 +3,7 @@ dependencies { implementation project(':core') implementation project(':ssz') implementation project(':util') + implementation project(':crypto') implementation 'io.projectreactor:reactor-core' implementation 'com.google.guava:guava' From 98194cee3a32faf2ebf44d8dab8763a4eddcb189 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 20 May 2019 21:47:45 +0300 Subject: [PATCH 12/14] pow: Merkle Tree interface method renamed to be abstract and clean --- .../ethereum/beacon/pow/AbstractDepositContract.java | 6 +++--- .../org/ethereum/beacon/pow/DepositBufferedMerkle.java | 4 ++-- .../org/ethereum/beacon/pow/DepositSimpleMerkle.java | 4 ++-- .../main/java/org/ethereum/beacon/pow/MerkleTree.java | 10 +++++----- .../beacon/pow/util/DepositDataMerkleTest.java | 8 ++++---- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java b/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java index 79572608c..d7008b3a5 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java @@ -102,7 +102,7 @@ private List restoreDeposits(List eventData, byte new DepositInfo( d, new Eth1Data( - tree.getDepositRoot(d.getIndex()), + tree.getRoot(d.getIndex()), d.getIndex().decrement(), Hash32.wrap(Bytes32.wrap(blockHash))))) .collect(Collectors.toList()); @@ -117,13 +117,13 @@ private List restoreDeposits(List eventData, byte */ private Deposit newDeposit(DepositEventData eventData) { Deposit deposit = createUnProofedDeposit(eventData); - tree.insertValue(deposit.getData()); + tree.addValue(deposit.getData()); return deposit; } public Hash32 getDepositRoot(byte[] merkleTreeIndex) { UInt64 index = UInt64.fromBytesLittleEndian(Bytes8.wrap(merkleTreeIndex)); - return tree.getDepositRoot(index); + return tree.getRoot(index); } protected synchronized void chainStart( diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositBufferedMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositBufferedMerkle.java index db15a8d93..b40444f10 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositBufferedMerkle.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositBufferedMerkle.java @@ -113,7 +113,7 @@ private List> buildTreeForIndex(int index) { } @Override - public Hash32 getDepositRoot(UInt64 index) { + public Hash32 getRoot(UInt64 index) { List> tree = buildTreeForIndex(index.intValue()); BytesValue root = get_merkle_root(tree); return Hash32.wrap(Bytes32.leftPad(root)); @@ -128,7 +128,7 @@ private void verifyIndexNotTooOld(int index) { } @Override - public void insertValue(DepositData value) { + public void addValue(DepositData value) { lastElements.add(createDepositDataValue(value, getHashFunction())); } diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java index 25c296bbd..5c560e770 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java @@ -98,13 +98,13 @@ public ReadVector getProof(int index, int size) { } @Override - public Hash32 getDepositRoot(UInt64 index) { + public Hash32 getRoot(UInt64 index) { verifyIndexNotTooBig(index.intValue()); return Hash32.wrap(Bytes32.leftPad(get_merkle_root(deposits.subList(0, index.intValue() + 1)))); } @Override - public void insertValue(DepositData depositData) { + public void addValue(DepositData depositData) { insertDepositData(createDepositDataValue(depositData, getHashFunction()).extractArray()); } diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java b/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java index 85d9c9b1f..6876eea96 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java @@ -13,7 +13,7 @@ */ public interface MerkleTree { /** - * Proofs for element + * Proofs for element with provided index on tree with specified size * * @param index at index * @param size with all tree made of size elements @@ -22,19 +22,19 @@ public interface MerkleTree { ReadVector getProof(int index, int size); /** - * Deposit Root of merkle tree with all elements up to index + * Root of merkle tree with all elements up to index * * @param index last element index - * @return deposit root + * @return tree root */ - Hash32 getDepositRoot(UInt64 index); + Hash32 getRoot(UInt64 index); /** * Inserts value in tree / storage * * @param value Element value */ - void insertValue(V value); + void addValue(V value); /** @return Index of last/highest element */ int getLastIndex(); diff --git a/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java b/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java index 9aadf7771..87eda65b7 100644 --- a/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java +++ b/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java @@ -26,15 +26,15 @@ public void test() { MerkleTree incremental = new DepositBufferedMerkle(Hashes::sha256, 32, 3); Consumer insertInBoth = integer -> { - simple.insertValue(createDepositData(integer)); - incremental.insertValue(createDepositData(integer)); + simple.addValue(createDepositData(integer)); + incremental.addValue(createDepositData(integer)); }; for (int i = 1; i < 20; ++i) { System.out.println(i); insertInBoth.accept(i); assertEquals( - simple.getDepositRoot(UInt64.valueOf(i - 1)), - incremental.getDepositRoot(UInt64.valueOf(i - 1))); + simple.getRoot(UInt64.valueOf(i - 1)), + incremental.getRoot(UInt64.valueOf(i - 1))); } int someIndex = simple.getLastIndex() + 1; for (int i = 20; i < 40; ++i) { From 55e20bc8940a222ecc75db903915b08106f1f03b Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 21 May 2019 11:10:43 +0300 Subject: [PATCH 13/14] pow: further MerkleTree interface cleanup and following changes --- .../ethereum/beacon/pow/AbstractDepositContract.java | 8 ++++---- .../ethereum/beacon/pow/DepositBufferedMerkle.java | 10 ++++------ .../org/ethereum/beacon/pow/DepositSimpleMerkle.java | 12 +++++------- .../java/org/ethereum/beacon/pow/MerkleTree.java | 8 ++++---- .../beacon/pow/util/DepositDataMerkleTest.java | 8 ++++---- 5 files changed, 21 insertions(+), 25 deletions(-) diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java b/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java index d7008b3a5..4a5bf0bed 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/AbstractDepositContract.java @@ -69,7 +69,7 @@ protected synchronized void newDeposits(List eventDataList, by for (Deposit deposit : deposits) { Deposit depositProofed = new Deposit( - tree.getProof(deposit.getIndex().intValue(), size), + ReadVector.wrap(tree.getProof(deposit.getIndex().intValue(), size), Integer::new), deposit.getIndex(), deposit.getData()); if (startChainSubscribed && !chainStartSink.isTerminated()) { @@ -94,7 +94,7 @@ private List restoreDeposits(List eventData, byte .map( deposit -> new Deposit( - tree.getProof(deposit.getIndex().intValue(), size), + ReadVector.wrap(tree.getProof(deposit.getIndex().intValue(), size), Integer::new), deposit.getIndex(), deposit.getData())) .map( @@ -102,7 +102,7 @@ private List restoreDeposits(List eventData, byte new DepositInfo( d, new Eth1Data( - tree.getRoot(d.getIndex()), + tree.getRoot(d.getIndex().intValue()), d.getIndex().decrement(), Hash32.wrap(Bytes32.wrap(blockHash))))) .collect(Collectors.toList()); @@ -123,7 +123,7 @@ private Deposit newDeposit(DepositEventData eventData) { public Hash32 getDepositRoot(byte[] merkleTreeIndex) { UInt64 index = UInt64.fromBytesLittleEndian(Bytes8.wrap(merkleTreeIndex)); - return tree.getRoot(index); + return tree.getRoot(index.intValue()); } protected synchronized void chainStart( diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositBufferedMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositBufferedMerkle.java index b40444f10..4c097a582 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositBufferedMerkle.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositBufferedMerkle.java @@ -5,8 +5,6 @@ import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; -import tech.pegasys.artemis.util.collections.ReadVector; -import tech.pegasys.artemis.util.uint.UInt64; import java.util.ArrayList; import java.util.List; @@ -79,7 +77,7 @@ private void addValue(List> tree, BytesValue value) { } @Override - public ReadVector getProof(int index, int size) { + public List getProof(int index, int size) { List> tree = buildTreeForIndex(size - 1); List proof = new ArrayList<>(); for (int i = 0; i < treeDepth; ++i) { @@ -91,7 +89,7 @@ public ReadVector getProof(int index, int size) { } } - return ReadVector.wrap(proof, Integer::new); + return proof; } private BytesValue get_merkle_root(List> tree) { @@ -113,8 +111,8 @@ private List> buildTreeForIndex(int index) { } @Override - public Hash32 getRoot(UInt64 index) { - List> tree = buildTreeForIndex(index.intValue()); + public Hash32 getRoot(int index) { + List> tree = buildTreeForIndex(index); BytesValue root = get_merkle_root(tree); return Hash32.wrap(Bytes32.leftPad(root)); } diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java index 5c560e770..647c236b1 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/DepositSimpleMerkle.java @@ -86,21 +86,19 @@ private List get_merkle_proof(List> tree, int item_inde } @Override - public ReadVector getProof(int index, int size) { + public List getProof(int index, int size) { verifyIndexNotTooBig(index); if (size > (getLastIndex() + 1)) { throw new RuntimeException( String.format("Max size is %s, asked for size %s!", getLastIndex() + 1, size)); } - return ReadVector.wrap( - get_merkle_proof(calc_merkle_tree_from_leaves(deposits.subList(0, size)), index), - Integer::new); + return get_merkle_proof(calc_merkle_tree_from_leaves(deposits.subList(0, size)), index); } @Override - public Hash32 getRoot(UInt64 index) { - verifyIndexNotTooBig(index.intValue()); - return Hash32.wrap(Bytes32.leftPad(get_merkle_root(deposits.subList(0, index.intValue() + 1)))); + public Hash32 getRoot(int index) { + verifyIndexNotTooBig(index); + return Hash32.wrap(Bytes32.leftPad(get_merkle_root(deposits.subList(0, index + 1)))); } @Override diff --git a/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java b/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java index 6876eea96..466b6e23d 100644 --- a/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java +++ b/pow/core/src/main/java/org/ethereum/beacon/pow/MerkleTree.java @@ -1,8 +1,8 @@ package org.ethereum.beacon.pow; import tech.pegasys.artemis.ethereum.core.Hash32; -import tech.pegasys.artemis.util.collections.ReadVector; -import tech.pegasys.artemis.util.uint.UInt64; + +import java.util.List; /** * Merkle Hash Tree { * @param size with all tree made of size elements * @return proofs */ - ReadVector getProof(int index, int size); + List getProof(int index, int size); /** * Root of merkle tree with all elements up to index @@ -27,7 +27,7 @@ public interface MerkleTree { * @param index last element index * @return tree root */ - Hash32 getRoot(UInt64 index); + Hash32 getRoot(int index); /** * Inserts value in tree / storage diff --git a/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java b/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java index 87eda65b7..afac5baf6 100644 --- a/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java +++ b/pow/core/src/test/java/org/ethereum/beacon/pow/util/DepositDataMerkleTest.java @@ -33,15 +33,15 @@ public void test() { System.out.println(i); insertInBoth.accept(i); assertEquals( - simple.getRoot(UInt64.valueOf(i - 1)), - incremental.getRoot(UInt64.valueOf(i - 1))); + simple.getRoot(i - 1), + incremental.getRoot(i - 1)); } int someIndex = simple.getLastIndex() + 1; for (int i = 20; i < 40; ++i) { insertInBoth.accept(i); assertArrayEquals( - simple.getProof(someIndex, i).listCopy().toArray(), - incremental.getProof(someIndex, i).listCopy().toArray()); + simple.getProof(someIndex, i).toArray(), + incremental.getProof(someIndex, i).toArray()); } } From 15d84b4d38d9c1125518d1f22d542dfd22c40c61 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 21 May 2019 11:21:58 +0300 Subject: [PATCH 14/14] util: clean-up of ConsumerList --- .../ethereum/beacon/util/ConsumerList.java | 108 +++++++++++++++--- 1 file changed, 94 insertions(+), 14 deletions(-) diff --git a/util/src/main/java/org/ethereum/beacon/util/ConsumerList.java b/util/src/main/java/org/ethereum/beacon/util/ConsumerList.java index 07e5d5e93..3cef692d9 100644 --- a/util/src/main/java/org/ethereum/beacon/util/ConsumerList.java +++ b/util/src/main/java/org/ethereum/beacon/util/ConsumerList.java @@ -8,20 +8,19 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.ListIterator; import java.util.function.Consumer; import java.util.stream.Stream; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -public final class ConsumerList extends LinkedList { +public final class ConsumerList implements List { + @VisibleForTesting final int maxSize; private final LinkedList delegate; private final Consumer spillOutConsumer; - @VisibleForTesting - final int maxSize; - private ConsumerList(int maxSize, Consumer spillOutConsumer) { checkArgument(maxSize >= 0, "maxSize (%s) must >= 0", maxSize); this.delegate = new LinkedList(); @@ -39,7 +38,6 @@ public static ConsumerList create(int maxSize, Consumer spillOutConsum return new ConsumerList<>(maxSize, spillOutConsumer); } - /** * Returns the number of additional elements that this list can accept without consuming; zero if * the list is currently full. @@ -78,7 +76,7 @@ public boolean add(E e) { public boolean addAll(Collection collection) { int size = collection.size(); if (size >= maxSize) { - while(!delegate.isEmpty()) { + while (!delegate.isEmpty()) { spillOutConsumer.accept(delegate.remove()); } int numberToSkip = size - maxSize; @@ -104,22 +102,104 @@ public boolean contains(Object o) { } @Override - public E getFirst() { - return delegate.getFirst(); + public E get(int index) { + return delegate.get(index); } @Override - public E getLast() { - return delegate.getLast(); + public Stream stream() { + return delegate.stream(); } @Override - public E get(int index) { - return delegate.get(index); + public boolean isEmpty() { + return delegate.isEmpty(); } @Override - public Stream stream() { - return delegate.stream(); + public Iterator iterator() { + return delegate.iterator(); + } + + @Override + public Object[] toArray() { + return delegate.toArray(); + } + + @Override + public T[] toArray(T[] a) { + return delegate.toArray(a); + } + + @Override + public boolean remove(Object o) { + return delegate.remove(o); + } + + @Override + public boolean containsAll(Collection c) { + return delegate.containsAll(c); + } + + @Override + public boolean addAll(int index, Collection c) { + throw new RuntimeException("Not implemented yet!"); + } + + @Override + public boolean removeAll(Collection c) { + return delegate.removeAll(c); + } + + @Override + public boolean retainAll(Collection c) { + return delegate.retainAll(c); + } + + @Override + public void clear() { + delegate.clear(); + } + + @Override + public E set(int index, E element) { + throw new RuntimeException("Not implememnted yet!"); + } + + @Override + public void add(int index, E element) { + throw new RuntimeException("Not implememnted yet!"); + } + + @Override + public E remove(int index) { + return delegate.remove(index); + } + + @Override + public int indexOf(Object o) { + return delegate.indexOf(o); + } + + @Override + public int lastIndexOf(Object o) { + return delegate.lastIndexOf(o); + } + + @Override + public ListIterator listIterator() { + return delegate.listIterator(); + } + + @Override + public ListIterator listIterator(int index) { + return delegate.listIterator(index); + } + + @Override + public List subList(int fromIndex, int toIndex) { + ConsumerList res = create(maxSize, spillOutConsumer); + res.addAll(delegate.subList(fromIndex, toIndex)); + return res; } }