Refactor SSZ, implement incremental hash

.circleci/config.yml

@@ -7,7 +7,7 @@ jobs:
     working_directory: ~/beacon-chain
 
     environment:
-      JAVA_TOOL_OPTIONS: -Xmx768m
+      JAVA_TOOL_OPTIONS: -Xmx2G
       GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=2
       GRADLE_MAX_TEST_FORKS: 2
 

chain/src/main/java/org/ethereum/beacon/chain/DefaultBeaconChain.java

@@ -87,7 +87,8 @@ private BeaconTuple fetchRecentTuple() {
 
   private void initializeStorage() {
     BeaconBlock initialGenesis = spec.get_empty_block();
-    BeaconStateEx initialState = initialTransition.apply(BeaconStateEx.getEmpty(), initialGenesis);
+    BeaconStateEx initialState =
+        initialTransition.apply(BeaconStateEx.getEmpty(spec.getConstants()), initialGenesis);
 
     Hash32 initialStateRoot = spec.hash_tree_root(initialState);
     BeaconBlock genesis = initialGenesis.withStateRoot(initialStateRoot);

chain/src/main/java/org/ethereum/beacon/chain/storage/BeaconChainStorageFactory.java

@@ -7,8 +7,4 @@
 public interface BeaconChainStorageFactory {
 
   BeaconChainStorage create(Database database);
-
-  static BeaconChainStorageFactory get() {
-    return new SSZBeaconChainStorageFactory();
-  }
 }

chain/src/main/java/org/ethereum/beacon/chain/storage/impl/MemBeaconChainStorageFactory.java

@@ -13,11 +13,14 @@
 import tech.pegasys.artemis.ethereum.core.Hash32;
 
 public class MemBeaconChainStorageFactory  implements BeaconChainStorageFactory {
+  private final ObjectHasher<Hash32> objectHasher;
+
+  public MemBeaconChainStorageFactory(ObjectHasher<Hash32> objectHasher) {
+    this.objectHasher = objectHasher;
+  }
 
   @Override
   public BeaconChainStorage create(Database database) {
-    ObjectHasher<Hash32> objectHasher = ObjectHasher.createSSZOverKeccak256();
-
     BeaconBlockStorage blockStorage =
         new BeaconBlockStorageImpl(objectHasher, new HashMapDataSource<>(), new HashMapHoleyList<>());
     BeaconStateStorage stateStorage =

chain/src/main/java/org/ethereum/beacon/chain/storage/impl/SSZBeaconChainStorageFactory.java

@@ -19,12 +19,18 @@
  * BeaconChainStorage} instance.
  */
 public class SSZBeaconChainStorageFactory implements BeaconChainStorageFactory {
+  private final ObjectHasher<Hash32> objectHasher;
+  private final SerializerFactory serializerFactory;
+
+  public SSZBeaconChainStorageFactory(
+      ObjectHasher<Hash32> objectHasher,
+      SerializerFactory serializerFactory) {
+    this.objectHasher = objectHasher;
+    this.serializerFactory = serializerFactory;
+  }
 
   @Override
   public BeaconChainStorage create(Database database) {
-    ObjectHasher<Hash32> objectHasher = ObjectHasher.createSSZOverKeccak256();
-    SerializerFactory serializerFactory = SerializerFactory.createSSZ();
-
     BeaconBlockStorage blockStorage =
         BeaconBlockStorageImpl.create(database, objectHasher, serializerFactory);
     BeaconStateStorage stateStorage =

chain/src/main/java/org/ethereum/beacon/chain/storage/impl/SSZSerializerFactory.java

@@ -1,14 +1,14 @@
 package org.ethereum.beacon.chain.storage.impl;
 
 import java.util.function.Function;
-import org.ethereum.beacon.ssz.Serializer;
+import org.ethereum.beacon.ssz.SSZSerializer;
 import tech.pegasys.artemis.util.bytes.BytesValue;
 
 public class SSZSerializerFactory implements SerializerFactory {
 
-  private final Serializer serializer;
+  private final SSZSerializer serializer;
 
-  public SSZSerializerFactory(Serializer serializer) {
+  public SSZSerializerFactory(SSZSerializer serializer) {
     this.serializer = serializer;
   }
 

chain/src/main/java/org/ethereum/beacon/chain/storage/impl/SerializerFactory.java

@@ -1,7 +1,9 @@
 package org.ethereum.beacon.chain.storage.impl;
 
 import java.util.function.Function;
-import org.ethereum.beacon.ssz.Serializer;
+import org.ethereum.beacon.core.spec.SpecConstants;
+import org.ethereum.beacon.core.spec.SpecConstantsResolver;
+import org.ethereum.beacon.ssz.SSZBuilder;
 import tech.pegasys.artemis.util.bytes.BytesValue;
 
 public interface SerializerFactory {
@@ -10,7 +12,10 @@ public interface SerializerFactory {
 
   <T> Function<T, BytesValue> getSerializer(Class<T> objectClass);
 
-  static SerializerFactory createSSZ() {
-    return new SSZSerializerFactory(Serializer.annotationSerializer());
+  static SerializerFactory createSSZ(SpecConstants specConstants) {
+    return new SSZSerializerFactory(
+        new SSZBuilder()
+            .withExternalVarResolver(new SpecConstantsResolver(specConstants))
+            .buildSerializer());
   }
 }

chain/src/test/java/org/ethereum/beacon/chain/DefaultBeaconChainTest.java

@@ -4,6 +4,8 @@
 import java.util.stream.IntStream;
 import org.ethereum.beacon.chain.storage.BeaconChainStorage;
 import org.ethereum.beacon.chain.storage.BeaconChainStorageFactory;
+import org.ethereum.beacon.chain.storage.impl.SSZBeaconChainStorageFactory;
+import org.ethereum.beacon.chain.storage.impl.SerializerFactory;
 import org.ethereum.beacon.consensus.BeaconChainSpec;
 import org.ethereum.beacon.consensus.BeaconStateEx;
 import org.ethereum.beacon.consensus.BlockTransition;
@@ -95,7 +97,9 @@ private MutableBeaconChain createBeaconChain(
     BeaconBlockVerifier blockVerifier = (block, state) -> VerificationResult.PASSED;
     BeaconStateVerifier stateVerifier = (block, state) -> VerificationResult.PASSED;
     Database database = Database.inMemoryDB();
-    BeaconChainStorage chainStorage = BeaconChainStorageFactory.get().create(database);
+    BeaconChainStorage chainStorage = new SSZBeaconChainStorageFactory(
+            spec.getObjectHasher(), SerializerFactory.createSSZ(spec.getConstants()))
+            .create(database);
 
     return new DefaultBeaconChain(
         spec,

chain/src/test/java/org/ethereum/beacon/chain/storage/BeaconBlockStorageTest.java

@@ -21,7 +21,7 @@ private BeaconBlockStorage create(BeaconChainSpec spec) {
     return BeaconBlockStorageImpl.create(
         Database.inMemoryDB(),
         ObjectHasher.createSSZOverKeccak256(),
-        SerializerFactory.createSSZ());
+        SerializerFactory.createSSZ(spec.getConstants()));
   }
 
   private BeaconBlock createBlock(long slot, BeaconBlock parent, Hash32 parentHash) {

chain/src/test/java/org/ethereum/beacon/chain/util/SampleObservableState.java

@@ -9,7 +9,8 @@
 import org.ethereum.beacon.chain.observer.ObservableStateProcessor;
 import org.ethereum.beacon.chain.observer.ObservableStateProcessorImpl;
 import org.ethereum.beacon.chain.storage.BeaconChainStorage;
-import org.ethereum.beacon.chain.storage.BeaconChainStorageFactory;
+import org.ethereum.beacon.chain.storage.impl.SSZBeaconChainStorageFactory;
+import org.ethereum.beacon.chain.storage.impl.SerializerFactory;
 import org.ethereum.beacon.consensus.BeaconChainSpec;
 import org.ethereum.beacon.consensus.StateTransitions;
 import org.ethereum.beacon.consensus.TestUtils;
@@ -92,7 +93,10 @@ public SlotNumber getGenesisSlot() {
     PerBlockTransition blockTransition = StateTransitions.blockTransition(spec);
 
     db = new InMemoryDatabase();
-    beaconChainStorage = BeaconChainStorageFactory.get().create(db);
+    beaconChainStorage =
+        new SSZBeaconChainStorageFactory(
+                spec.getObjectHasher(), SerializerFactory.createSSZ(specConstants))
+            .create(db);
 
     // TODO
     BeaconBlockVerifier blockVerifier = (block, state) -> VerificationResult.PASSED;

consensus/src/main/java/org/ethereum/beacon/consensus/BeaconChainSpec.java

@@ -52,7 +52,7 @@ static BeaconChainSpec createWithSSZHasher(@Nonnull SpecConstants constants) {
     Objects.requireNonNull(constants);
 
     Function<BytesValue, Hash32> hashFunction = Hashes::keccak256;
-    ObjectHasher<Hash32> sszHasher = SSZObjectHasher.create(hashFunction);
+    ObjectHasher<Hash32> sszHasher = SSZObjectHasher.create(constants, hashFunction);
     return new BeaconChainSpecImpl(constants, hashFunction, sszHasher);
   }
 

consensus/src/main/java/org/ethereum/beacon/consensus/BeaconStateEx.java

@@ -16,7 +16,12 @@ static BeaconStateEx getEmpty() {
     return new BeaconStateExImpl(BeaconState.getEmpty(), Hash32.ZERO, TransitionType.UNKNOWN);
   }
 
-  Hash32 getHeadBlockHash();
+  static BeaconStateEx getEmpty(SpecConstants specConst) {
+    return new BeaconStateExImpl(BeaconState.getEmpty(specConst), Hash32.ZERO, TransitionType.UNKNOWN);
+  }
+
+
+    Hash32 getHeadBlockHash();
 
   TransitionType getTransition();
 

consensus/src/main/java/org/ethereum/beacon/consensus/hasher/ObjectHasher.java

@@ -1,5 +1,6 @@
 package org.ethereum.beacon.consensus.hasher;
 
+import org.ethereum.beacon.core.spec.SpecConstants;
 import org.ethereum.beacon.crypto.Hashes;
 import tech.pegasys.artemis.ethereum.core.Hash32;
 import tech.pegasys.artemis.util.bytes.BytesValue;
@@ -16,6 +17,10 @@ static ObjectHasher<Hash32> createSSZOverKeccak256() {
     return SSZObjectHasher.create(Hashes::keccak256);
   }
 
+  static ObjectHasher<Hash32> createSSZOverKeccak256(SpecConstants specConstants) {
+    return SSZObjectHasher.create(specConstants, Hashes::keccak256);
+  }
+
   /**
    * Calculates hash of given object.
    *

consensus/src/main/java/org/ethereum/beacon/consensus/hasher/SSZObjectHasher.java

@@ -1,17 +1,16 @@
 package org.ethereum.beacon.consensus.hasher;
 
-import java.util.function.Consumer;
+import java.util.List;
+import java.util.function.Function;
+import org.ethereum.beacon.core.spec.SpecConstants;
+import org.ethereum.beacon.core.spec.SpecConstantsResolver;
 import org.ethereum.beacon.core.types.Hashable;
-import org.ethereum.beacon.ssz.SSZHashSerializer;
-import org.ethereum.beacon.ssz.SSZHashSerializers;
+import org.ethereum.beacon.ssz.SSZBuilder;
+import org.ethereum.beacon.ssz.SSZHasher;
 import tech.pegasys.artemis.ethereum.core.Hash32;
 import tech.pegasys.artemis.util.bytes.Bytes32;
 import tech.pegasys.artemis.util.bytes.BytesValue;
 
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Function;
-
 /**
  * An object hasher implementation using Tree Hash algorithm described in the spec.
  *
@@ -21,22 +20,35 @@
  */
 public class SSZObjectHasher implements ObjectHasher<Hash32> {
 
-  private static final int SSZ_SCHEMES_CACHE_CAPACITY = 128;
-  private final SSZHashSerializer sszHashSerializer;
+  private final SSZHasher sszHasher;
 
-  SSZObjectHasher(SSZHashSerializer sszHashSerializer) {
-    this.sszHashSerializer = sszHashSerializer;
+  SSZObjectHasher(SSZHasher sszHasher) {
+    this.sszHasher = sszHasher;
+  }
+
+  public static SSZObjectHasher create(
+      SpecConstants constants, Function<BytesValue, Hash32> hashFunction) {
+
+    SSZHasher sszHasher =
+        new SSZBuilder()
+            .withExternalVarResolver(new SpecConstantsResolver(constants))
+            .withIncrementalHasher(true)
+            .buildHasher(hashFunction);
+    return new SSZObjectHasher(sszHasher);
   }
 
   public static SSZObjectHasher create(Function<BytesValue, Hash32> hashFunction) {
-    SSZHashSerializer sszHashSerializer =
-        SSZHashSerializers.createWithBeaconChainTypes(hashFunction, true, SSZ_SCHEMES_CACHE_CAPACITY);
-    return new SSZObjectHasher(sszHashSerializer);
+
+    SSZHasher sszHasher =
+        new SSZBuilder()
+            .withIncrementalHasher(true)
+            .buildHasher(hashFunction);
+    return new SSZObjectHasher(sszHasher);
   }
 
   @Override
   public Hash32 getHash(Object input) {
-    Function<Object, Hash32> hasher = o -> Hash32.wrap(Bytes32.wrap(sszHashSerializer.hash(o)));
+    Function<Object, Hash32> hasher = o -> Hash32.wrap(Bytes32.wrap(sszHasher.hash(o)));
     if (input instanceof Hashable) {
       return ((Hashable<Hash32>) input).getHash(hasher);
     } else {
@@ -49,7 +61,7 @@ public Hash32 getHashTruncateLast(Object input) {
     if (input instanceof List) {
       throw new RuntimeException("Lists are not supported in truncated hash");
     } else {
-      return Hash32.wrap(Bytes32.wrap(sszHashSerializer.hashTruncateLast(input, input.getClass())));
+      return Hash32.wrap(Bytes32.wrap(sszHasher.hashTruncateLast(input, input.getClass())));
     }
   }
 }

consensus/src/main/java/org/ethereum/beacon/consensus/spec/OnGenesis.java

@@ -56,7 +56,7 @@ default BeaconBlock get_empty_block() {
   */
   default BeaconState get_genesis_beacon_state(
       List<Deposit> genesisValidatorDeposits, Time genesisTime, Eth1Data genesisEth1Data) {
-    MutableBeaconState state = BeaconState.getEmpty().createMutableCopy();
+    MutableBeaconState state = BeaconState.getEmpty(getConstants()).createMutableCopy();
 
     // Misc
     state.setSlot(getConstants().getGenesisSlot());
@@ -72,8 +72,7 @@ default BeaconState get_genesis_beacon_state(
     state.setValidatorRegistryUpdateEpoch(getConstants().getGenesisEpoch());
 
     // Randomness and committees
-    state.getLatestRandaoMixes().addAll(
-        nCopies(getConstants().getLatestRandaoMixesLength().getIntValue(), Hash32.ZERO));
+    state.getLatestRandaoMixes().setAll(Hash32.ZERO);
     state.setPreviousShufflingStartShard(getConstants().getGenesisStartShard());
     state.setCurrentShufflingStartShard(getConstants().getGenesisStartShard());
     state.setPreviousShufflingEpoch(getConstants().getGenesisEpoch());
@@ -99,14 +98,10 @@ default BeaconState get_genesis_beacon_state(
     state.getCurrentCrosslinks().addAll(
         nCopies(getConstants().getShardCount().getIntValue(),
             new Crosslink(getConstants().getGenesisEpoch(), Hash32.ZERO)));
-    state.getLatestBlockRoots().addAll(
-        nCopies(getConstants().getSlotsPerHistoricalRoot().getIntValue(), Hash32.ZERO));
-    state.getLatestStateRoots().addAll(
-        nCopies(getConstants().getSlotsPerHistoricalRoot().getIntValue(), Hash32.ZERO));
-    state.getLatestActiveIndexRoots().addAll(
-        nCopies(getConstants().getLatestActiveIndexRootsLength().getIntValue(), Hash32.ZERO));
-    state.getLatestSlashedBalances().addAll(
-        nCopies(getConstants().getLatestSlashedExitLength().getIntValue(), Gwei.ZERO));
+    state.getLatestBlockRoots().setAll(Hash32.ZERO);
+    state.getLatestStateRoots().setAll(Hash32.ZERO);
+    state.getLatestActiveIndexRoots().setAll(Hash32.ZERO);
+    state.getLatestSlashedBalances().setAll(Gwei.ZERO);
     state.setLatestBlockHeader(get_temporary_block_header(get_empty_block()));
     state.getHistoricalRoots().clear();
 

consensus/src/main/java/org/ethereum/beacon/consensus/transition/DelegateBeaconState.java

@@ -1,5 +1,7 @@
 package org.ethereum.beacon.consensus.transition;
 
+import java.util.Map;
+import java.util.function.Supplier;
 import javax.annotation.Nullable;
 import org.ethereum.beacon.core.BeaconBlockHeader;
 import org.ethereum.beacon.core.BeaconState;
@@ -18,8 +20,10 @@
 import org.ethereum.beacon.core.types.SlotNumber;
 import org.ethereum.beacon.core.types.Time;
 import org.ethereum.beacon.core.types.ValidatorIndex;
+import org.ethereum.beacon.ssz.incremental.UpdateListener;
 import tech.pegasys.artemis.ethereum.core.Hash32;
 import tech.pegasys.artemis.util.collections.ReadList;
+import tech.pegasys.artemis.util.collections.ReadVector;
 import tech.pegasys.artemis.util.uint.UInt64;
 
 public class DelegateBeaconState implements BeaconState {
@@ -29,14 +33,21 @@ public DelegateBeaconState(BeaconState delegate) {
     this.delegate = delegate;
   }
 
-  public static BeaconState getEmpty() {
-    return BeaconState.getEmpty();
-  }
-
   public BeaconState getDelegate() {
     return delegate;
   }
 
+  @Override
+  public UpdateListener getUpdateListener(String observerId,
+      Supplier<UpdateListener> listenerFactory) {
+    return delegate.getUpdateListener(observerId, listenerFactory);
+  }
+
+  @Override
+  public Map<String, UpdateListener> getAllUpdateListeners() {
+    return delegate.getAllUpdateListeners();
+  }
+
   @Override
   public SlotNumber getSlot() {
     return delegate.getSlot();
@@ -68,7 +79,7 @@ public EpochNumber getValidatorRegistryUpdateEpoch() {
   }
 
   @Override
-  public ReadList<EpochNumber, Hash32> getLatestRandaoMixes() {
+  public ReadVector<EpochNumber, Hash32> getLatestRandaoMixes() {
     return delegate.getLatestRandaoMixes();
   }
 
@@ -158,22 +169,22 @@ public ReadList<ShardNumber, Crosslink> getCurrentCrosslinks() {
   }
 
   @Override
-  public ReadList<SlotNumber, Hash32> getLatestBlockRoots() {
+  public ReadVector<SlotNumber, Hash32> getLatestBlockRoots() {
     return delegate.getLatestBlockRoots();
   }
 
   @Override
-  public ReadList<SlotNumber, Hash32> getLatestStateRoots() {
+  public ReadVector<SlotNumber, Hash32> getLatestStateRoots() {
     return delegate.getLatestStateRoots();
   }
 
   @Override
-  public ReadList<EpochNumber, Hash32> getLatestActiveIndexRoots() {
+  public ReadVector<EpochNumber, Hash32> getLatestActiveIndexRoots() {
     return delegate.getLatestActiveIndexRoots();
   }
 
   @Override
-  public ReadList<EpochNumber, Gwei> getLatestSlashedBalances() {
+  public ReadVector<EpochNumber, Gwei> getLatestSlashedBalances() {
     return delegate.getLatestSlashedBalances();
   }