Spec v0.8.3

.circleci/config.yml

@@ -25,6 +25,7 @@ jobs:
             sudo sh -c "echo 'deb http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list"
             curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
             sudo apt-get update && sudo apt-get install -y git-lfs
+
       - run:
           name: Downloading dependencies
           command: ./gradlew allDependencies
@@ -46,6 +47,19 @@ jobs:
           destination: jars
           when: always
 
+      - run:
+          name: Save commit of tests repo in env var
+          command: eval cd $(echo $CIRCLE_WORKING_DIRECTORY)/test/src/test/resources/ && git submodule status eth2.0-spec-tests/ |  awk '{print $1;}' > /tmp/tests-version
+
+      - run:
+          name: Verify tests repo commit is saved
+          command: cat /tmp/tests-version
+
+      - restore_cache:
+          name: Restoring spec tests repo
+          keys:
+            - v1-tests-dir-{{ checksum "/tmp/tests-version" }}
+
       - run:
           name: Running tests
           command: ./gradlew --info clean cleanTest test --continue
@@ -80,6 +94,12 @@ jobs:
             done
           when: always
 
+      - save_cache:
+          name: Caching spec tests repo
+          paths:
+            - ~/beacon-chain/test/src/test/resources/eth2.0-spec-tests
+          key: v1-tests-dir-{{ checksum "/tmp/tests-version" }}
+
       - store_artifacts:
           name: Uploading reports
           path: ~/reports

consensus/src/main/java/org/ethereum/beacon/consensus/spec/BlockProcessing.java

@@ -248,6 +248,15 @@ assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)
       return false;
     }
 
+    /* committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard)
+    assert len(attestation.aggregation_bits) == len(attestation.custody_bits) == len(committee) */
+    List<ValidatorIndex> committee =
+        get_crosslink_committee(state, data.getTarget().getEpoch(), data.getCrosslink().getShard());
+    if (attestation.getAggregationBits().size() != attestation.getCustodyBits().size()
+        || attestation.getAggregationBits().size() != committee.size()) {
+      return false;
+    }
+
     /* if data.target.epoch == get_current_epoch(state):
          assert data.source == state.current_justified_checkpoint
        else:

consensus/src/main/java/org/ethereum/beacon/consensus/spec/EpochProcessing.java

@@ -1,5 +1,12 @@
 package org.ethereum.beacon.consensus.spec;
 
+import static java.util.stream.Collectors.toList;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
 import org.ethereum.beacon.core.BeaconState;
 import org.ethereum.beacon.core.MutableBeaconState;
 import org.ethereum.beacon.core.operations.attestation.Crosslink;
@@ -19,14 +26,6 @@
 import tech.pegasys.artemis.util.uint.UInt64;
 import tech.pegasys.artemis.util.uint.UInt64s;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import static java.util.stream.Collectors.toList;
-
 /**
  * Epoch processing part.
  *
@@ -80,7 +79,7 @@ def get_unslashed_attesting_indices(state: BeaconState,
       output = set()  # type: Set[ValidatorIndex]
       for a in attestations:
           output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits))
-      return set(filter(lambda index: not state.validators[index].slashed, list(output)))
+      return set(filter(lambda index: not state.validators[index].slashed, output))
    */
   default List<ValidatorIndex> get_unslashed_attesting_indices(BeaconState state, List<PendingAttestation> attestations) {
     return attestations.stream()
@@ -103,10 +102,10 @@ def get_winning_crosslink_and_attesting_indices(state: BeaconState,
                                                 epoch: Epoch,
                                                 shard: Shard) -> Tuple[Crosslink, List[ValidatorIndex]]:
       attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard]
-      crosslinks = list(filter(
+      crosslinks = filter(
           lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.parent_root, hash_tree_root(c)),
           [a.data.crosslink for a in attestations]
-      ))
+      )
       # Winning crosslink has the crosslink data root with the most balance voting for it (ties broken lexicographically)
       winning_crosslink = max(crosslinks, key=lambda c: (
           get_attesting_balance(state, [a for a in attestations if a.data.crosslink == c]), c.data_root
@@ -596,11 +595,6 @@ default void process_final_updates(MutableBeaconState state) {
       }
     }
 
-    /* Update start shard
-      state.latest_start_shard = (state.latest_start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT */
-    state.setStartShard(state.getStartShard()
-        .plusModulo(get_shard_delta(state, current_epoch), getConstants().getShardCount()));
-
     /* # Set active index root
       index_epoch = Epoch(next_epoch + ACTIVATION_EXIT_DELAY)
       index_root_position = index_epoch % EPOCHS_PER_HISTORICAL_VECTOR
@@ -637,6 +631,12 @@ default void process_final_updates(MutableBeaconState state) {
       state.getHistoricalRoots().add(hash_tree_root(historical_batch));
     }
 
+    /* # Update start shard
+    state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT) */
+    state.setStartShard(state
+            .getStartShard()
+            .plusModulo(get_shard_delta(state, current_epoch), getConstants().getShardCount()));
+
     /* # Rotate current/previous epoch attestations
       state.previous_epoch_attestations = state.current_epoch_attestations
       state.current_epoch_attestations = [] */

consensus/src/main/java/org/ethereum/beacon/consensus/spec/ForkChoice.java

@@ -24,21 +24,27 @@ public interface ForkChoice extends HelperFunction {
   /*
     def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash:
       block = store.blocks[root]
-      assert block.slot >= slot
-      return root if block.slot == slot else get_ancestor(store, block.parent_root, slot)
+      if block.slot > slot:
+          return get_ancestor(store, block.parent_root, slot)
+      elif block.slot == slot:
+          return root
+      else:
+          return Bytes32()  # root is older than queried slot: no results.
    */
   default Optional<Hash32> get_ancestor(Store store, Hash32 root, SlotNumber slot) {
     Optional<BeaconBlock> aBlock = store.getBlock(root);
     if (!aBlock.isPresent()) {
       return Optional.empty();
     }
-    if (aBlock.get().getSlot().less(slot)) {
-      return Optional.empty();
-    }
 
     BeaconBlock block = aBlock.get();
-    return block.getSlot().equals(slot) ?
-        Optional.of(root) : get_ancestor(store, block.getParentRoot(), slot);
+    if (block.getSlot().greater(slot)) {
+      return get_ancestor(store, block.getParentRoot(), slot);
+    } else if (block.getSlot().equals(slot)) {
+      return Optional.of(root);
+    } else {
+      return Optional.of(Hash32.ZERO);
+    }
   }
 
   /*

crypto/src/main/java/org/ethereum/beacon/crypto/MessageParameters.java

@@ -1,9 +1,7 @@
 package org.ethereum.beacon.crypto;
 
 import tech.pegasys.artemis.ethereum.core.Hash32;
-import tech.pegasys.artemis.util.bytes.Bytes32;
 import tech.pegasys.artemis.util.bytes.Bytes8;
-import tech.pegasys.artemis.util.bytes.BytesValue;
 import tech.pegasys.artemis.util.uint.UInt64;
 
 /**
@@ -17,6 +15,14 @@
  */
 public interface MessageParameters {
 
+  static MessageParameters create(Hash32 hash, Bytes8 domain) {
+    return new Impl(hash, domain);
+  }
+
+  static MessageParameters create(Hash32 hash, UInt64 domain) {
+    return new Impl(hash, domain.toBytes8LittleEndian());
+  }
+
   /**
    * Returns a hash of the message.
    *
@@ -31,14 +37,6 @@ public interface MessageParameters {
    */
   Bytes8 getDomain();
 
-  static MessageParameters create(Hash32 hash, Bytes8 domain) {
-    return new Impl(hash, domain);
-  }
-
-  static MessageParameters create(Hash32 hash, UInt64 domain) {
-    return new Impl(hash, domain.toBytesBigEndian());
-  }
-
   /** A straightforward implementation of {@link MessageParameters}. */
   class Impl implements MessageParameters {
     private final Hash32 hash;

ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntCodec.java

@@ -1,16 +1,20 @@
 package org.ethereum.beacon.ssz.access.basic;
 
 import net.consensys.cava.bytes.Bytes;
-import org.ethereum.beacon.ssz.visitor.SSZReader;
-import org.ethereum.beacon.ssz.visitor.SSZWriter;
 import net.consensys.cava.ssz.SSZException;
+import org.ethereum.beacon.ssz.SSZSchemeException;
 import org.ethereum.beacon.ssz.access.SSZBasicAccessor;
 import org.ethereum.beacon.ssz.access.SSZField;
-import org.ethereum.beacon.ssz.SSZSchemeException;
+import org.ethereum.beacon.ssz.visitor.SSZReader;
+import org.ethereum.beacon.ssz.visitor.SSZWriter;
 import tech.pegasys.artemis.util.bytes.BytesValue;
+import tech.pegasys.artemis.util.uint.UInt16;
 import tech.pegasys.artemis.util.uint.UInt24;
 import tech.pegasys.artemis.util.uint.UInt256;
+import tech.pegasys.artemis.util.uint.UInt32;
 import tech.pegasys.artemis.util.uint.UInt64;
+import tech.pegasys.artemis.util.uint.UInt8;
+
 import java.io.IOException;
 import java.io.OutputStream;
 import java.util.HashMap;
@@ -32,7 +36,10 @@ public class UIntCodec implements SSZBasicAccessor {
   private static Set<Class> supportedClassTypes = new HashSet<>();
 
   static {
+    classToNumericType.put(UInt8.class, NumericType.of(Type.LONG, 8));
+    classToNumericType.put(UInt16.class, NumericType.of(Type.LONG, 16));
     classToNumericType.put(UInt24.class, NumericType.of(Type.LONG, 24));
+    classToNumericType.put(UInt32.class, NumericType.of(Type.LONG, 32));
     classToNumericType.put(UInt64.class, NumericType.of(Type.LONG, 64));
     classToNumericType.put(UInt256.class, NumericType.of(Type.BIGINT, 256));
   }
@@ -41,7 +48,10 @@ public class UIntCodec implements SSZBasicAccessor {
   }
 
   static {
+    supportedClassTypes.add(UInt8.class);
+    supportedClassTypes.add(UInt16.class);
     supportedClassTypes.add(UInt24.class);
+    supportedClassTypes.add(UInt32.class);
     supportedClassTypes.add(UInt64.class);
     supportedClassTypes.add(UInt256.class);
   }
@@ -82,13 +92,34 @@ public void encode(Object value, SSZField field, OutputStream result) {
     NumericType numericType = parseFieldType(field);
 
     switch (numericType.size) {
+      case 8:
+        {
+          UInt8 uValue = (UInt8) value;
+          Bytes bytes = SSZWriter.encodeULong(uValue.getValue(), numericType.size);
+          writeBytes(bytes, result);
+          break;
+        }
+      case 16:
+        {
+          UInt16 uValue = (UInt16) value;
+          Bytes bytes = SSZWriter.encodeULong(uValue.getValue(), numericType.size);
+          writeBytes(bytes, result);
+          break;
+        }
       case 24:
         {
           UInt24 uValue = (UInt24) value;
           Bytes bytes = SSZWriter.encodeULong(uValue.getValue(), numericType.size);
           writeBytes(bytes, result);
           break;
         }
+      case 32:
+        {
+          UInt32 uValue = (UInt32) value;
+          Bytes bytes = SSZWriter.encodeULong(uValue.getValue(), numericType.size);
+          writeBytes(bytes, result);
+          break;
+        }
       case 64:
         {
           UInt64 uValue = (UInt64) value;
@@ -129,10 +160,22 @@ public Object decode(SSZField field, SSZReader reader) {
   private Object decodeLong(NumericType type, SSZReader reader) {
     // XXX: reader.readULong is buggy
     switch (type.size) {
+      case 8:
+        {
+          return UInt8.valueOf(reader.readUnsignedBigInteger(type.size).intValue());
+        }
+      case 16:
+        {
+          return UInt16.valueOf(reader.readUnsignedBigInteger(type.size).intValue());
+        }
       case 24:
         {
           return UInt24.valueOf(reader.readUnsignedBigInteger(type.size).intValue());
         }
+      case 32:
+        {
+          return UInt32.valueOf(reader.readUnsignedBigInteger(type.size).intValue());
+        }
       case 64:
         {
           return UInt64.valueOf(reader.readUnsignedBigInteger(type.size).longValue());

ssz/src/main/java/org/ethereum/beacon/ssz/access/list/ReadListAccessor.java

@@ -3,6 +3,7 @@
 import org.ethereum.beacon.ssz.SSZSerializeException;
 import org.ethereum.beacon.ssz.access.SSZField;
 import org.ethereum.beacon.ssz.type.SSZType;
+import org.ethereum.beacon.ssz.type.list.SSZListType;
 import tech.pegasys.artemis.util.collections.ReadList;
 import tech.pegasys.artemis.util.collections.ReadVector;
 
@@ -54,23 +55,26 @@ public ListInstanceBuilder createInstanceBuilder(SSZType sszType) {
     return new SimpleInstanceBuilder() {
       @Override
       protected Object buildImpl(List<Object> children) {
-        return sszType.isFixedSize()
+        SSZListType listType = (SSZListType) sszType;
+        return listType.isVector()
             ? ReadVector.wrap(
                 children,
                 resolveIndexConverter(
                     (Class<?>)
                         sszType
                             .getTypeDescriptor()
                             .getParametrizedType()
-                            .getActualTypeArguments()[0]))
+                            .getActualTypeArguments()[0]),
+                listType.getVectorLength())
             : ReadList.wrap(
                 children,
                 resolveIndexConverter(
                     (Class<?>)
                         sszType
                             .getTypeDescriptor()
                             .getParametrizedType()
-                            .getActualTypeArguments()[0]));
+                            .getActualTypeArguments()[0]),
+                listType.getMaxSize());
       }
     };
   }

ssz/src/main/java/org/ethereum/beacon/ssz/creator/SettersObjCreator.java

@@ -66,9 +66,13 @@ public <C> C createObject(Class<? extends C> clazz,
         clazz.getField(currentField.getName()).set(result, values[i]);
       } catch (Exception e) {
         try { // Try to set using setter
-          fieldSetters.get(currentField.getName()).invoke(result, values[i]);
+          Method setter = fieldSetters.get(currentField.getName());
+          if (setter == null && currentField.getName().length() == 1) { // Bug in inspector with single letter fields
+            setter = fieldSetters.get(currentField.getName().toLowerCase());
+          }
+          setter.invoke(result, values[i]);
         } catch (Exception ex) { // Cannot set the field
-          throw new SSZSchemeException(String.format("Setter net found for field %s", currentField.getName()), ex);
+          throw new SSZSchemeException(String.format("Setter not found for field %s", currentField.getName()), ex);
         }
       }
     }

ssz/src/main/java/org/ethereum/beacon/ssz/type/list/SSZListType.java

@@ -63,6 +63,10 @@ public int getSize() {
     return getElementType().getSize() * getVectorLength();
   }
 
+  public boolean isVector() {
+    return getVectorLength() > VARIABLE_SIZE;
+  }
+
   public long getMaxSize() {
     return maxSize;
   }

test/build.gradle

@@ -30,6 +30,7 @@ task activateLfs(type:Exec) {
 task submodulesUpdate(type:Exec) {
   description 'Updates (and inits) git submodules'
   dependsOn(activateLfs)
+  ignoreExitValue true // CI hack for cache use
   commandLine 'git', 'submodule', 'update', '--init', '--recursive'
   group 'Build Setup'
 }