From 71cc7065f8403f1109a62e5912531a4aae772cb2 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 13 May 2019 18:27:03 +0300 Subject: [PATCH 01/15] ssz: remove encodeList and decodeList specific methods --- .../beacon/ssz/access/SSZBasicAccessor.java | 20 --- .../ssz/access/basic/BooleanPrimitive.java | 23 --- .../beacon/ssz/access/basic/BytesCodec.java | 102 ------------ .../ssz/access/basic/BytesPrimitive.java | 157 ++---------------- .../beacon/ssz/access/basic/HashCodec.java | 55 ------ .../ssz/access/basic/StringPrimitive.java | 20 --- .../ssz/access/basic/SubclassCodec.java | 21 --- .../beacon/ssz/access/basic/UIntCodec.java | 132 --------------- .../ssz/access/basic/UIntPrimitive.java | 84 ---------- .../ethereum/beacon/ssz/annotation/SSZ.java | 4 +- .../ssz/visitor/SSZSimpleDeserializer.java | 10 +- .../beacon/ssz/SSZSerializerTest.java | 50 +++++- .../ssz/fixtures/AttestationRecord.java | 2 +- 13 files changed, 66 insertions(+), 614 deletions(-) diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/SSZBasicAccessor.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/SSZBasicAccessor.java index 6d54423da..53710d4c4 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/SSZBasicAccessor.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/SSZBasicAccessor.java @@ -44,16 +44,6 @@ public interface SSZBasicAccessor { */ void encode(Object value, SSZField field, OutputStream result); - /** - * Encodes list field as SSZ type and writes it to output stream - * - * @param value Field value - * @param field Field type - * @param result Output stream - */ - void encodeList( - List value, SSZField field, OutputStream result); - /** * Decodes SSZ encoded data and returns result * @@ -64,16 +54,6 @@ void encodeList( */ Object decode(SSZField field, BytesSSZReaderProxy reader); - /** - * Decodes SSZ encoded data and returns result - * - * @param field Type of field to read at this point, list - * @param reader Reader which holds SSZ encoded data at the appropriate point. Pointer will be - * moved to the end of this field/beginning of next one after reading is performed. - * @return field list value - */ - List decodeList(SSZField field, BytesSSZReaderProxy reader); - /** * Helper designed to throw usual error * diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BooleanPrimitive.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BooleanPrimitive.java index a4970f498..93c9965e0 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BooleanPrimitive.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BooleanPrimitive.java @@ -7,7 +7,6 @@ import java.io.IOException; import java.io.OutputStream; import java.util.HashSet; -import java.util.List; import java.util.Set; import org.ethereum.beacon.ssz.access.SSZField; import org.ethereum.beacon.ssz.access.SSZBasicAccessor; @@ -56,30 +55,8 @@ public void encode(Object value, SSZField field, OutputStream result) { } } - @Override - public void encodeList( - List value, SSZField field, OutputStream result) { - try { - boolean[] data = new boolean[value.size()]; - for (int i = 0; i < value.size(); ++i) { - data[i] = (boolean) value.get(i); - } - result.write(SSZ.encodeBooleanList(data).toArrayUnsafe()); - } catch (IOException ex) { - String error = String.format("Failed to write data from field \"%s\" to stream", - field.getName()); - throw new SSZException(error, ex); - } - } - @Override public Object decode(SSZField field, BytesSSZReaderProxy reader) { return reader.readBoolean(); } - - @Override - public List decodeList( - SSZField field, BytesSSZReaderProxy reader) { - return (List) (List) reader.readBooleanList(); - } } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesCodec.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesCodec.java index 4c4115efb..177391795 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesCodec.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesCodec.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; /** * SSZ Codec designed to work with fixed size bytes data classes, check list in {@link @@ -98,27 +97,6 @@ public void encode(Object value, SSZField field, OutputStream result) { } } - @Override - public void encodeList( - List value, SSZField field, OutputStream result) { - Bytes[] data = repackBytesList((List) (List) value); - - try { - Bytes res; - BytesType bytesType = parseFieldType(field); - if (bytesType.size == null) { - res = SSZ.encodeBytesList(data); - } else { - res = SSZ.encodeHashList(data); - } - result.write(res.toArrayUnsafe()); - } catch (IOException ex) { - String error = String.format("Failed to write data from field \"%s\" to stream", - field.getName()); - throw new SSZException(error, ex); - } - } - @Override public Object decode(SSZField field, BytesSSZReaderProxy reader) { BytesType bytesType = parseFieldType(field); @@ -162,86 +140,6 @@ public Object decode(SSZField field, BytesSSZReaderProxy reader) { return throwUnsupportedType(field); } - @Override - public List decodeList( - SSZField field, BytesSSZReaderProxy reader) { - BytesType bytesType = parseFieldType(field); - - if (bytesType.size == null) { - return reader.readBytesList().stream() - .map(Bytes::toArrayUnsafe) - .map(BytesValue::wrap) - .collect(Collectors.toList()); - } - List res = null; - try { - List bytesList = reader.readHashList(bytesType.size); - switch (bytesType.size) { - case 1: - { - res = - bytesList.stream() - .map(Bytes::toArrayUnsafe) - .map(Bytes1::wrap) - .collect(Collectors.toList()); - break; - } - case 4: - { - res = - bytesList.stream() - .map(Bytes::toArrayUnsafe) - .map(Bytes4::wrap) - .collect(Collectors.toList()); - break; - } - case 20: - { - res = - bytesList.stream() - .map(Bytes::toArrayUnsafe) - .map(BytesValue::wrap) - .map(Address::wrap) - .collect(Collectors.toList()); - break; - } - case 32: - { - res = - bytesList.stream() - .map(Bytes::toArrayUnsafe) - .map(Bytes32::wrap) - .collect(Collectors.toList()); - break; - } - case 48: - { - res = - bytesList.stream() - .map(Bytes::toArrayUnsafe) - .map(Bytes48::wrap) - .collect(Collectors.toList()); - break; - } - case 96: - { - res = - bytesList.stream() - .map(Bytes::toArrayUnsafe) - .map(Bytes96::wrap) - .collect(Collectors.toList()); - break; - } - } - } catch (Exception ex) { - String error = - String.format("Failed to read list data from stream to field \"%s\"", field.getName()); - throw new SSZException(error, ex); - } - - return (List) (List) res; - } - private BytesType parseFieldType(SSZField field) { if (classToByteType.containsKey(field.getRawClass())) { return classToByteType.get(field.getRawClass()); diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesPrimitive.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesPrimitive.java index 3376bc45d..0eda5c5c6 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesPrimitive.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesPrimitive.java @@ -31,7 +31,7 @@ * * * Type could be clarified by {@link - * SSZField#extraType} + * SSZField#getExtraType()} */ public class BytesPrimitive implements SSZBasicAccessor { @@ -40,24 +40,12 @@ public class BytesPrimitive implements SSZBasicAccessor { static { supportedTypes.add("bytes"); - supportedTypes.add("hash"); - supportedTypes.add("address"); } static { supportedClassTypes.add(byte[].class); } - private static Bytes[] repackBytesList(List list) { - Bytes[] data = new Bytes[list.size()]; - for (int i = 0; i < list.size(); i++) { - byte[] el = list.get(i); - data[i] = Bytes.of(el); - } - - return data; - } - @Override public Set getSupportedSSZTypes() { return supportedTypes; @@ -79,31 +67,10 @@ public void encode(Object value, SSZField field, OutputStream result) { BytesType byteType = parseFieldType(field); Bytes res = null; byte[] data = (byte[]) value; - - switch (byteType.type) { - case HASH: - { - res = SSZ.encodeHash(Bytes.of(data)); - break; - } - case ADDRESS: - { - res = SSZ.encodeAddress(Bytes.of(data)); - break; - } - case BYTES: - { - if (byteType.size == null) { - res = SSZ.encodeByteArray(data); - } else { - res = SSZ.encodeHash(Bytes.wrap(data)); // w/o length prefix - } - break; - } - default: - { - throwUnsupportedType(field); - } + if (byteType.size == null) { + res = SSZ.encodeByteArray(data); + } else { + res = SSZ.encodeHash(Bytes.wrap(data)); // w/o length prefix } try { @@ -115,124 +82,20 @@ public void encode(Object value, SSZField field, OutputStream result) { } } - @Override - public void encodeList( - List value, SSZField field, OutputStream result) { - BytesType bytesType = parseFieldType(field); - Bytes[] data = repackBytesList((List) (List) value); - - try { - switch (bytesType.type) { - case HASH: - { - result.write(SSZ.encodeHashList(data).toArrayUnsafe()); - break; - } - case BYTES: - { - result.write(SSZ.encodeBytesList(data).toArrayUnsafe()); - break; - } - case ADDRESS: - { - if (bytesType.size == null) { - result.write(SSZ.encodeAddressList(data).toArrayUnsafe()); - } else { - result.write(SSZ.encodeHashList(data).toArrayUnsafe()); - } - break; - } - default: - { - throwUnsupportedType(field); - } - } - } catch (IOException ex) { - String error = String.format("Failed to write data from field \"%s\" to stream", - field.getName()); - throw new SSZException(error, ex); - } - } - @Override public Object decode(SSZField field, BytesSSZReaderProxy reader) { BytesType bytesType = parseFieldType(field); - switch (bytesType.type) { - case BYTES: - { - return (bytesType.size == null) - ? reader.readBytes().toArrayUnsafe() - : reader.readHash(bytesType.size).toArrayUnsafe(); - } - case HASH: - { - return reader.readHash(bytesType.size).toArrayUnsafe(); - } - case ADDRESS: - { - return reader.readAddress().toArrayUnsafe(); - } - } - - return throwUnsupportedType(field); - } - - @Override - public List decodeList( - SSZField field, BytesSSZReaderProxy reader) { - BytesType bytesType = parseFieldType(field); - - switch (bytesType.type) { - case BYTES: - { - if (bytesType.size == null) { - return (List) (List) reader.readByteArrayList(); - } else { - return (List) (List) reader.readHashList(bytesType.size); - } - } - case HASH: - { - return (List) (List) reader.readHashList(bytesType.size); - } - case ADDRESS: - { - return (List) (List) reader.readAddressList(); - } - - default: - { - return throwUnsupportedListType(field); - } - } + return (bytesType.size == null) + ? reader.readBytes().toArrayUnsafe() + : reader.readHash(bytesType.size).toArrayUnsafe(); } private BytesType parseFieldType(SSZField field) { - Type type = Type.fromValue(field.getExtraType()); - - if (type == null || type.equals(Type.BYTES)) { - return BytesType.of(Type.BYTES, field.getExtraSize()); - } - - if (type.equals(Type.ADDRESS)) { - if (field.getExtraSize() != null) { - throw new SSZSchemeException("Address is fixed 20 bytes type"); - } else { - return BytesType.of(Type.ADDRESS, 20); - } - } else { - if (field.getExtraSize() == null) { - throw new SSZSchemeException("Hash size is required!"); - } else { - return BytesType.of(Type.HASH, field.getExtraSize()); - } - } + return BytesType.of(Type.BYTES, field.getExtraSize()); } enum Type { - BYTES("bytes"), - HASH("hash"), - ADDRESS("address"); + BYTES("bytes"); private static final Map ENUM_MAP; diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/HashCodec.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/HashCodec.java index 9a84abe57..406cf6b13 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/HashCodec.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/HashCodec.java @@ -5,7 +5,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; import net.consensys.cava.bytes.Bytes; import net.consensys.cava.ssz.BytesSSZReaderProxy; import net.consensys.cava.ssz.SSZ; @@ -15,7 +14,6 @@ import org.ethereum.beacon.ssz.access.SSZBasicAccessor; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; -import tech.pegasys.artemis.util.bytes.Bytes48; import tech.pegasys.artemis.util.bytes.BytesValue; /** @@ -73,21 +71,6 @@ public void encode(Object value, SSZField field, OutputStream result) { } } - @Override - public void encodeList( - List value, SSZField field, OutputStream result) { - HashType hashType = parseFieldType(field); - Bytes[] data = repackBytesList((List) (List) value); - - try { - result.write(SSZ.encodeHashList(data).toArrayUnsafe()); - } catch (IOException ex) { - String error = String.format("Failed to write data from field \"%s\" to stream", - field.getName()); - throw new SSZException(error, ex); - } - } - @Override public Object decode(SSZField field, BytesSSZReaderProxy reader) { HashType hashType = parseFieldType(field); @@ -108,44 +91,6 @@ public Object decode(SSZField field, BytesSSZReaderProxy reader) { return throwUnsupportedType(field); } - @Override - public List decodeList( - SSZField field, BytesSSZReaderProxy reader) { - HashType hashType = parseFieldType(field); - - List bytesList = reader.readHashList(hashType.size); - List res = null; - try { - switch (hashType.size) { - case 32: - { - res = - bytesList.stream() - .map(Bytes::toArrayUnsafe) - .map(Bytes32::wrap) - .map(Hash32::wrap) - .collect(Collectors.toList()); - break; - } - case 48: - { - res = - bytesList.stream() - .map(Bytes::toArrayUnsafe) - .map(Bytes48::wrap) - .collect(Collectors.toList()); - break; - } - } - } catch (Exception ex) { - String error = - String.format("Failed to read list data from stream to field \"%s\"", field.getName()); - throw new SSZException(error, ex); - } - - return (List) (List) res; - } - private HashType parseFieldType(SSZField field) { if (field.getRawClass().equals(Hash32.class)) { return HashType.of(32); diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/StringPrimitive.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/StringPrimitive.java index 4ddf1ef2a..75a4182c1 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/StringPrimitive.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/StringPrimitive.java @@ -7,7 +7,6 @@ import java.io.IOException; import java.io.OutputStream; import java.util.HashSet; -import java.util.List; import java.util.Set; import org.ethereum.beacon.ssz.access.SSZBasicAccessor; @@ -54,27 +53,8 @@ public void encode(Object value, SSZField field, OutputStream result) { } } - @Override - public void encodeList( - List value, SSZField field, OutputStream result) { - try { - String[] data = value.toArray(new String[0]); - result.write(SSZ.encodeStringList(data).toArrayUnsafe()); - } catch (IOException ex) { - String error = String.format("Failed to write data from field \"%s\" to stream", - field.getName()); - throw new SSZException(error, ex); - } - } - @Override public Object decode(SSZField field, BytesSSZReaderProxy reader) { return reader.readString(); } - - @Override - public List decodeList( - SSZField field, BytesSSZReaderProxy reader) { - return (List) (List) reader.readStringList(); - } } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/SubclassCodec.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/SubclassCodec.java index 2ed3296e6..999532096 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/SubclassCodec.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/SubclassCodec.java @@ -45,12 +45,6 @@ public void encode(Object value, SSZField field, superclassCodec.encode(value, getSerializableField(field), result); } - @Override - public void encodeList(List value, - SSZField field, OutputStream result) { - superclassCodec.encodeList(value, getSerializableField(field), result); - } - @Override public Object decode(SSZField field, BytesSSZReaderProxy reader) { @@ -60,21 +54,6 @@ public Object decode(SSZField field, field.getRawClass(), new Class[] {serializableField.getRawClass()}, new Object[] {serializableTypeObject}); } - @Override - public List decodeList(SSZField field, - BytesSSZReaderProxy reader) { - SSZField serializableField = getSerializableField(field); - List serializableTypeList = superclassCodec.decodeList(serializableField, reader); - return serializableTypeList.stream() - .map( - serializableTypeObject -> - ConstructorObjCreator.createInstanceWithConstructor( - field.getRawClass(), - new Class[] {serializableField.getRawClass()}, - new Object[] {serializableTypeObject})) - .collect(Collectors.toList()); - } - private static SSZField getSerializableField(SSZField field) { return new SSZField(getSerializableClass(field.getRawClass()), field.getFieldAnnotation(), diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntCodec.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntCodec.java index 983073350..72bdd2530 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntCodec.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntCodec.java @@ -13,10 +13,8 @@ import tech.pegasys.artemis.util.uint.UInt64; import java.io.IOException; import java.io.OutputStream; -import java.math.BigInteger; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -111,75 +109,6 @@ public void encode(Object value, SSZField field, OutputStream result) { } } - @Override - public void encodeList( - List value, SSZField field, OutputStream result) { - NumericType numericType = parseFieldType(field); - - try { - switch (numericType.type) { - case BIGINT: - { - encodeBigIntList(value, numericType, result); - break; - } - case LONG: - { - encodeLongList(value, numericType, result); - break; - } - default: - { - throwUnsupportedType(field); - } - } - } catch (IOException ex) { - String error = String.format("Failed to write data from field \"%s\" to stream", - field.getName()); - throw new SSZException(error, ex); - } - } - - private void encodeLongList(List value, NumericType type, OutputStream result) - throws IOException { - long[] data = new long[value.size()]; - for (int i = 0; i < value.size(); ++i) { - switch (type.size) { - case 24: - { - data[i] = ((UInt24) value.get(i)).getValue(); - break; - } - case 64: - { - data[i] = ((UInt64) value.get(i)).getValue(); - break; - } - } - } - result.write(SSZ.encodeULongIntList(type.size, data).toArrayUnsafe()); - } - - private void encodeBigIntList(List value, NumericType type, OutputStream result) - throws IOException { - Bytes[] data = new Bytes[value.size()]; - for (int i = 0; i < value.size(); ++i) { - switch (type.size) { - case 256: - { - data[i] = Bytes.of(((UInt256) value.get(i)).bytes().getArrayUnsafe()); - break; - } - default: - { - String error = String.format("Unsupported type \"%s\"", type); - throw new SSZSchemeException(error); - } - } - } - result.write(SSZ.encodeBytesList(data).toArrayUnsafe()); - } - @Override public Object decode(SSZField field, BytesSSZReaderProxy reader) { NumericType numericType = parseFieldType(field); @@ -228,67 +157,6 @@ private Object decodeBigInt(NumericType type, BytesSSZReaderProxy reader) { } } - @Override - public List decodeList( - SSZField field, BytesSSZReaderProxy reader) { - NumericType numericType = parseFieldType(field); - - switch (numericType.type) { - case LONG: - { - switch (numericType.size) { - case 24: - { - return readUInt24List(numericType, reader); - } - case 64: - { - return readUInt64List(numericType, reader); - } - } - } - case BIGINT: - { - if (numericType.size == 256) { - return readUInt256List(numericType, reader); - } - } - default: - { - return throwUnsupportedListType(field); - } - } - } - - private List readUInt24List(NumericType numericType, BytesSSZReaderProxy reader) { - List longList = reader.readUnsignedBigIntegerList(numericType.size); - List res = - longList.stream() - .map(BigInteger::intValue) - .map(UInt24::valueOf) - .collect(Collectors.toList()); - - return res; - } - - private List readUInt64List(NumericType numericType, BytesSSZReaderProxy reader) { - List longList = reader.readUnsignedBigIntegerList(numericType.size); - List res = - longList.stream() - .map(BigInteger::longValue) - .map(UInt64::valueOf) - .collect(Collectors.toList()); - - return res; - } - - private List readUInt256List(NumericType numericType, BytesSSZReaderProxy reader) { - List bigIntList = reader.readBigIntegerList(256); - List res = bigIntList.stream().map(UInt256::of).collect(Collectors.toList()); - - return res; - } - private NumericType parseFieldType(SSZField field) { return classToNumericType.get(field.getRawClass()); } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntPrimitive.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntPrimitive.java index b1ce44e9e..d203fa125 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntPrimitive.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntPrimitive.java @@ -12,7 +12,6 @@ import java.math.BigInteger; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -135,64 +134,6 @@ public void encode(Object value, SSZField field, OutputStream result) { } } - @Override - public void encodeList( - List value, SSZField field, OutputStream result) { - NumericType numericType = parseFieldType(field); - - try { - switch (numericType.type) { - case BIGINT: - { - encodeBigIntList(value, numericType, result); - break; - } - case INT: - { - encodeIntList(value, numericType, result); - break; - } - case LONG: - { - encodeLongList(value, numericType, result); - break; - } - default: - { - throwUnsupportedType(field); - } - } - } catch (IOException ex) { - String error = String.format("Failed to write data from field \"%s\" to stream", - field.getName()); - throw new SSZException(error, ex); - } - } - - private void encodeIntList(List value, NumericType type, OutputStream result) - throws IOException { - int[] data = new int[value.size()]; - for (int i = 0; i < value.size(); ++i) { - data[i] = (int) value.get(i); - } - result.write(SSZ.encodeUIntList(type.size, data).toArrayUnsafe()); - } - - private void encodeLongList(List value, NumericType type, OutputStream result) - throws IOException { - long[] data = new long[value.size()]; - for (int i = 0; i < value.size(); ++i) { - data[i] = (long) value.get(i); - } - result.write(SSZ.encodeULongIntList(type.size, data).toArrayUnsafe()); - } - - private void encodeBigIntList(List value, NumericType type, OutputStream result) - throws IOException { - BigInteger[] data = value.toArray(new BigInteger[0]); - result.write(SSZ.encodeBigIntegerList(type.size, data).toArrayUnsafe()); - } - @Override public Object decode(SSZField field, BytesSSZReaderProxy reader) { NumericType numericType = parseFieldType(field); @@ -226,31 +167,6 @@ private Object decodeBigInt(NumericType type, BytesSSZReaderProxy reader) { return reader.readUnsignedBigInteger(type.size); } - @Override - public List decodeList( - SSZField field, BytesSSZReaderProxy reader) { - NumericType numericType = parseFieldType(field); - - switch (numericType.type) { - case INT: - { - return reader.readUIntList(numericType.size); - } - case LONG: - { - return reader.readULongIntList(numericType.size); - } - case BIGINT: - { - return reader.readUnsignedBigIntegerList(numericType.size); - } - default: - { - return throwUnsupportedListType(field); - } - } - } - private NumericType parseFieldType(SSZField field) { if (field.getExtraSize() != null && field.getExtraSize() % Byte.SIZE != 0) { String error = diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/annotation/SSZ.java b/ssz/src/main/java/org/ethereum/beacon/ssz/annotation/SSZ.java index e87f6a237..0bfd30617 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/annotation/SSZ.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/annotation/SSZ.java @@ -31,9 +31,7 @@ String UInt384 = "uint384"; String UInt512 = "uint512"; String Bytes = "bytes"; - String Hash = "hash"; - String Hash32 = "hash32"; - String Hash48 = "hash48"; + String Hash32 = "bytes32"; String Bool = "bool"; String Address = "address"; String String = "string"; diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java index 9a1231c76..6d4a35816 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java @@ -26,10 +26,12 @@ public DecodeResult(Object decodedInstance, int readBytes) { @Override public DecodeResult visitBasicValue(SSZBasicType sszType, Bytes param) { BytesSSZReaderProxy reader = new BytesSSZReaderProxy(param); - // TODO support basic codecs with variable size -// int readBytes = sszType.isFixedSize() ? sszType.getSize() : reader. + int readBytes = sszType.getSize(); + if (readBytes == -1) { + readBytes = BYTES_PER_LENGTH_PREFIX + deserializeLength(param.slice(0, BYTES_PER_LENGTH_PREFIX)); + } return new DecodeResult(sszType.getAccessor().decode(sszType.getTypeDescriptor(), - reader), sszType.getSize()); + reader), readBytes); } @Override @@ -38,7 +40,7 @@ public DecodeResult visitComposite(SSZCompositeType type, Bytes param, int pos = 0; int size; if (type.isVariableSize()) { - size = deserializeLength(param.slice(0, 4)) + BYTES_PER_LENGTH_PREFIX; + size = deserializeLength(param.slice(0, BYTES_PER_LENGTH_PREFIX)) + BYTES_PER_LENGTH_PREFIX; pos += BYTES_PER_LENGTH_PREFIX; } else { size = type.getSize(); diff --git a/ssz/src/test/java/org/ethereum/beacon/ssz/SSZSerializerTest.java b/ssz/src/test/java/org/ethereum/beacon/ssz/SSZSerializerTest.java index bc6cde12b..0569bc459 100644 --- a/ssz/src/test/java/org/ethereum/beacon/ssz/SSZSerializerTest.java +++ b/ssz/src/test/java/org/ethereum/beacon/ssz/SSZSerializerTest.java @@ -1,6 +1,8 @@ package org.ethereum.beacon.ssz; import java.util.Arrays; + +import com.google.common.base.Objects; import org.ethereum.beacon.crypto.Hashes; import org.ethereum.beacon.ssz.access.container.SSZAnnotationSchemeBuilder; import org.ethereum.beacon.ssz.annotation.SSZSerializable; @@ -75,10 +77,13 @@ public void SignatureTest() { @Test public void simpleTest() { + List obliqueParentHashes = new ArrayList<>(); + obliqueParentHashes.add(new byte[] {0x12, 0x34, 0x56}); + obliqueParentHashes.add(new byte[] {0x55, 0x66, 0x77}); AttestationRecord expected = new AttestationRecord( 123, - Collections.emptyList(), + obliqueParentHashes, DEFAULT_HASH, new Bitfield(BytesValue.fromHexString("abcdef45").getArrayUnsafe()), DEFAULT_HASH, @@ -156,8 +161,49 @@ public void nullListTest() { sszSerializer.encode(expected4); } + @SSZSerializable + public static class ListsObject { + private final List list1; + private final List list2; + + public ListsObject(List list1, List list2) { + this.list1 = list1; + this.list2 = list2; + } + + public List getList1() { + return list1; + } + + public List getList2() { + return list2; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ListsObject that = (ListsObject) o; + return Objects.equal(list1, that.list1) && + Objects.equal(list2, that.list2); + } + } + + @Test + public void shouldHandleLists() { + List list1 = new ArrayList<>(); + list1.add(1); + list1.add(2); + List list2 = new ArrayList<>(); + list2.add("aa"); + ListsObject expected = new ListsObject(list1, list2); + byte[] encoded = sszSerializer.encode(expected); + ListsObject actual = sszSerializer.decode(encoded, ListsObject.class); + + assertEquals(expected, actual); + } + /** Checks that we could handle list placed inside another list */ - @Ignore("Implement me!") @Test public void shouldHandleListList() { List list1 = new ArrayList<>(); diff --git a/ssz/src/test/java/org/ethereum/beacon/ssz/fixtures/AttestationRecord.java b/ssz/src/test/java/org/ethereum/beacon/ssz/fixtures/AttestationRecord.java index 5202727f6..781f9356d 100644 --- a/ssz/src/test/java/org/ethereum/beacon/ssz/fixtures/AttestationRecord.java +++ b/ssz/src/test/java/org/ethereum/beacon/ssz/fixtures/AttestationRecord.java @@ -105,7 +105,7 @@ public boolean equals(Object o) { return slot == that.slot && shardId == that.shardId && justifiedSlot == that.justifiedSlot - && Objects.equals(obliqueParentHashes, that.obliqueParentHashes) + && Arrays.deepEquals(obliqueParentHashes.toArray(), that.obliqueParentHashes.toArray()) && Arrays.equals(shardBlockHash, that.shardBlockHash) && Objects.equals(attesterBitfield, that.attesterBitfield) && Arrays.equals(justifiedBlockHash, that.justifiedBlockHash) From b24c251a5b3d3d1f0df71551968dd225036f0e88 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 13 May 2019 20:02:52 +0300 Subject: [PATCH 02/15] ssz: use length prefix length from one definition --- .../org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java | 1 - .../java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java index 6d4a35816..52c551070 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java @@ -1,7 +1,6 @@ package org.ethereum.beacon.ssz.visitor; import java.nio.ByteOrder; -import java.util.function.BiFunction; import net.consensys.cava.bytes.Bytes; import net.consensys.cava.ssz.BytesSSZReaderProxy; import org.ethereum.beacon.ssz.SSZSerializeException; diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java index 3ba059f7d..a21189a02 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java @@ -1,5 +1,6 @@ package org.ethereum.beacon.ssz.visitor; +import static org.ethereum.beacon.ssz.visitor.SSZSimpleDeserializer.BYTES_PER_LENGTH_PREFIX; import static tech.pegasys.artemis.util.bytes.BytesValue.concat; import java.util.ArrayList; @@ -129,6 +130,6 @@ public Hash32 getZeroHash(int distanceFromBottom) { } BytesValue serializeLength(long len) { - return concat(BytesValues.ofUnsignedIntLittleEndian(len), BytesValue.wrap(new byte[32 - 4])); + return concat(BytesValues.ofUnsignedIntLittleEndian(len), BytesValue.wrap(new byte[Hash32.SIZE - BYTES_PER_LENGTH_PREFIX])); } } From bb17004a79f8ff9bc83c861b9c4fbe39451b38a8 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Mon, 13 May 2019 20:04:56 +0300 Subject: [PATCH 03/15] ssz: research on cava part that uses prefix length --- .../cava/ssz/BytesSSZReaderProxy.java | 170 +----------------- 1 file changed, 2 insertions(+), 168 deletions(-) diff --git a/ssz/src/main/java/net/consensys/cava/ssz/BytesSSZReaderProxy.java b/ssz/src/main/java/net/consensys/cava/ssz/BytesSSZReaderProxy.java index e6ac28b4a..0298a4ac3 100644 --- a/ssz/src/main/java/net/consensys/cava/ssz/BytesSSZReaderProxy.java +++ b/ssz/src/main/java/net/consensys/cava/ssz/BytesSSZReaderProxy.java @@ -30,41 +30,15 @@ private BytesSSZReaderProxy(BytesSSZReader reader) { } public Bytes readBytes() { + // FIXME: uses length prefix return reader.readBytes(); } - public byte[] readByteArray() { - return reader.readByteArray(); - } - - public byte[] readByteArray(int limit) { - return reader.readByteArray(limit); - } - public String readString() { + // FIXME: uses length prefix return reader.readString(); } - public String readString(int limit) { - return reader.readString(limit); - } - - public int readInt8() { - return reader.readInt8(); - } - - public int readInt16() { - return reader.readInt16(); - } - - public int readInt32() { - return reader.readInt32(); - } - - public long readInt64() { - return reader.readInt64(); - } - public int readUInt(int bitLength) { return reader.readUInt(bitLength); } @@ -73,155 +47,15 @@ public long readULong(int bitLength) { return reader.readULong(bitLength); } - public int readUInt8() { - return reader.readUInt8(); - } - - public int readUInt16() { - return reader.readUInt16(); - } - - public long readUInt32() { - return reader.readUInt32(); - } - - public long readUInt64() { - return reader.readUInt64(); - } - public boolean readBoolean() { return reader.readBoolean(); } - public List readBytesList() { - return reader.readBytesList(); - } - - public List readByteArrayList() { - return reader.readByteArrayList(); - } - - public List readByteArrayList(int limit) { - return reader.readByteArrayList(limit); - } - - public List readStringList() { - return reader.readStringList(); - } - - public List readInt8List() { - return reader.readInt8List(); - } - - public List readInt16List() { - return reader.readInt16List(); - } - - public List readInt32List() { - return reader.readInt32List(); - } - - public List readInt64List() { - return reader.readInt64List(); - } - - public List readUIntList(int bitLength) { - return reader.readUIntList(bitLength); - } - - public List readULongIntList(int bitLength) { - return reader.readULongIntList(bitLength); - } - - public List readUInt8List() { - return reader.readUInt8List(); - } - - public List readUInt16List() { - return reader.readUInt16List(); - } - - public List readUInt32List() { - return reader.readUInt32List(); - } - - public List readUInt64List() { - return reader.readUInt64List(); - } - - public Bytes readBytes(int limit) { - return reader.readBytes(limit); - } - - public int readInt(int bitLength) { - return reader.readInt(bitLength); - } - - public long readLong(int bitLength) { - return reader.readLong(bitLength); - } - - public BigInteger readBigInteger(int bitLength) { - return reader.readBigInteger(bitLength); - } - public BigInteger readUnsignedBigInteger(int bitLength) { return reader.readUnsignedBigInteger(bitLength); } - public UInt256 readUInt256() { - return reader.readUInt256(); - } - - public Bytes readAddress() { - return reader.readAddress(); - } - public Bytes readHash(int hashLength) { return reader.readHash(hashLength); } - - public List readBytesList(int limit) { - return reader.readBytesList(limit); - } - - public List readStringList(int limit) { - return reader.readStringList(limit); - } - - public List readIntList(int bitLength) { - return reader.readIntList(bitLength); - } - - public List readLongIntList(int bitLength) { - return reader.readLongIntList(bitLength); - } - - public List readBigIntegerList(int bitLength) { - return reader.readBigIntegerList(bitLength); - } - - public List readUnsignedBigIntegerList(int bitLength) { - return reader.readUnsignedBigIntegerList(bitLength); - } - - public List readUInt256List() { - return reader.readUInt256List(); - } - - public List readAddressList() { - return reader.readAddressList(); - } - - public List readHashList(int hashLength) { - return reader.readHashList(hashLength); - } - - public List readBooleanList() { - return reader.readBooleanList(); - } - - public boolean isComplete() { - return reader.isComplete(); - } } From aa847be49bd77e61943d6df86cd6e32306bec1da Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 14 May 2019 18:08:48 +0300 Subject: [PATCH 04/15] ssz: implement SOS (offset based variable part serialization) in ssz --- .../cava/ssz/BytesSSZReaderProxy.java | 61 ------ .../ethereum/beacon/ssz/SSZSerializer.java | 10 +- .../beacon/ssz/access/SSZBasicAccessor.java | 5 +- .../ssz/access/basic/BooleanPrimitive.java | 8 +- .../beacon/ssz/access/basic/BytesCodec.java | 10 +- .../ssz/access/basic/BytesPrimitive.java | 12 +- .../beacon/ssz/access/basic/HashCodec.java | 18 +- .../ssz/access/basic/StringPrimitive.java | 8 +- .../ssz/access/basic/SubclassCodec.java | 8 +- .../beacon/ssz/access/basic/UIntCodec.java | 14 +- .../ssz/access/basic/UIntPrimitive.java | 16 +- .../beacon/ssz/visitor/SSZReader.java | 122 ++++++++++++ .../ssz/visitor/SSZSimpleDeserializer.java | 187 ++++++++++++++---- .../beacon/ssz/visitor/SSZSimpleHasher.java | 7 +- .../ssz/visitor/SSZSimpleSerializer.java | 139 ++++++++----- .../beacon/ssz/visitor/SSZWriter.java | 171 ++++++++++++++++ 16 files changed, 578 insertions(+), 218 deletions(-) delete mode 100644 ssz/src/main/java/net/consensys/cava/ssz/BytesSSZReaderProxy.java create mode 100644 ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZReader.java create mode 100644 ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZWriter.java diff --git a/ssz/src/main/java/net/consensys/cava/ssz/BytesSSZReaderProxy.java b/ssz/src/main/java/net/consensys/cava/ssz/BytesSSZReaderProxy.java deleted file mode 100644 index 0298a4ac3..000000000 --- a/ssz/src/main/java/net/consensys/cava/ssz/BytesSSZReaderProxy.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package net.consensys.cava.ssz; - -import net.consensys.cava.bytes.Bytes; -import net.consensys.cava.units.bigints.UInt256; -import java.math.BigInteger; -import java.util.List; - -public class BytesSSZReaderProxy { - private BytesSSZReader reader; - - public BytesSSZReaderProxy(Bytes bytes) { - this(new BytesSSZReader(bytes)); - } - - private BytesSSZReaderProxy(BytesSSZReader reader) { - this.reader = reader; - } - - public Bytes readBytes() { - // FIXME: uses length prefix - return reader.readBytes(); - } - - public String readString() { - // FIXME: uses length prefix - return reader.readString(); - } - - public int readUInt(int bitLength) { - return reader.readUInt(bitLength); - } - - public long readULong(int bitLength) { - return reader.readULong(bitLength); - } - - public boolean readBoolean() { - return reader.readBoolean(); - } - - public BigInteger readUnsignedBigInteger(int bitLength) { - return reader.readUnsignedBigInteger(bitLength); - } - - public Bytes readHash(int hashLength) { - return reader.readHash(hashLength); - } -} diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java index f612c12d1..22a76035e 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java @@ -13,6 +13,7 @@ import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; import org.ethereum.beacon.ssz.visitor.SSZVisitorHandler; import org.ethereum.beacon.ssz.visitor.SSZVisitorHost; +import org.javatuples.Pair; /** SSZ serializer/deserializer */ public class SSZSerializer implements BytesSerializer, SSZVisitorHandler { @@ -35,7 +36,7 @@ public SSZSerializer(SSZVisitorHost sszVisitorHost, */ @Override public byte[] encode(@Nullable C inputObject, Class inputClazz) { - return visit(inputObject, inputClazz).getSerialized().getArrayUnsafe(); + return visit(inputObject, inputClazz).getSerializedBody().getArrayUnsafe(); } private SSZSerializerResult visit(C input, Class clazz) { @@ -62,11 +63,10 @@ public SSZSerializerResult visitList(SSZListType descriptor, Object listValue, i public C decode(byte[] data, Class clazz) { DecodeResult decodeResult = sszVisitorHost.handleAny( typeResolver.resolveSSZType(new SSZField(clazz)), - Bytes.wrap(data), + Pair.with(Bytes.wrap(data), null), new SSZSimpleDeserializer()); - if (data.length > decodeResult.readBytes) { - throw new SSZSerializeException("Invalid SSZ data length (data is bigger than required): " - + data.length + " > " + decodeResult.readBytes); + if (data.length != decodeResult.readBytes) { + throw new SSZSerializeException(String.format("Invalid SSZ encoding, calculated data size %s bytes, while provided %s bytes", decodeResult.readBytes, data.length)); } return (C) decodeResult.decodedInstance; } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/SSZBasicAccessor.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/SSZBasicAccessor.java index 53710d4c4..57b9ce253 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/SSZBasicAccessor.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/SSZBasicAccessor.java @@ -1,10 +1,9 @@ package org.ethereum.beacon.ssz.access; -import net.consensys.cava.ssz.BytesSSZReaderProxy; +import org.ethereum.beacon.ssz.visitor.SSZReader; import org.ethereum.beacon.ssz.SSZSchemeException; import org.ethereum.beacon.ssz.SSZSerializer; import java.io.OutputStream; -import java.util.Arrays; import java.util.List; import java.util.Set; @@ -52,7 +51,7 @@ public interface SSZBasicAccessor { * moved to the end of this field/beginning of next one after reading is performed. * @return field value */ - Object decode(SSZField field, BytesSSZReaderProxy reader); + Object decode(SSZField field, SSZReader reader); /** * Helper designed to throw usual error diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BooleanPrimitive.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BooleanPrimitive.java index 93c9965e0..b0955ad3d 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BooleanPrimitive.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BooleanPrimitive.java @@ -1,8 +1,8 @@ package org.ethereum.beacon.ssz.access.basic; import net.consensys.cava.bytes.Bytes; -import net.consensys.cava.ssz.BytesSSZReaderProxy; -import net.consensys.cava.ssz.SSZ; +import org.ethereum.beacon.ssz.visitor.SSZReader; +import org.ethereum.beacon.ssz.visitor.SSZWriter; import net.consensys.cava.ssz.SSZException; import java.io.IOException; import java.io.OutputStream; @@ -46,7 +46,7 @@ public int getSize(SSZField field) { @Override public void encode(Object value, SSZField field, OutputStream result) { boolean boolValue = (boolean) value; - Bytes res = SSZ.encodeBoolean(boolValue); + Bytes res = SSZWriter.encodeBoolean(boolValue); try { result.write(res.toArrayUnsafe()); } catch (IOException e) { @@ -56,7 +56,7 @@ public void encode(Object value, SSZField field, OutputStream result) { } @Override - public Object decode(SSZField field, BytesSSZReaderProxy reader) { + public Object decode(SSZField field, SSZReader reader) { return reader.readBoolean(); } } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesCodec.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesCodec.java index 177391795..9ad889688 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesCodec.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesCodec.java @@ -1,8 +1,8 @@ package org.ethereum.beacon.ssz.access.basic; import net.consensys.cava.bytes.Bytes; -import net.consensys.cava.ssz.BytesSSZReaderProxy; -import net.consensys.cava.ssz.SSZ; +import org.ethereum.beacon.ssz.visitor.SSZReader; +import org.ethereum.beacon.ssz.visitor.SSZWriter; import net.consensys.cava.ssz.SSZException; import org.ethereum.beacon.ssz.access.SSZField; import org.ethereum.beacon.ssz.SSZSchemeException; @@ -83,9 +83,9 @@ public void encode(Object value, SSZField field, OutputStream result) { BytesValue data = (BytesValue) value; BytesType bytesType = parseFieldType(field); if (bytesType.size == null) { - res = SSZ.encodeBytes(Bytes.of(data.getArrayUnsafe())); + res = SSZWriter.encodeBytes(Bytes.of(data.getArrayUnsafe())); } else { - res = SSZ.encodeHash(Bytes.of(data.getArrayUnsafe())); + res = SSZWriter.encodeBytes(Bytes.of(data.getArrayUnsafe()), bytesType.size); } try { @@ -98,7 +98,7 @@ public void encode(Object value, SSZField field, OutputStream result) { } @Override - public Object decode(SSZField field, BytesSSZReaderProxy reader) { + public Object decode(SSZField field, SSZReader reader) { BytesType bytesType = parseFieldType(field); if (bytesType.size == null) { diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesPrimitive.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesPrimitive.java index 0eda5c5c6..f002236a4 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesPrimitive.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/BytesPrimitive.java @@ -1,17 +1,15 @@ package org.ethereum.beacon.ssz.access.basic; import net.consensys.cava.bytes.Bytes; -import net.consensys.cava.ssz.BytesSSZReaderProxy; -import net.consensys.cava.ssz.SSZ; +import org.ethereum.beacon.ssz.visitor.SSZReader; +import org.ethereum.beacon.ssz.visitor.SSZWriter; import net.consensys.cava.ssz.SSZException; import org.ethereum.beacon.ssz.access.SSZBasicAccessor; import org.ethereum.beacon.ssz.access.SSZField; -import org.ethereum.beacon.ssz.SSZSchemeException; import java.io.IOException; import java.io.OutputStream; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -68,9 +66,9 @@ public void encode(Object value, SSZField field, OutputStream result) { Bytes res = null; byte[] data = (byte[]) value; if (byteType.size == null) { - res = SSZ.encodeByteArray(data); + res = SSZWriter.encodeByteArray(data); } else { - res = SSZ.encodeHash(Bytes.wrap(data)); // w/o length prefix + res = SSZWriter.encodeBytes(Bytes.wrap(data), byteType.size); } try { @@ -83,7 +81,7 @@ public void encode(Object value, SSZField field, OutputStream result) { } @Override - public Object decode(SSZField field, BytesSSZReaderProxy reader) { + public Object decode(SSZField field, SSZReader reader) { BytesType bytesType = parseFieldType(field); return (bytesType.size == null) ? reader.readBytes().toArrayUnsafe() diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/HashCodec.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/HashCodec.java index 406cf6b13..bba626af9 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/HashCodec.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/HashCodec.java @@ -6,8 +6,8 @@ import java.util.List; import java.util.Set; import net.consensys.cava.bytes.Bytes; -import net.consensys.cava.ssz.BytesSSZReaderProxy; -import net.consensys.cava.ssz.SSZ; +import org.ethereum.beacon.ssz.visitor.SSZReader; +import org.ethereum.beacon.ssz.visitor.SSZWriter; import net.consensys.cava.ssz.SSZException; import org.ethereum.beacon.ssz.access.SSZField; import org.ethereum.beacon.ssz.SSZSchemeException; @@ -57,10 +57,9 @@ public int getSize(SSZField field) { @Override public void encode(Object value, SSZField field, OutputStream result) { - HashType hashType = parseFieldType(field); Bytes res = null; BytesValue data = (BytesValue) value; - res = SSZ.encodeHash(Bytes.of(data.getArrayUnsafe())); + res = SSZWriter.encodeBytes(Bytes.of(data.getArrayUnsafe()), getSize(field)); try { result.write(res.toArrayUnsafe()); @@ -72,23 +71,16 @@ public void encode(Object value, SSZField field, OutputStream result) { } @Override - public Object decode(SSZField field, BytesSSZReaderProxy reader) { + public Object decode(SSZField field, SSZReader reader) { HashType hashType = parseFieldType(field); try { - switch (hashType.size) { - case 32: - { - return Hash32.wrap(Bytes32.wrap(reader.readHash(hashType.size).toArrayUnsafe())); - } - } + return Hash32.wrap(Bytes32.wrap(reader.readHash(hashType.size).toArrayUnsafe())); } catch (Exception ex) { String error = String.format("Failed to read data from stream to field \"%s\"", field.getName()); throw new SSZException(error, ex); } - - return throwUnsupportedType(field); } private HashType parseFieldType(SSZField field) { diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/StringPrimitive.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/StringPrimitive.java index 75a4182c1..4d92dbc1b 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/StringPrimitive.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/StringPrimitive.java @@ -1,8 +1,8 @@ package org.ethereum.beacon.ssz.access.basic; import net.consensys.cava.bytes.Bytes; -import net.consensys.cava.ssz.BytesSSZReaderProxy; -import net.consensys.cava.ssz.SSZ; +import org.ethereum.beacon.ssz.visitor.SSZReader; +import org.ethereum.beacon.ssz.visitor.SSZWriter; import net.consensys.cava.ssz.SSZException; import java.io.IOException; import java.io.OutputStream; @@ -44,7 +44,7 @@ public Set getSupportedClasses() { @Override public void encode(Object value, SSZField field, OutputStream result) { String sValue = (String) value; - Bytes res = SSZ.encodeString(sValue); + Bytes res = SSZWriter.encodeString(sValue); try { result.write(res.toArrayUnsafe()); } catch (IOException e) { @@ -54,7 +54,7 @@ public void encode(Object value, SSZField field, OutputStream result) { } @Override - public Object decode(SSZField field, BytesSSZReaderProxy reader) { + public Object decode(SSZField field, SSZReader reader) { return reader.readString(); } } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/SubclassCodec.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/SubclassCodec.java index 999532096..bfb425d17 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/SubclassCodec.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/SubclassCodec.java @@ -1,10 +1,9 @@ package org.ethereum.beacon.ssz.access.basic; import java.io.OutputStream; -import java.util.List; import java.util.Set; -import java.util.stream.Collectors; -import net.consensys.cava.ssz.BytesSSZReaderProxy; + +import org.ethereum.beacon.ssz.visitor.SSZReader; import org.ethereum.beacon.ssz.access.SSZBasicAccessor; import org.ethereum.beacon.ssz.creator.ConstructorObjCreator; import org.ethereum.beacon.ssz.access.SSZField; @@ -46,8 +45,7 @@ public void encode(Object value, SSZField field, } @Override - public Object decode(SSZField field, - BytesSSZReaderProxy reader) { + public Object decode(SSZField field, SSZReader reader) { SSZField serializableField = getSerializableField(field); Object serializableTypeObject = superclassCodec.decode(serializableField, reader); return ConstructorObjCreator.createInstanceWithConstructor( diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntCodec.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntCodec.java index 72bdd2530..8f600e062 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntCodec.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntCodec.java @@ -1,8 +1,8 @@ package org.ethereum.beacon.ssz.access.basic; import net.consensys.cava.bytes.Bytes; -import net.consensys.cava.ssz.BytesSSZReaderProxy; -import net.consensys.cava.ssz.SSZ; +import org.ethereum.beacon.ssz.visitor.SSZReader; +import org.ethereum.beacon.ssz.visitor.SSZWriter; import net.consensys.cava.ssz.SSZException; import org.ethereum.beacon.ssz.access.SSZBasicAccessor; import org.ethereum.beacon.ssz.access.SSZField; @@ -85,14 +85,14 @@ public void encode(Object value, SSZField field, OutputStream result) { case 24: { UInt24 uValue = (UInt24) value; - Bytes bytes = SSZ.encodeULong(uValue.getValue(), numericType.size); + Bytes bytes = SSZWriter.encodeULong(uValue.getValue(), numericType.size); writeBytes(bytes, result); break; } case 64: { UInt64 uValue = (UInt64) value; - Bytes bytes = SSZ.encodeULong(uValue.getValue(), numericType.size); + Bytes bytes = SSZWriter.encodeULong(uValue.getValue(), numericType.size); writeBytes(bytes, result); break; } @@ -110,7 +110,7 @@ public void encode(Object value, SSZField field, OutputStream result) { } @Override - public Object decode(SSZField field, BytesSSZReaderProxy reader) { + public Object decode(SSZField field, SSZReader reader) { NumericType numericType = parseFieldType(field); switch (numericType.type) { case LONG: @@ -126,7 +126,7 @@ public Object decode(SSZField field, BytesSSZReaderProxy reader) { return throwUnsupportedType(field); } - private Object decodeLong(NumericType type, BytesSSZReaderProxy reader) { + private Object decodeLong(NumericType type, SSZReader reader) { // XXX: reader.readULong is buggy switch (type.size) { case 24: @@ -142,7 +142,7 @@ private Object decodeLong(NumericType type, BytesSSZReaderProxy reader) { throw new SSZSchemeException(error); } - private Object decodeBigInt(NumericType type, BytesSSZReaderProxy reader) { + private Object decodeBigInt(NumericType type, SSZReader reader) { switch (type.size) { case 256: { diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntPrimitive.java b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntPrimitive.java index d203fa125..117fb5e24 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntPrimitive.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/access/basic/UIntPrimitive.java @@ -1,8 +1,8 @@ package org.ethereum.beacon.ssz.access.basic; import net.consensys.cava.bytes.Bytes; -import net.consensys.cava.ssz.BytesSSZReaderProxy; -import net.consensys.cava.ssz.SSZ; +import org.ethereum.beacon.ssz.visitor.SSZReader; +import org.ethereum.beacon.ssz.visitor.SSZWriter; import net.consensys.cava.ssz.SSZException; import org.ethereum.beacon.ssz.access.SSZBasicAccessor; import org.ethereum.beacon.ssz.access.SSZField; @@ -73,7 +73,7 @@ private static void encodeLong(Object value, NumericType type, OutputStream resu } private static void encodeLong(long value, int bitLength, OutputStream result) { - Bytes res = SSZ.encodeULong(value, bitLength); + Bytes res = SSZWriter.encodeULong(value, bitLength); try { result.write(res.toArrayUnsafe()); } catch (IOException e) { @@ -83,7 +83,7 @@ private static void encodeLong(long value, int bitLength, OutputStream result) { private static void encodeBigInt(Object value, NumericType type, OutputStream result) { BigInteger valueBI = (BigInteger) value; - Bytes res = SSZ.encodeBigInteger(valueBI, type.size); + Bytes res = SSZWriter.encodeBigInteger(valueBI, type.size); try { result.write(res.toArrayUnsafe()); @@ -135,7 +135,7 @@ public void encode(Object value, SSZField field, OutputStream result) { } @Override - public Object decode(SSZField field, BytesSSZReaderProxy reader) { + public Object decode(SSZField field, SSZReader reader) { NumericType numericType = parseFieldType(field); switch (numericType.type) { case INT: @@ -155,15 +155,15 @@ public Object decode(SSZField field, BytesSSZReaderProxy reader) { return throwUnsupportedType(field); } - private Object decodeInt(NumericType type, BytesSSZReaderProxy reader) { + private Object decodeInt(NumericType type, SSZReader reader) { return reader.readUInt(type.size); } - private Object decodeLong(NumericType type, BytesSSZReaderProxy reader) { + private Object decodeLong(NumericType type, SSZReader reader) { return reader.readULong(type.size); } - private Object decodeBigInt(NumericType type, BytesSSZReaderProxy reader) { + private Object decodeBigInt(NumericType type, SSZReader reader) { return reader.readUnsignedBigInteger(type.size); } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZReader.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZReader.java new file mode 100644 index 000000000..05973298d --- /dev/null +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZReader.java @@ -0,0 +1,122 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.ethereum.beacon.ssz.visitor; + +import net.consensys.cava.bytes.Bytes; +import net.consensys.cava.ssz.EndOfSSZException; +import net.consensys.cava.ssz.InvalidSSZTypeException; +import org.javatuples.Pair; + +import java.math.BigInteger; +import java.util.function.Supplier; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.nio.ByteOrder.LITTLE_ENDIAN; +import static java.nio.charset.StandardCharsets.UTF_8; + +/** Reader with some adoption of {@link net.consensys.cava.ssz.BytesSSZReader} code */ +public class SSZReader { + + private final Bytes content; + private int index = 0; + + public SSZReader(Bytes bytes) { + this.content = bytes; + } + + public Bytes readBytes() { + return readHash(content.size() - index); + } + + public String readString() { + return new String(readBytes().toArray(), UTF_8); + } + + public int readUInt(int bitLength) { + Pair params = + checkAndConsumeNumber(bitLength, Integer.BYTES, "decoded integer is too large for an int"); + return params + .getValue0() + .slice(0, params.getValue0().size() - params.getValue1()) + .toInt(LITTLE_ENDIAN); + } + + private Pair checkAndConsumeNumber( + int bitLength, int maxByteSize, String errorMessage) { + checkArgument( + bitLength % Byte.SIZE == 0, String.format("bitLength must be a multiple of %s", Byte.SIZE)); + int byteLength = bitLength / Byte.SIZE; + ensureBytes( + byteLength, + () -> "SSZ encoded data has insufficient length to read a " + bitLength + "-bit numeric"); + Bytes bytes = consumeBytes(byteLength); + int zeroBytes = bytes.numberOfTrailingZeroBytes(); + if ((byteLength - zeroBytes) > maxByteSize) { + throw new InvalidSSZTypeException(errorMessage); + } + + return Pair.with(bytes, zeroBytes); + } + + private Bytes consumeBytes(int size) { + Bytes bytes = content.slice(index, size); + index += size; + return bytes; + } + + public long readULong(int bitLength) { + Pair params = + checkAndConsumeNumber(bitLength, Long.BYTES, "decoded number is too large for an long"); + return params + .getValue0() + .slice(0, params.getValue0().size() - params.getValue1()) + .toLong(LITTLE_ENDIAN); + } + + public boolean readBoolean() { + int value = readUInt(Byte.SIZE); + if (value == 0) { + return false; + } else if (value == 1) { + return true; + } else { + throw new InvalidSSZTypeException("decoded value is not a boolean"); + } + } + + public BigInteger readUnsignedBigInteger(int bitLength) { + checkArgument(bitLength % 8 == 0, "bitLength must be a multiple of 8"); + int byteLength = bitLength / 8; + ensureBytes( + byteLength, + () -> "SSZ encoded data has insufficient length to read a " + bitLength + "-bit integer"); + return consumeBytes(byteLength).toUnsignedBigInteger(LITTLE_ENDIAN); + } + + public Bytes readHash(int hashLength) { + ensureBytes( + hashLength, + () -> "SSZ encoded data has insufficient length to read a " + hashLength + "-byte hash"); + return consumeBytes(hashLength); + } + + private void ensureBytes(int byteLength, Supplier message) { + if (index == content.size() && byteLength != 0) { + throw new EndOfSSZException(); + } + if (content.size() - index - byteLength < 0) { + throw new InvalidSSZTypeException(message.get()); + } + } +} diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java index 52c551070..9e508d321 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java @@ -1,66 +1,177 @@ package org.ethereum.beacon.ssz.visitor; -import java.nio.ByteOrder; import net.consensys.cava.bytes.Bytes; -import net.consensys.cava.ssz.BytesSSZReaderProxy; -import org.ethereum.beacon.ssz.SSZSerializeException; import org.ethereum.beacon.ssz.access.SSZCompositeAccessor.CompositeInstanceBuilder; import org.ethereum.beacon.ssz.type.SSZBasicType; import org.ethereum.beacon.ssz.type.SSZCompositeType; +import org.ethereum.beacon.ssz.type.SSZContainerType; +import org.ethereum.beacon.ssz.type.SSZListType; +import org.ethereum.beacon.ssz.type.SSZType; import org.ethereum.beacon.ssz.visitor.SSZSimpleDeserializer.DecodeResult; +import org.javatuples.Pair; -public class SSZSimpleDeserializer implements SSZVisitor { - static final int BYTES_PER_LENGTH_PREFIX = 4; +import java.nio.ByteOrder; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; - public static class DecodeResult { - public final Object decodedInstance; - public final int readBytes; +public class SSZSimpleDeserializer implements SSZVisitor> { + static final int BYTES_PER_LENGTH_OFFSET = 4; - public DecodeResult(Object decodedInstance, int readBytes) { - this.decodedInstance = decodedInstance; - this.readBytes = readBytes; + /** + * Decodes basic value + * + * @param sszType Type of field, should be some basic type + * @param param Bytes data / Boolean flag whether to extract body or offset for variable size types. + * If type is with variable size and this flag is set to TRUE, offset is extracted. Otherwise body. + * @return DecodeResult + */ + @Override + public DecodeResult visitBasicValue(SSZBasicType sszType, Pair param) { + Optional offsetLength = extractVariableTypeOffset(sszType, param); + if (offsetLength.isPresent()) { + return offsetLength.get(); + } + + SSZReader reader = new SSZReader(param.getValue0()); + if (sszType.isFixedSize()) { + int readBytes = sszType.getSize(); + return new DecodeResult( + sszType.getAccessor().decode(sszType.getTypeDescriptor(), reader), readBytes, true); + } else { + return new DecodeResult( + sszType.getAccessor().decode(sszType.getTypeDescriptor(), reader), + param.getValue0().size(), + false); } } - @Override - public DecodeResult visitBasicValue(SSZBasicType sszType, Bytes param) { - BytesSSZReaderProxy reader = new BytesSSZReaderProxy(param); - int readBytes = sszType.getSize(); - if (readBytes == -1) { - readBytes = BYTES_PER_LENGTH_PREFIX + deserializeLength(param.slice(0, BYTES_PER_LENGTH_PREFIX)); + private Optional extractVariableTypeOffset(SSZType sszType, Pair param) { + if (Boolean.TRUE.equals(param.getValue1()) && !sszType.isFixedSize()) { + return Optional.of(new DecodeResult( + deserializeLength(param.getValue0().slice(0, BYTES_PER_LENGTH_OFFSET)), + BYTES_PER_LENGTH_OFFSET, + false)); } - return new DecodeResult(sszType.getAccessor().decode(sszType.getTypeDescriptor(), - reader), readBytes); + + return Optional.empty(); } + /** + * Decodes composite value + * + * @param type Type of field, should be some composite type + * @param param Bytes data / Boolean flag whether to extract body or offset for variable size types. + * If type is with variable size and this flag is set to TRUE, offset is extracted. Otherwise body. + * @param childVisitor Visitor which will be used for children + * @return DecodeResult + */ @Override - public DecodeResult visitComposite(SSZCompositeType type, Bytes param, - ChildVisitor childVisitor) { - int pos = 0; - int size; - if (type.isVariableSize()) { - size = deserializeLength(param.slice(0, BYTES_PER_LENGTH_PREFIX)) + BYTES_PER_LENGTH_PREFIX; - pos += BYTES_PER_LENGTH_PREFIX; - } else { - size = type.getSize(); + public DecodeResult visitComposite( + SSZCompositeType type, + Pair param, + ChildVisitor, DecodeResult> childVisitor) { + Optional offsetLength = extractVariableTypeOffset(type, param); + if (offsetLength.isPresent()) { + return offsetLength.get(); } + + int fixedPos = 0; + AtomicInteger variablePartConsumed = new AtomicInteger(0); int idx = 0; - CompositeInstanceBuilder instanceBuilder = type.getAccessor() - .createInstanceBuilder(type.getTypeDescriptor()); - while (pos < size) { - DecodeResult childRes = childVisitor.apply(idx, param.slice(pos)); - instanceBuilder.setChild(idx, childRes.decodedInstance); - pos += childRes.readBytes; + CompositeInstanceBuilder instanceBuilder = + type.getAccessor().createInstanceBuilder(type.getTypeDescriptor()); + boolean isFixedSize = true; + VisitLater visitLater = null; + int maxIndex = calcTypeMaxIndex(type); + int firstOffset = Integer.MAX_VALUE; + while (fixedPos < param.getValue0().size() && idx < maxIndex && fixedPos < firstOffset) { + DecodeResult childRes = + childVisitor.apply(idx, Pair.with(param.getValue0().slice(fixedPos), true)); + if (childRes.isFixedSize) { + instanceBuilder.setChild(idx, childRes.decodedInstance); + } else { + isFixedSize = false; + int offset = (Integer) childRes.decodedInstance; + + // First time we found some item with variable size + if (firstOffset == Integer.MAX_VALUE) { + firstOffset = offset; + } + + final int idxBackup = idx; + if (visitLater != null) { + visitLater.run(offset); + } + visitLater = + new VisitLater( + param.getValue0().slice(offset), + offset, + objects -> { + DecodeResult res = + childVisitor.apply( + idxBackup, + Pair.with(objects.getValue0().slice(0, objects.getValue1()), false)); + variablePartConsumed.addAndGet(res.readBytes); + instanceBuilder.setChild(idxBackup, res.decodedInstance); + }); + } + fixedPos += childRes.readBytes; idx++; } - if (pos != size) { - throw new SSZSerializeException("Error reading serialized composite, expected to read " + size - + " bytes but read " + pos + " bytes"); + + if (visitLater != null) { + visitLater.run(); + } + fixedPos += variablePartConsumed.get(); + + return new DecodeResult(instanceBuilder.build(), fixedPos, isFixedSize); + } + + private int calcTypeMaxIndex(SSZType type) { + int maxIndex = Integer.MAX_VALUE; + if (type instanceof SSZContainerType) { + maxIndex = ((SSZContainerType) type).getChildTypes().size(); + } else if (type instanceof SSZListType && ((SSZListType) type).isVector()) { + maxIndex = ((SSZListType) type).getVectorLength(); } - return new DecodeResult(instanceBuilder.build(), pos); + + return maxIndex; } private int deserializeLength(Bytes lenBytes) { return lenBytes.toInt(ByteOrder.LITTLE_ENDIAN); } + + public static class DecodeResult { + public final Object decodedInstance; + public final int readBytes; + public final boolean isFixedSize; + + public DecodeResult(Object decodedInstance, int readBytes, boolean isFixedSize) { + this.decodedInstance = decodedInstance; + this.readBytes = readBytes; + this.isFixedSize = isFixedSize; + } + } + + class VisitLater { + private final Bytes input; + private final int inputOffset; + private final Consumer> runLater; + + public VisitLater(Bytes input, int inputOffset, Consumer> runLater) { + this.input = input; + this.inputOffset = inputOffset; + this.runLater = runLater; + } + + public void run(int end) { + runLater.accept(Pair.with(input, end - inputOffset)); + } + + public void run() { + runLater.accept(Pair.with(input, input.size())); + } + } } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java index a21189a02..b3699acbc 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java @@ -1,12 +1,9 @@ package org.ethereum.beacon.ssz.visitor; -import static org.ethereum.beacon.ssz.visitor.SSZSimpleDeserializer.BYTES_PER_LENGTH_PREFIX; import static tech.pegasys.artemis.util.bytes.BytesValue.concat; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; -import java.util.function.BiFunction; import java.util.function.Function; import org.ethereum.beacon.ssz.type.SSZBasicType; import org.ethereum.beacon.ssz.type.SSZCompositeType; @@ -129,7 +126,7 @@ public Hash32 getZeroHash(int distanceFromBottom) { return zeroHashes[distanceFromBottom]; } - BytesValue serializeLength(long len) { - return concat(BytesValues.ofUnsignedIntLittleEndian(len), BytesValue.wrap(new byte[Hash32.SIZE - BYTES_PER_LENGTH_PREFIX])); + static BytesValue serializeLength(long len) { + return concat(BytesValues.ofUnsignedIntLittleEndian(len), BytesValue.wrap(new byte[Hash32.SIZE - Integer.BYTES])); } } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java index 42f8929e0..1c912dbc5 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java @@ -1,9 +1,5 @@ package org.ethereum.beacon.ssz.visitor; -import java.io.ByteArrayOutputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.function.BiFunction; import org.ethereum.beacon.ssz.SSZSerializeException; import org.ethereum.beacon.ssz.type.SSZBasicType; import org.ethereum.beacon.ssz.type.SSZCompositeType; @@ -11,90 +7,127 @@ import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.bytes.BytesValues; -public class SSZSimpleSerializer implements SSZVisitor { - - public static class SSZSerializerResult { - BytesValue serializedBody; - BytesValue serializedLength; - - public SSZSerializerResult(BytesValue serializedBody) { - this.serializedBody = serializedBody; - this.serializedLength = BytesValue.EMPTY; - } - - public SSZSerializerResult(BytesValue serializedBody, - BytesValue serializedLength) { - this.serializedBody = serializedBody; - this.serializedLength = serializedLength; - } - - public BytesValue getSerializedBody() { - return serializedBody; - } +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.List; - public BytesValue getSerializedLength() { - return serializedLength; - } +import static org.ethereum.beacon.ssz.visitor.SSZSimpleDeserializer.BYTES_PER_LENGTH_OFFSET; - public BytesValue getSerialized() { - return BytesValue.concat(getSerializedLength(), getSerializedBody()); - } +public class SSZSimpleSerializer + implements SSZVisitor { + static final int FIXED_LENGTH = -1; - public boolean isFixedSize() { - return serializedLength.isEmpty(); - } + private static BytesValue serializeLength(long len) { + return BytesValues.ofUnsignedIntLittleEndian(len); } @Override public SSZSerializerResult visitBasicValue(SSZBasicType type, Object value) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); type.getAccessor().encode(value, type.getTypeDescriptor(), baos); - return new SSZSerializerResult(BytesValue.wrap(baos.toByteArray())); + return new SSZSerializerResult(BytesValue.wrap(baos.toByteArray()), type.isFixedSize() ? FIXED_LENGTH : baos.size()); } @Override - public SSZSerializerResult visitList(SSZListType type, Object param, - ChildVisitor childVisitor) { + public SSZSerializerResult visitList( + SSZListType type, Object param, ChildVisitor childVisitor) { if (type.isVector()) { if (type.getChildrenCount(param) != type.getVectorLength()) { - throw new SSZSerializeException("Vector type length doesn't match actual list length: " - + type.getVectorLength() + " != " + type.getChildrenCount(param) + " for " + type.toStringHelper()); + throw new SSZSerializeException( + "Vector type length doesn't match actual list length: " + + type.getVectorLength() + + " != " + + type.getChildrenCount(param) + + " for " + + type.toStringHelper()); } } return visitComposite(type, param, childVisitor); } @Override - public SSZSerializerResult visitSubList(SSZListType type, Object param, - int startIdx, int len, ChildVisitor childVisitor) { + public SSZSerializerResult visitSubList( + SSZListType type, + Object param, + int startIdx, + int len, + ChildVisitor childVisitor) { return visitComposite(type, param, childVisitor, startIdx, len); } @Override - public SSZSerializerResult visitComposite(SSZCompositeType type, Object rawValue, + public SSZSerializerResult visitComposite( + SSZCompositeType type, + Object rawValue, ChildVisitor childVisitor) { return visitComposite(type, rawValue, childVisitor, 0, type.getChildrenCount(rawValue)); } - private SSZSerializerResult visitComposite(SSZCompositeType type, Object rawValue, - ChildVisitor childVisitor, int startIdx, int len) { - List childSerializations = new ArrayList<>(); + private SSZSerializerResult visitComposite( + SSZCompositeType type, + Object rawValue, + ChildVisitor childVisitor, + int startIdx, + int len) { + + List childSerializations = new ArrayList<>(); boolean fixedSize = type.isFixedSize(); - int length = 0; for (int i = startIdx; i < startIdx + len; i++) { SSZSerializerResult res = childVisitor.apply(i, type.getChild(rawValue, i)); - childSerializations.add(res.serializedLength); - childSerializations.add(res.serializedBody); - fixedSize &= res.isFixedSize(); - length += res.serializedBody.size() + res.serializedLength.size(); + childSerializations.add(res); + } + + int currentOffset = + childSerializations.stream() + .mapToInt(r -> r.isFixedSize() ? r.serializedBody.size() : BYTES_PER_LENGTH_OFFSET) + .sum(); + BytesValue composite = BytesValue.EMPTY; + + // Fixed part + for (SSZSerializerResult s : childSerializations) { + composite = + composite.concat(s.isFixedSize() ? s.serializedBody : serializeLength(currentOffset)); + if (!s.isFixedSize()) { + currentOffset = currentOffset + s.variableLength; + } + } + + // Variable part + for (SSZSerializerResult s : childSerializations) { + if (s.isFixedSize()) { + continue; + } + composite = composite.concat(s.serializedBody); } - return new SSZSerializerResult(BytesValue.concat(childSerializations), - fixedSize ? BytesValue.EMPTY : serializeLength(length)); + return new SSZSerializerResult(composite, fixedSize ? FIXED_LENGTH : currentOffset); } - private BytesValue serializeLength(int len) { - return BytesValues.ofUnsignedIntLittleEndian(len); + public static class SSZSerializerResult { + final BytesValue serializedBody; + final int variableLength; + + public SSZSerializerResult(BytesValue serializedBody) { + this.serializedBody = serializedBody; + this.variableLength = FIXED_LENGTH; + } + + public SSZSerializerResult(BytesValue serializedBody, int variableLength) { + this.serializedBody = serializedBody; + this.variableLength = variableLength; + } + + public BytesValue getSerializedBody() { + return serializedBody; + } + + public int getVariableLength() { + return variableLength; + } + + public boolean isFixedSize() { + return variableLength == FIXED_LENGTH; + } } } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZWriter.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZWriter.java new file mode 100644 index 000000000..133b9447a --- /dev/null +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZWriter.java @@ -0,0 +1,171 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package org.ethereum.beacon.ssz.visitor; + +import net.consensys.cava.bytes.Bytes; +import net.consensys.cava.ssz.InvalidSSZTypeException; + +import java.math.BigInteger; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.nio.charset.StandardCharsets.UTF_8; + +/** Writer with some adoption of {@link net.consensys.cava.ssz.SSZ} code */ +public class SSZWriter { + + private static final Bytes TRUE = Bytes.of((byte) 1); + private static final Bytes FALSE = Bytes.of((byte) 0); + + private SSZWriter() {} + + /** + * Encode {@link Bytes}. + * + * @param value The value to encode. + * @return The SSZ encoding in a {@link Bytes} value. + */ + public static Bytes encodeBytes(Bytes value) { + return Bytes.wrap(value); + } + + /** + * Encode fixed-size bytes value as ssz element + * + * @param bytes Bytes data + * @param byteLength Byte length of ssz element + * @return the SSZ encoding in a {@link Bytes} value + */ + public static Bytes encodeBytes(Bytes bytes, int byteLength) { + if (bytes.size() != byteLength) { + throw new InvalidSSZTypeException( + String.format( + "Error attempt to encode bytes data with length of %s to ssz element with fixed length %s. Lengths must match.", + bytes.size(), byteLength)); + } + return bytes; + } + + /** + * Encode a value to a {@link Bytes} value. + * + * @param value The value to encode. + * @return The SSZ encoding in a {@link Bytes} value. + */ + public static Bytes encodeByteArray(byte[] value) { + return encodeBytes(Bytes.wrap(value)); + } + + /** + * Encode a string to a {@link Bytes} value. + * + * @param str The string to encode. + * @return The SSZ encoding in a {@link Bytes} value. + */ + public static Bytes encodeString(String str) { + return encodeByteArray(str.getBytes(UTF_8)); + } + + /** + * Encode a big integer to a {@link Bytes} value. + * + * @param value the big integer to encode + * @param bitLength the bit length of the integer value (must be a multiple of 8) + * @return the SSZ encoding in a {@link Bytes} value + * @throws IllegalArgumentException if the value is too large for the specified {@code bitLength} + */ + public static Bytes encodeBigInteger(BigInteger value, int bitLength) { + return Bytes.wrap(encodeBigIntegerToByteArray(value, bitLength)); + } + + private static byte[] encodeBigIntegerToByteArray(BigInteger value, int bitLength) { + checkArgument(bitLength % 8 == 0, "bitLength must be a multiple of 8"); + + byte[] bytes = value.toByteArray(); + int valueBytes = bytes.length; + int offset = 0; + if (value.signum() >= 0 && bytes[0] == 0) { + valueBytes = bytes.length - 1; + offset = 1; + } + + int byteLength = bitLength / 8; + checkArgument(valueBytes <= byteLength, "value is too large for the desired bitLength"); + + byte[] encoded; + if (valueBytes == byteLength && offset == 0) { + encoded = bytes; + } else { + encoded = new byte[byteLength]; + int padLength = byteLength - valueBytes; + System.arraycopy(bytes, offset, encoded, padLength, valueBytes); + if (value.signum() < 0) { + // Extend the two's-compliment integer by setting all leading bits to 1. + for (int i = 0; i < padLength; i++) { + encoded[i] = (byte) 0xFF; + } + } + } + // reverse the array to make it little endian + for (int i = 0; i < (encoded.length / 2); i++) { + byte swapped = encoded[i]; + encoded[i] = encoded[encoded.length - i - 1]; + encoded[encoded.length - i - 1] = swapped; + } + return encoded; + } + + /** + * Encode an unsigned long integer to a {@link Bytes} value. + * + *

Note that {@code value} is a native signed long, but will be interpreted as an unsigned + * value. + * + * @param value the long to encode + * @param bitLength the bit length of the integer value (must be a multiple of 8) + * @return the SSZ encoding in a {@link Bytes} value + * @throws IllegalArgumentException if the value is too large for the specified {@code bitLength} + */ + public static Bytes encodeULong(long value, int bitLength) { + return Bytes.wrap(encodeULongToByteArray(value, bitLength)); + } + + private static byte[] encodeULongToByteArray(long value, int bitLength) { + checkArgument(bitLength % 8 == 0, "bitLength must be a multiple of 8"); + + int zeros = Long.numberOfLeadingZeros(value); + int valueBytes = 8 - (zeros / 8); + checkArgument(zeros == 0 || value >= 0, "Value must be positive or zero"); + + int byteLength = bitLength / 8; + checkArgument(valueBytes <= byteLength, "value is too large for the desired bitLength"); + + byte[] encoded = new byte[byteLength]; + + int shift = 0; + for (int i = 0; i < valueBytes; i++) { + encoded[i] = (byte) ((value >> shift) & 0xFF); + shift += 8; + } + return encoded; + } + + /** + * Encode a boolean to a {@link Bytes} value. + * + * @param value the boolean to encode + * @return the SSZ encoding in a {@link Bytes} value + */ + public static Bytes encodeBoolean(boolean value) { + return value ? TRUE : FALSE; + } +} From 107726202424a246a6a48c512e3dcabf2f863d0b Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 14 May 2019 18:20:54 +0300 Subject: [PATCH 05/15] ssz: refactor SSZSimpleSerializer to make cleaner return --- .../ssz/visitor/SSZSimpleSerializer.java | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java index 1c912dbc5..16d8bf6a5 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java @@ -15,8 +15,6 @@ public class SSZSimpleSerializer implements SSZVisitor { - static final int FIXED_LENGTH = -1; - private static BytesValue serializeLength(long len) { return BytesValues.ofUnsignedIntLittleEndian(len); } @@ -25,7 +23,8 @@ private static BytesValue serializeLength(long len) { public SSZSerializerResult visitBasicValue(SSZBasicType type, Object value) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); type.getAccessor().encode(value, type.getTypeDescriptor(), baos); - return new SSZSerializerResult(BytesValue.wrap(baos.toByteArray()), type.isFixedSize() ? FIXED_LENGTH : baos.size()); + return new SSZSerializerResult( + BytesValue.wrap(baos.toByteArray()), baos.size(), type.isFixedSize()); } @Override @@ -89,7 +88,7 @@ private SSZSerializerResult visitComposite( composite = composite.concat(s.isFixedSize() ? s.serializedBody : serializeLength(currentOffset)); if (!s.isFixedSize()) { - currentOffset = currentOffset + s.variableLength; + currentOffset = currentOffset + s.serializedLength; } } @@ -101,33 +100,30 @@ private SSZSerializerResult visitComposite( composite = composite.concat(s.serializedBody); } - return new SSZSerializerResult(composite, fixedSize ? FIXED_LENGTH : currentOffset); + return new SSZSerializerResult(composite, currentOffset, fixedSize); } public static class SSZSerializerResult { final BytesValue serializedBody; - final int variableLength; - - public SSZSerializerResult(BytesValue serializedBody) { - this.serializedBody = serializedBody; - this.variableLength = FIXED_LENGTH; - } + final int serializedLength; + final boolean fixedSize; - public SSZSerializerResult(BytesValue serializedBody, int variableLength) { + public SSZSerializerResult(BytesValue serializedBody, int serializedLength, boolean fixedSize) { this.serializedBody = serializedBody; - this.variableLength = variableLength; + this.serializedLength = serializedLength; + this.fixedSize = fixedSize; } public BytesValue getSerializedBody() { return serializedBody; } - public int getVariableLength() { - return variableLength; + public int getSerializedLength() { + return serializedLength; } public boolean isFixedSize() { - return variableLength == FIXED_LENGTH; + return fixedSize; } } } From ecc44f1032b7f8ce20320a5861f708a292a820d9 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 14 May 2019 19:50:11 +0300 Subject: [PATCH 06/15] ssz: make hasher run independent of SOS-serialize --- .../ethereum/beacon/ssz/HasherSerializer.java | 58 ++++++++++ .../org/ethereum/beacon/ssz/SSZBuilder.java | 14 ++- .../ssz/visitor/SSZHasherSerializer.java | 105 ++++++++++++++++++ .../ssz/visitor/SSZIncrementalHasher.java | 6 +- .../beacon/ssz/visitor/SSZSimpleHasher.java | 10 +- .../beacon/ssz/SSZIncrementalTest.java | 14 +-- 6 files changed, 189 insertions(+), 18 deletions(-) create mode 100644 ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java create mode 100644 ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZHasherSerializer.java diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java new file mode 100644 index 000000000..8230ff638 --- /dev/null +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java @@ -0,0 +1,58 @@ +package org.ethereum.beacon.ssz; + +import net.consensys.cava.ssz.SSZException; +import org.ethereum.beacon.ssz.access.SSZField; +import org.ethereum.beacon.ssz.type.SSZListType; +import org.ethereum.beacon.ssz.type.SSZType; +import org.ethereum.beacon.ssz.type.TypeResolver; +import org.ethereum.beacon.ssz.visitor.SSZHasherSerializer; +import org.ethereum.beacon.ssz.visitor.SSZHasherSerializer.SSZHasherSerializerResult; +import org.ethereum.beacon.ssz.visitor.SSZVisitorHandler; +import org.ethereum.beacon.ssz.visitor.SSZVisitorHost; + +import javax.annotation.Nullable; + +/** SSZ hasher serializer */ +public class HasherSerializer + implements BytesSerializer, SSZVisitorHandler { + + private final SSZVisitorHost sszVisitorHost; + private final TypeResolver typeResolver; + + public HasherSerializer(SSZVisitorHost sszVisitorHost, TypeResolver typeResolver) { + this.sszVisitorHost = sszVisitorHost; + this.typeResolver = typeResolver; + } + + /** + * Serializes input to byte[] data + * + * @param inputObject input value + * @param inputClazz Class of value + * @return SSZ serialization + */ + @Override + public byte[] encode(@Nullable C inputObject, Class inputClazz) { + return visit(inputObject, inputClazz).getSerializedBody().getArrayUnsafe(); + } + + private SSZHasherSerializerResult visit(C input, Class clazz) { + return visitAny(typeResolver.resolveSSZType(new SSZField(clazz)), input); + } + + @Override + public SSZHasherSerializerResult visitAny(SSZType sszType, Object value) { + return sszVisitorHost.handleAny(sszType, value, new SSZHasherSerializer()); + } + + @Override + public SSZHasherSerializerResult visitList( + SSZListType descriptor, Object listValue, int startIdx, int len) { + return sszVisitorHost.handleSubList( + descriptor, listValue, startIdx, len, new SSZHasherSerializer()); + } + + public C decode(byte[] data, Class clazz) { + throw new SSZException("Hasher doesn't support decoding!"); + } +} diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZBuilder.java b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZBuilder.java index f3968a0ad..e0a8a1d3f 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZBuilder.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZBuilder.java @@ -264,13 +264,23 @@ public SSZSerializer buildSerializer() { return new SSZSerializer(visitorHost, typeResolver); } + /** + * Finalizes build of {@link HasherSerializer} with builder + * + * @return {@link HasherSerializer} + */ + public HasherSerializer buildHasherSerializer() { + buildCommon(); + return new HasherSerializer(visitorHost, typeResolver); + } + public SSZHasher buildHasher(Function hashFunction) { buildCommon(); SSZVisitor hasherVisitor; if (incrementalHasher) { - hasherVisitor = new SSZIncrementalHasher(buildSerializer(), hashFunction, sszHashBytesPerChunk); + hasherVisitor = new SSZIncrementalHasher(buildHasherSerializer(), hashFunction, sszHashBytesPerChunk); } else { - hasherVisitor = new SSZSimpleHasher(buildSerializer(), hashFunction, sszHashBytesPerChunk); + hasherVisitor = new SSZSimpleHasher(buildHasherSerializer(), hashFunction, sszHashBytesPerChunk); } return new SSZHasher(typeResolver, visitorHost, hasherVisitor); } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZHasherSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZHasherSerializer.java new file mode 100644 index 000000000..756eb4f5e --- /dev/null +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZHasherSerializer.java @@ -0,0 +1,105 @@ +package org.ethereum.beacon.ssz.visitor; + +import org.ethereum.beacon.ssz.SSZSerializeException; +import org.ethereum.beacon.ssz.type.SSZBasicType; +import org.ethereum.beacon.ssz.type.SSZCompositeType; +import org.ethereum.beacon.ssz.type.SSZListType; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.bytes.BytesValues; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class SSZHasherSerializer implements SSZVisitor { + + public static class SSZHasherSerializerResult { + BytesValue serializedBody; + BytesValue serializedLength; + + public SSZHasherSerializerResult(BytesValue serializedBody) { + this.serializedBody = serializedBody; + this.serializedLength = BytesValue.EMPTY; + } + + public SSZHasherSerializerResult(BytesValue serializedBody, + BytesValue serializedLength) { + this.serializedBody = serializedBody; + this.serializedLength = serializedLength; + } + + public BytesValue getSerializedBody() { + return serializedBody; + } + + public BytesValue getSerializedLength() { + return serializedLength; + } + + public BytesValue getSerialized() { + return BytesValue.concat(getSerializedLength(), getSerializedBody()); + } + + public boolean isFixedSize() { + return serializedLength.isEmpty(); + } + } + + @Override + public SSZHasherSerializerResult visitBasicValue(SSZBasicType type, Object value) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + type.getAccessor().encode(value, type.getTypeDescriptor(), baos); + BytesValue res = BytesValue.wrap(baos.toByteArray()); + if (!type.isFixedSize()) { + res = serializeLength(res.size()).concat(res); + } + return new SSZHasherSerializerResult(res); + } + + @Override + public SSZHasherSerializerResult visitList(SSZListType type, Object param, + ChildVisitor childVisitor) { + + if (type.isVector()) { + if (type.getChildrenCount(param) != type.getVectorLength()) { + throw new SSZSerializeException("Vector type length doesn't match actual list length: " + + type.getVectorLength() + " != " + type.getChildrenCount(param) + " for " + type.toStringHelper()); + } + } + return visitComposite(type, param, childVisitor); + } + + @Override + public SSZHasherSerializerResult visitSubList(SSZListType type, Object param, + int startIdx, int len, ChildVisitor childVisitor) { + return visitComposite(type, param, childVisitor, startIdx, len); + } + + @Override + public SSZHasherSerializerResult visitComposite(SSZCompositeType type, Object rawValue, + ChildVisitor childVisitor) { + return visitComposite(type, rawValue, childVisitor, 0, type.getChildrenCount(rawValue)); + } + + private SSZHasherSerializerResult visitComposite(SSZCompositeType type, Object rawValue, + ChildVisitor childVisitor, int startIdx, int len) { + List childSerializations = new ArrayList<>(); + boolean fixedSize = type.isFixedSize(); + int length = 0; + for (int i = startIdx; i < startIdx + len; i++) { + SSZHasherSerializerResult res = childVisitor.apply(i, type.getChild(rawValue, i)); + childSerializations.add(res.serializedLength); + childSerializations.add(res.serializedBody); + fixedSize &= res.isFixedSize(); + length += res.serializedBody.size() + res.serializedLength.size(); + } + + return new SSZHasherSerializerResult(BytesValue.concat(childSerializations), + fixedSize ? BytesValue.EMPTY : serializeLength(length)); + } + + private BytesValue serializeLength(int len) { + return BytesValues.ofUnsignedIntLittleEndian(len); + } +} diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java index 4dea95c89..fc66f57be 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java @@ -11,14 +11,12 @@ import java.util.TreeSet; import java.util.function.BiFunction; import java.util.function.Function; -import java.util.stream.Collectors; import org.ethereum.beacon.ssz.incremental.ObservableComposite; import org.ethereum.beacon.ssz.incremental.UpdateListener; import org.ethereum.beacon.ssz.type.SSZCompositeType; import org.ethereum.beacon.ssz.type.SSZListType; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; +import org.ethereum.beacon.ssz.visitor.SSZHasherSerializer.SSZHasherSerializerResult; import tech.pegasys.artemis.ethereum.core.Hash32; -import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; public class SSZIncrementalHasher extends SSZSimpleHasher { @@ -51,7 +49,7 @@ public UpdateListener fork() { } public SSZIncrementalHasher( - SSZVisitorHandler serializer, + SSZVisitorHandler serializer, Function hashFunction, int bytesPerChunk) { super(serializer, hashFunction, bytesPerChunk); } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java index b3699acbc..7516f8895 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java @@ -8,7 +8,7 @@ import org.ethereum.beacon.ssz.type.SSZBasicType; import org.ethereum.beacon.ssz.type.SSZCompositeType; import org.ethereum.beacon.ssz.type.SSZListType; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; +import org.ethereum.beacon.ssz.visitor.SSZHasherSerializer.SSZHasherSerializerResult; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -17,12 +17,12 @@ public class SSZSimpleHasher implements SSZVisitor { private final Hash32[] zeroHashes = new Hash32[32]; - final SSZVisitorHandler serializer; + final SSZVisitorHandler serializer; final Function hashFunction; final int bytesPerChunk; public SSZSimpleHasher( - SSZVisitorHandler serializer, + SSZVisitorHandler serializer, Function hashFunction, int bytesPerChunk) { this.serializer = serializer; this.hashFunction = hashFunction; @@ -31,7 +31,7 @@ public SSZSimpleHasher( @Override public MerkleTrie visitBasicValue(SSZBasicType descriptor, Object value) { - SSZSerializerResult sszSerializerResult = serializer.visitAny(descriptor, value); + SSZHasherSerializerResult sszSerializerResult = serializer.visitAny(descriptor, value); return merkleize(pack(sszSerializerResult.serializedBody)); } @@ -43,7 +43,7 @@ public MerkleTrie visitComposite(SSZCompositeType type, Object rawValue, if (type.getChildrenCount(rawValue) == 0) { // empty chunk list } else if (type.isList() && ((SSZListType) type).getElementType().isBasicType()) { - SSZSerializerResult sszSerializerResult = serializer.visitAny(type, rawValue); + SSZHasherSerializerResult sszSerializerResult = serializer.visitAny(type, rawValue); chunks = pack(sszSerializerResult.serializedBody); } else { diff --git a/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java b/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java index 6e9bcc0f6..58bc5761d 100644 --- a/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java +++ b/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java @@ -103,7 +103,7 @@ public void testHashIncremental1() throws Exception { TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); + HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, 32); @@ -212,7 +212,7 @@ public void testReadList() { TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); + HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, 32); @@ -350,7 +350,7 @@ public void testPackedList1() { TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); + HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, @@ -441,7 +441,7 @@ private void listRandomTest(WriteList list, Supplier numSuppl TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); + HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, @@ -512,7 +512,7 @@ public void testPackedListRemove1() { TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); + HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, 32); @@ -587,7 +587,7 @@ public void testListRemove() { TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); + HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, 32); @@ -749,7 +749,7 @@ public void testComplexStruct() { TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); + HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, 32); From 10c2543de4db796163a8b4223675f169704e5b27 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 14 May 2019 21:27:25 +0300 Subject: [PATCH 07/15] types: fix stackoverflow for concatenated bytes value --- .../artemis/util/bytes/BytesValue.java | 110 ++++++++++-------- 1 file changed, 64 insertions(+), 46 deletions(-) diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/BytesValue.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/BytesValue.java index 74b2d10da..cce54e214 100644 --- a/types/src/main/java/tech/pegasys/artemis/util/bytes/BytesValue.java +++ b/types/src/main/java/tech/pegasys/artemis/util/bytes/BytesValue.java @@ -13,16 +13,16 @@ package tech.pegasys.artemis.util.bytes; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkElementIndex; - -import java.security.MessageDigest; - import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import io.vertx.core.buffer.Buffer; + +import java.security.MessageDigest; import java.util.List; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkElementIndex; + /** * A value made of bytes. * @@ -92,49 +92,63 @@ static BytesValue wrap(byte[] value, int offset, int length) { * @return A value representing a view over the concatenation of {@code v1} and {@code v2}. */ static BytesValue wrap(BytesValue v1, BytesValue v2) { - return new AbstractBytesValue() { - @Override - public int size() { - return v1.size() + v2.size(); - } + return new MergedBytesValue(v1, v2); + } - @Override - public byte get(int i) { - checkElementIndex(i, size()); - return i < v1.size() ? v1.get(i) : v2.get(i - v1.size()); - } + class MergedBytesValue extends AbstractBytesValue { + private final BytesValue v1; + private final BytesValue v2; + private final int generation; + + private MergedBytesValue(BytesValue v1, BytesValue v2) { + this.v1 = v1; + this.v2 = v2; + int v1Generation = v1 instanceof MergedBytesValue ? ((MergedBytesValue) v1).generation : 1; + int v2Generation = v2 instanceof MergedBytesValue ? ((MergedBytesValue) v2).generation : 1; + this.generation = Math.max(v1Generation, v2Generation) + 1; + } - @Override - public BytesValue slice(int i, int length) { - if (i == 0 && length == size()) - return this; - if (length == 0) - return BytesValue.EMPTY; - - checkElementIndex(i, size()); - checkArgument(i + length <= size(), - "Provided length %s is too big: the value has size %s and has only %s bytes from %s", - length, size(), size() - i, i); - - if (i + length < v1.size()) - return v1.slice(i, length); - - if (i >= v1.size()) - return v2.slice(i - v1.size(), length); - - MutableBytesValue res = MutableBytesValue.create(length); - int lengthInV1 = v1.size() - i; - v1.slice(i, lengthInV1).copyTo(res, 0); - v2.slice(0, length - lengthInV1).copyTo(res, lengthInV1); - return res; - } + @Override + public int size() { + return v1.size() + v2.size(); + } - @Override - public void update(MessageDigest digest) { - v1.update(digest); - v2.update(digest); - } - }; + @Override + public byte get(int i) { + checkElementIndex(i, size()); + return i < v1.size() ? v1.get(i) : v2.get(i - v1.size()); + } + + @Override + public BytesValue slice(int i, int length) { + if (i == 0 && length == size()) + return this; + if (length == 0) + return BytesValue.EMPTY; + + checkElementIndex(i, size()); + checkArgument(i + length <= size(), + "Provided length %s is too big: the value has size %s and has only %s bytes from %s", + length, size(), size() - i, i); + + if (i + length < v1.size()) + return v1.slice(i, length); + + if (i >= v1.size()) + return v2.slice(i - v1.size(), length); + + MutableBytesValue res = MutableBytesValue.create(length); + int lengthInV1 = v1.size() - i; + v1.slice(i, lengthInV1).copyTo(res, 0); + v2.slice(0, length - lengthInV1).copyTo(res, lengthInV1); + return res; + } + + @Override + public void update(MessageDigest digest) { + v1.update(digest); + v2.update(digest); + } } /** @@ -145,7 +159,11 @@ public void update(MessageDigest digest) { * depending on the target value size */ static BytesValue concat(BytesValue v1, BytesValue v2) { - if (v1.size() + v2.size() < 512) { + int MAX_GENERATION = 32; + if (v1.size() + v2.size() < 512 + || (v1 instanceof MergedBytesValue && ((MergedBytesValue) v1).generation > MAX_GENERATION) + || (v2 instanceof MergedBytesValue && ((MergedBytesValue) v2).generation > MAX_GENERATION) + ) { // it should be generally cheaper for further usage byte[] bb = new byte[v1.size() + v2.size()]; byte[] arr1 = v1.getArrayUnsafe(); From 19321252642ebaf9548587c09510f9526266449f Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Tue, 14 May 2019 21:38:45 +0300 Subject: [PATCH 08/15] types: remove concatenate optimization which was measured to be too slow --- .../artemis/util/bytes/BytesValue.java | 92 ++----------------- .../artemis/util/bytes/BytesValueTest.java | 5 +- 2 files changed, 12 insertions(+), 85 deletions(-) diff --git a/types/src/main/java/tech/pegasys/artemis/util/bytes/BytesValue.java b/types/src/main/java/tech/pegasys/artemis/util/bytes/BytesValue.java index cce54e214..b9a774322 100644 --- a/types/src/main/java/tech/pegasys/artemis/util/bytes/BytesValue.java +++ b/types/src/main/java/tech/pegasys/artemis/util/bytes/BytesValue.java @@ -80,100 +80,28 @@ static BytesValue wrap(byte[] value, int offset, int length) { } /** - * Wraps two other value into a concatenated view. - * - *

- * Note that the values are not copied, only wrapped, and thus any future update to {@code v1} or - * {@code v2} will be reflected in the returned value. If copying the inputs is desired instead, - * please use {@link BytesValues#concatenate(BytesValue...)}. + * Wraps two other value into the new concatenated value. * * @param v1 The first value to wrap. * @param v2 The second value to wrap. - * @return A value representing a view over the concatenation of {@code v1} and {@code v2}. + * @return v1 concatenated with v2. */ static BytesValue wrap(BytesValue v1, BytesValue v2) { - return new MergedBytesValue(v1, v2); - } - - class MergedBytesValue extends AbstractBytesValue { - private final BytesValue v1; - private final BytesValue v2; - private final int generation; - - private MergedBytesValue(BytesValue v1, BytesValue v2) { - this.v1 = v1; - this.v2 = v2; - int v1Generation = v1 instanceof MergedBytesValue ? ((MergedBytesValue) v1).generation : 1; - int v2Generation = v2 instanceof MergedBytesValue ? ((MergedBytesValue) v2).generation : 1; - this.generation = Math.max(v1Generation, v2Generation) + 1; - } - - @Override - public int size() { - return v1.size() + v2.size(); - } - - @Override - public byte get(int i) { - checkElementIndex(i, size()); - return i < v1.size() ? v1.get(i) : v2.get(i - v1.size()); - } - - @Override - public BytesValue slice(int i, int length) { - if (i == 0 && length == size()) - return this; - if (length == 0) - return BytesValue.EMPTY; - - checkElementIndex(i, size()); - checkArgument(i + length <= size(), - "Provided length %s is too big: the value has size %s and has only %s bytes from %s", - length, size(), size() - i, i); - - if (i + length < v1.size()) - return v1.slice(i, length); - - if (i >= v1.size()) - return v2.slice(i - v1.size(), length); - - MutableBytesValue res = MutableBytesValue.create(length); - int lengthInV1 = v1.size() - i; - v1.slice(i, lengthInV1).copyTo(res, 0); - v2.slice(0, length - lengthInV1).copyTo(res, lengthInV1); - return res; - } - - @Override - public void update(MessageDigest digest) { - v1.update(digest); - v2.update(digest); - } + return v1.concat(v2); } /** * Concatenates two values * - * The resulting value would be either backed by supplied values with - * {@link #wrap(BytesValue, BytesValue)} or create a copy of backing bytes - * depending on the target value size + * The resulting value would be a copy of backing bytes */ static BytesValue concat(BytesValue v1, BytesValue v2) { - int MAX_GENERATION = 32; - if (v1.size() + v2.size() < 512 - || (v1 instanceof MergedBytesValue && ((MergedBytesValue) v1).generation > MAX_GENERATION) - || (v2 instanceof MergedBytesValue && ((MergedBytesValue) v2).generation > MAX_GENERATION) - ) { - // it should be generally cheaper for further usage - byte[] bb = new byte[v1.size() + v2.size()]; - byte[] arr1 = v1.getArrayUnsafe(); - byte[] arr2 = v2.getArrayUnsafe(); - System.arraycopy(arr1, 0, bb, 0, arr1.length); - System.arraycopy(arr2, 0, bb, arr1.length, arr2.length); - return wrap(bb); - } else { - return wrap(v1, v2); - } + byte[] bb = new byte[v1.size() + v2.size()]; + byte[] arr1 = v1.getArrayUnsafe(); + byte[] arr2 = v2.getArrayUnsafe(); + System.arraycopy(arr1, 0, bb, 0, arr1.length); + System.arraycopy(arr2, 0, bb, arr1.length, arr2.length); + return wrap(bb); } static BytesValue concat(List vals) { diff --git a/types/src/test/java/tech/pegasys/artemis/util/bytes/BytesValueTest.java b/types/src/test/java/tech/pegasys/artemis/util/bytes/BytesValueTest.java index 4f5e4aa68..762682236 100644 --- a/types/src/test/java/tech/pegasys/artemis/util/bytes/BytesValueTest.java +++ b/types/src/test/java/tech/pegasys/artemis/util/bytes/BytesValueTest.java @@ -173,7 +173,7 @@ private static void assertConcatenatedWrap(byte[] first, byte[] second) { } @Test - public void concatenatedWrapReflectsUpdates() { + public void concatenatedWrapDoesntReflectsUpdates() { byte[] first = new byte[] {1, 2, 3}; byte[] second = new byte[] {4, 5}; byte[] expected1 = new byte[] {1, 2, 3, 4, 5}; @@ -182,8 +182,7 @@ public void concatenatedWrapReflectsUpdates() { first[1] = 42; second[0] = 42; - byte[] expected2 = new byte[] {1, 42, 3, 42, 5}; - assertArrayEquals(expected2, res.extractArray()); + assertArrayEquals(expected1, res.extractArray()); } @Test From 1270448408805f5a4fd434a477c9d2f639174ac5 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 15 May 2019 11:07:09 +0300 Subject: [PATCH 09/15] ssz: change class names so pre-sos implementation match old names --- .../ethereum/beacon/ssz/HasherSerializer.java | 8 +- .../ethereum/beacon/ssz/SSZSerializer.java | 16 +- .../ssz/visitor/SSZHasherSerializer.java | 105 ------------- .../ssz/visitor/SSZIncrementalHasher.java | 2 +- .../beacon/ssz/visitor/SSZSimpleHasher.java | 2 +- .../ssz/visitor/SSZSimpleSerializer.java | 141 +++++++----------- ...erializer.java => SSZSosDeserializer.java} | 4 +- .../beacon/ssz/visitor/SSZSosSerializer.java | 132 ++++++++++++++++ 8 files changed, 206 insertions(+), 204 deletions(-) delete mode 100644 ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZHasherSerializer.java rename ssz/src/main/java/org/ethereum/beacon/ssz/visitor/{SSZSimpleDeserializer.java => SSZSosDeserializer.java} (97%) create mode 100644 ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosSerializer.java diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java index 8230ff638..59fb5d049 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java @@ -5,8 +5,8 @@ import org.ethereum.beacon.ssz.type.SSZListType; import org.ethereum.beacon.ssz.type.SSZType; import org.ethereum.beacon.ssz.type.TypeResolver; -import org.ethereum.beacon.ssz.visitor.SSZHasherSerializer; -import org.ethereum.beacon.ssz.visitor.SSZHasherSerializer.SSZHasherSerializerResult; +import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer; +import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZHasherSerializerResult; import org.ethereum.beacon.ssz.visitor.SSZVisitorHandler; import org.ethereum.beacon.ssz.visitor.SSZVisitorHost; @@ -42,14 +42,14 @@ private SSZHasherSerializerResult visit(C input, Class clazz) { @Override public SSZHasherSerializerResult visitAny(SSZType sszType, Object value) { - return sszVisitorHost.handleAny(sszType, value, new SSZHasherSerializer()); + return sszVisitorHost.handleAny(sszType, value, new SSZSimpleSerializer()); } @Override public SSZHasherSerializerResult visitList( SSZListType descriptor, Object listValue, int startIdx, int len) { return sszVisitorHost.handleSubList( - descriptor, listValue, startIdx, len, new SSZHasherSerializer()); + descriptor, listValue, startIdx, len, new SSZSimpleSerializer()); } public C decode(byte[] data, Class clazz) { diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java index 22a76035e..c6b5fe965 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java @@ -7,16 +7,16 @@ import org.ethereum.beacon.ssz.type.SSZListType; import org.ethereum.beacon.ssz.type.SSZType; import org.ethereum.beacon.ssz.type.TypeResolver; -import org.ethereum.beacon.ssz.visitor.SSZSimpleDeserializer; -import org.ethereum.beacon.ssz.visitor.SSZSimpleDeserializer.DecodeResult; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; +import org.ethereum.beacon.ssz.visitor.SSZSosDeserializer; +import org.ethereum.beacon.ssz.visitor.SSZSosDeserializer.DecodeResult; +import org.ethereum.beacon.ssz.visitor.SSZSosSerializer; +import org.ethereum.beacon.ssz.visitor.SSZSosSerializer.SSZSerializerResult; import org.ethereum.beacon.ssz.visitor.SSZVisitorHandler; import org.ethereum.beacon.ssz.visitor.SSZVisitorHost; import org.javatuples.Pair; /** SSZ serializer/deserializer */ -public class SSZSerializer implements BytesSerializer, SSZVisitorHandler { +public class SSZSerializer implements BytesSerializer, SSZVisitorHandler { private final SSZVisitorHost sszVisitorHost; private final TypeResolver typeResolver; @@ -45,12 +45,12 @@ private SSZSerializerResult visit(C input, Class clazz) { @Override public SSZSerializerResult visitAny(SSZType sszType, Object value) { - return sszVisitorHost.handleAny(sszType, value, new SSZSimpleSerializer()); + return sszVisitorHost.handleAny(sszType, value, new SSZSosSerializer()); } @Override public SSZSerializerResult visitList(SSZListType descriptor, Object listValue, int startIdx, int len) { - return sszVisitorHost.handleSubList(descriptor, listValue, startIdx, len, new SSZSimpleSerializer()); + return sszVisitorHost.handleSubList(descriptor, listValue, startIdx, len, new SSZSosSerializer()); } /** @@ -64,7 +64,7 @@ public C decode(byte[] data, Class clazz) { DecodeResult decodeResult = sszVisitorHost.handleAny( typeResolver.resolveSSZType(new SSZField(clazz)), Pair.with(Bytes.wrap(data), null), - new SSZSimpleDeserializer()); + new SSZSosDeserializer()); if (data.length != decodeResult.readBytes) { throw new SSZSerializeException(String.format("Invalid SSZ encoding, calculated data size %s bytes, while provided %s bytes", decodeResult.readBytes, data.length)); } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZHasherSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZHasherSerializer.java deleted file mode 100644 index 756eb4f5e..000000000 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZHasherSerializer.java +++ /dev/null @@ -1,105 +0,0 @@ -package org.ethereum.beacon.ssz.visitor; - -import org.ethereum.beacon.ssz.SSZSerializeException; -import org.ethereum.beacon.ssz.type.SSZBasicType; -import org.ethereum.beacon.ssz.type.SSZCompositeType; -import org.ethereum.beacon.ssz.type.SSZListType; -import tech.pegasys.artemis.util.bytes.BytesValue; -import tech.pegasys.artemis.util.bytes.BytesValues; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class SSZHasherSerializer implements SSZVisitor { - - public static class SSZHasherSerializerResult { - BytesValue serializedBody; - BytesValue serializedLength; - - public SSZHasherSerializerResult(BytesValue serializedBody) { - this.serializedBody = serializedBody; - this.serializedLength = BytesValue.EMPTY; - } - - public SSZHasherSerializerResult(BytesValue serializedBody, - BytesValue serializedLength) { - this.serializedBody = serializedBody; - this.serializedLength = serializedLength; - } - - public BytesValue getSerializedBody() { - return serializedBody; - } - - public BytesValue getSerializedLength() { - return serializedLength; - } - - public BytesValue getSerialized() { - return BytesValue.concat(getSerializedLength(), getSerializedBody()); - } - - public boolean isFixedSize() { - return serializedLength.isEmpty(); - } - } - - @Override - public SSZHasherSerializerResult visitBasicValue(SSZBasicType type, Object value) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - type.getAccessor().encode(value, type.getTypeDescriptor(), baos); - BytesValue res = BytesValue.wrap(baos.toByteArray()); - if (!type.isFixedSize()) { - res = serializeLength(res.size()).concat(res); - } - return new SSZHasherSerializerResult(res); - } - - @Override - public SSZHasherSerializerResult visitList(SSZListType type, Object param, - ChildVisitor childVisitor) { - - if (type.isVector()) { - if (type.getChildrenCount(param) != type.getVectorLength()) { - throw new SSZSerializeException("Vector type length doesn't match actual list length: " - + type.getVectorLength() + " != " + type.getChildrenCount(param) + " for " + type.toStringHelper()); - } - } - return visitComposite(type, param, childVisitor); - } - - @Override - public SSZHasherSerializerResult visitSubList(SSZListType type, Object param, - int startIdx, int len, ChildVisitor childVisitor) { - return visitComposite(type, param, childVisitor, startIdx, len); - } - - @Override - public SSZHasherSerializerResult visitComposite(SSZCompositeType type, Object rawValue, - ChildVisitor childVisitor) { - return visitComposite(type, rawValue, childVisitor, 0, type.getChildrenCount(rawValue)); - } - - private SSZHasherSerializerResult visitComposite(SSZCompositeType type, Object rawValue, - ChildVisitor childVisitor, int startIdx, int len) { - List childSerializations = new ArrayList<>(); - boolean fixedSize = type.isFixedSize(); - int length = 0; - for (int i = startIdx; i < startIdx + len; i++) { - SSZHasherSerializerResult res = childVisitor.apply(i, type.getChild(rawValue, i)); - childSerializations.add(res.serializedLength); - childSerializations.add(res.serializedBody); - fixedSize &= res.isFixedSize(); - length += res.serializedBody.size() + res.serializedLength.size(); - } - - return new SSZHasherSerializerResult(BytesValue.concat(childSerializations), - fixedSize ? BytesValue.EMPTY : serializeLength(length)); - } - - private BytesValue serializeLength(int len) { - return BytesValues.ofUnsignedIntLittleEndian(len); - } -} diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java index fc66f57be..d0cb25129 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java @@ -15,7 +15,7 @@ import org.ethereum.beacon.ssz.incremental.UpdateListener; import org.ethereum.beacon.ssz.type.SSZCompositeType; import org.ethereum.beacon.ssz.type.SSZListType; -import org.ethereum.beacon.ssz.visitor.SSZHasherSerializer.SSZHasherSerializerResult; +import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZHasherSerializerResult; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.BytesValue; diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java index 7516f8895..a2d07a6fd 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java @@ -8,7 +8,7 @@ import org.ethereum.beacon.ssz.type.SSZBasicType; import org.ethereum.beacon.ssz.type.SSZCompositeType; import org.ethereum.beacon.ssz.type.SSZListType; -import org.ethereum.beacon.ssz.visitor.SSZHasherSerializer.SSZHasherSerializerResult; +import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZHasherSerializerResult; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java index 16d8bf6a5..ed15781da 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java @@ -11,119 +11,94 @@ import java.util.ArrayList; import java.util.List; -import static org.ethereum.beacon.ssz.visitor.SSZSimpleDeserializer.BYTES_PER_LENGTH_OFFSET; +public class SSZSimpleSerializer implements SSZVisitor { -public class SSZSimpleSerializer - implements SSZVisitor { - private static BytesValue serializeLength(long len) { - return BytesValues.ofUnsignedIntLittleEndian(len); + public static class SSZHasherSerializerResult { + BytesValue serializedBody; + BytesValue serializedLength; + + public SSZHasherSerializerResult(BytesValue serializedBody) { + this.serializedBody = serializedBody; + this.serializedLength = BytesValue.EMPTY; + } + + public SSZHasherSerializerResult(BytesValue serializedBody, + BytesValue serializedLength) { + this.serializedBody = serializedBody; + this.serializedLength = serializedLength; + } + + public BytesValue getSerializedBody() { + return serializedBody; + } + + public BytesValue getSerializedLength() { + return serializedLength; + } + + public BytesValue getSerialized() { + return BytesValue.concat(getSerializedLength(), getSerializedBody()); + } + + public boolean isFixedSize() { + return serializedLength.isEmpty(); + } } @Override - public SSZSerializerResult visitBasicValue(SSZBasicType type, Object value) { + public SSZHasherSerializerResult visitBasicValue(SSZBasicType type, Object value) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); type.getAccessor().encode(value, type.getTypeDescriptor(), baos); - return new SSZSerializerResult( - BytesValue.wrap(baos.toByteArray()), baos.size(), type.isFixedSize()); + BytesValue res = BytesValue.wrap(baos.toByteArray()); + if (!type.isFixedSize()) { + res = serializeLength(res.size()).concat(res); + } + return new SSZHasherSerializerResult(res); } @Override - public SSZSerializerResult visitList( - SSZListType type, Object param, ChildVisitor childVisitor) { + public SSZHasherSerializerResult visitList(SSZListType type, Object param, + ChildVisitor childVisitor) { if (type.isVector()) { if (type.getChildrenCount(param) != type.getVectorLength()) { - throw new SSZSerializeException( - "Vector type length doesn't match actual list length: " - + type.getVectorLength() - + " != " - + type.getChildrenCount(param) - + " for " - + type.toStringHelper()); + throw new SSZSerializeException("Vector type length doesn't match actual list length: " + + type.getVectorLength() + " != " + type.getChildrenCount(param) + " for " + type.toStringHelper()); } } return visitComposite(type, param, childVisitor); } @Override - public SSZSerializerResult visitSubList( - SSZListType type, - Object param, - int startIdx, - int len, - ChildVisitor childVisitor) { + public SSZHasherSerializerResult visitSubList(SSZListType type, Object param, + int startIdx, int len, ChildVisitor childVisitor) { return visitComposite(type, param, childVisitor, startIdx, len); } @Override - public SSZSerializerResult visitComposite( - SSZCompositeType type, - Object rawValue, - ChildVisitor childVisitor) { + public SSZHasherSerializerResult visitComposite(SSZCompositeType type, Object rawValue, + ChildVisitor childVisitor) { return visitComposite(type, rawValue, childVisitor, 0, type.getChildrenCount(rawValue)); } - private SSZSerializerResult visitComposite( - SSZCompositeType type, - Object rawValue, - ChildVisitor childVisitor, - int startIdx, - int len) { - - List childSerializations = new ArrayList<>(); + private SSZHasherSerializerResult visitComposite(SSZCompositeType type, Object rawValue, + ChildVisitor childVisitor, int startIdx, int len) { + List childSerializations = new ArrayList<>(); boolean fixedSize = type.isFixedSize(); + int length = 0; for (int i = startIdx; i < startIdx + len; i++) { - SSZSerializerResult res = childVisitor.apply(i, type.getChild(rawValue, i)); - childSerializations.add(res); + SSZHasherSerializerResult res = childVisitor.apply(i, type.getChild(rawValue, i)); + childSerializations.add(res.serializedLength); + childSerializations.add(res.serializedBody); + fixedSize &= res.isFixedSize(); + length += res.serializedBody.size() + res.serializedLength.size(); } - int currentOffset = - childSerializations.stream() - .mapToInt(r -> r.isFixedSize() ? r.serializedBody.size() : BYTES_PER_LENGTH_OFFSET) - .sum(); - BytesValue composite = BytesValue.EMPTY; - - // Fixed part - for (SSZSerializerResult s : childSerializations) { - composite = - composite.concat(s.isFixedSize() ? s.serializedBody : serializeLength(currentOffset)); - if (!s.isFixedSize()) { - currentOffset = currentOffset + s.serializedLength; - } - } - - // Variable part - for (SSZSerializerResult s : childSerializations) { - if (s.isFixedSize()) { - continue; - } - composite = composite.concat(s.serializedBody); - } - - return new SSZSerializerResult(composite, currentOffset, fixedSize); + return new SSZHasherSerializerResult(BytesValue.concat(childSerializations), + fixedSize ? BytesValue.EMPTY : serializeLength(length)); } - public static class SSZSerializerResult { - final BytesValue serializedBody; - final int serializedLength; - final boolean fixedSize; - - public SSZSerializerResult(BytesValue serializedBody, int serializedLength, boolean fixedSize) { - this.serializedBody = serializedBody; - this.serializedLength = serializedLength; - this.fixedSize = fixedSize; - } - - public BytesValue getSerializedBody() { - return serializedBody; - } - - public int getSerializedLength() { - return serializedLength; - } - - public boolean isFixedSize() { - return fixedSize; - } + private BytesValue serializeLength(int len) { + return BytesValues.ofUnsignedIntLittleEndian(len); } } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosDeserializer.java similarity index 97% rename from ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java rename to ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosDeserializer.java index 9e508d321..34c094701 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleDeserializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosDeserializer.java @@ -7,7 +7,7 @@ import org.ethereum.beacon.ssz.type.SSZContainerType; import org.ethereum.beacon.ssz.type.SSZListType; import org.ethereum.beacon.ssz.type.SSZType; -import org.ethereum.beacon.ssz.visitor.SSZSimpleDeserializer.DecodeResult; +import org.ethereum.beacon.ssz.visitor.SSZSosDeserializer.DecodeResult; import org.javatuples.Pair; import java.nio.ByteOrder; @@ -15,7 +15,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; -public class SSZSimpleDeserializer implements SSZVisitor> { +public class SSZSosDeserializer implements SSZVisitor> { static final int BYTES_PER_LENGTH_OFFSET = 4; /** diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosSerializer.java new file mode 100644 index 000000000..c9c607074 --- /dev/null +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosSerializer.java @@ -0,0 +1,132 @@ +package org.ethereum.beacon.ssz.visitor; + +import org.ethereum.beacon.ssz.SSZSerializeException; +import org.ethereum.beacon.ssz.type.SSZBasicType; +import org.ethereum.beacon.ssz.type.SSZCompositeType; +import org.ethereum.beacon.ssz.type.SSZListType; +import tech.pegasys.artemis.util.bytes.BytesValue; +import tech.pegasys.artemis.util.bytes.BytesValues; + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.List; + +import static org.ethereum.beacon.ssz.visitor.SSZSosDeserializer.BYTES_PER_LENGTH_OFFSET; + +/** + * SSZ serializer with offset-based encoding of variable sized elements + */ +public class SSZSosSerializer + implements SSZVisitor { + private static BytesValue serializeLength(long len) { + return BytesValues.ofUnsignedIntLittleEndian(len); + } + + @Override + public SSZSerializerResult visitBasicValue(SSZBasicType type, Object value) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + type.getAccessor().encode(value, type.getTypeDescriptor(), baos); + return new SSZSerializerResult( + BytesValue.wrap(baos.toByteArray()), baos.size(), type.isFixedSize()); + } + + @Override + public SSZSerializerResult visitList( + SSZListType type, Object param, ChildVisitor childVisitor) { + + if (type.isVector()) { + if (type.getChildrenCount(param) != type.getVectorLength()) { + throw new SSZSerializeException( + "Vector type length doesn't match actual list length: " + + type.getVectorLength() + + " != " + + type.getChildrenCount(param) + + " for " + + type.toStringHelper()); + } + } + return visitComposite(type, param, childVisitor); + } + + @Override + public SSZSerializerResult visitSubList( + SSZListType type, + Object param, + int startIdx, + int len, + ChildVisitor childVisitor) { + return visitComposite(type, param, childVisitor, startIdx, len); + } + + @Override + public SSZSerializerResult visitComposite( + SSZCompositeType type, + Object rawValue, + ChildVisitor childVisitor) { + return visitComposite(type, rawValue, childVisitor, 0, type.getChildrenCount(rawValue)); + } + + private SSZSerializerResult visitComposite( + SSZCompositeType type, + Object rawValue, + ChildVisitor childVisitor, + int startIdx, + int len) { + + List childSerializations = new ArrayList<>(); + boolean fixedSize = type.isFixedSize(); + for (int i = startIdx; i < startIdx + len; i++) { + SSZSerializerResult res = childVisitor.apply(i, type.getChild(rawValue, i)); + childSerializations.add(res); + } + + int currentOffset = + childSerializations.stream() + .mapToInt(r -> r.isFixedSize() ? r.serializedBody.size() : BYTES_PER_LENGTH_OFFSET) + .sum(); + BytesValue composite = BytesValue.EMPTY; + + // Fixed part + for (SSZSerializerResult s : childSerializations) { + composite = + composite.concat(s.isFixedSize() ? s.serializedBody : serializeLength(currentOffset)); + if (!s.isFixedSize()) { + currentOffset = currentOffset + s.serializedLength; + } + } + + // Variable part + for (SSZSerializerResult s : childSerializations) { + if (s.isFixedSize()) { + continue; + } + composite = composite.concat(s.serializedBody); + } + + return new SSZSerializerResult(composite, currentOffset, fixedSize); + } + + public static class SSZSerializerResult { + final BytesValue serializedBody; + final int serializedLength; + final boolean fixedSize; + + public SSZSerializerResult(BytesValue serializedBody, int serializedLength, boolean fixedSize) { + this.serializedBody = serializedBody; + this.serializedLength = serializedLength; + this.fixedSize = fixedSize; + } + + public BytesValue getSerializedBody() { + return serializedBody; + } + + public int getSerializedLength() { + return serializedLength; + } + + public boolean isFixedSize() { + return fixedSize; + } + } +} From f8e081657878db903e0a5e9dd491a04a8e268f14 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 15 May 2019 11:08:37 +0300 Subject: [PATCH 10/15] ssz: add javadoc to SSZSosDeserializer --- .../org/ethereum/beacon/ssz/visitor/SSZSosDeserializer.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosDeserializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosDeserializer.java index 34c094701..684e610a3 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosDeserializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosDeserializer.java @@ -15,6 +15,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; +/** + * SSZ deserializer with offset-based decoding of variable sized elements + */ public class SSZSosDeserializer implements SSZVisitor> { static final int BYTES_PER_LENGTH_OFFSET = 4; From 0d8db5cf4094e280f0fd76475bfc399be2b420b9 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 15 May 2019 11:12:36 +0300 Subject: [PATCH 11/15] ssz: serializer results renamed to match pre-sos names --- .../ethereum/beacon/ssz/HasherSerializer.java | 10 +++--- .../ethereum/beacon/ssz/SSZSerializer.java | 10 +++--- .../ssz/visitor/SSZIncrementalHasher.java | 4 +-- .../beacon/ssz/visitor/SSZSimpleHasher.java | 10 +++--- .../ssz/visitor/SSZSimpleSerializer.java | 32 ++++++++--------- .../beacon/ssz/visitor/SSZSosSerializer.java | 36 +++++++++---------- 6 files changed, 51 insertions(+), 51 deletions(-) diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java index 59fb5d049..b66443319 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java @@ -6,7 +6,7 @@ import org.ethereum.beacon.ssz.type.SSZType; import org.ethereum.beacon.ssz.type.TypeResolver; import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZHasherSerializerResult; +import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; import org.ethereum.beacon.ssz.visitor.SSZVisitorHandler; import org.ethereum.beacon.ssz.visitor.SSZVisitorHost; @@ -14,7 +14,7 @@ /** SSZ hasher serializer */ public class HasherSerializer - implements BytesSerializer, SSZVisitorHandler { + implements BytesSerializer, SSZVisitorHandler { private final SSZVisitorHost sszVisitorHost; private final TypeResolver typeResolver; @@ -36,17 +36,17 @@ public byte[] encode(@Nullable C inputObject, Class inputClazz) return visit(inputObject, inputClazz).getSerializedBody().getArrayUnsafe(); } - private SSZHasherSerializerResult visit(C input, Class clazz) { + private SSZSerializerResult visit(C input, Class clazz) { return visitAny(typeResolver.resolveSSZType(new SSZField(clazz)), input); } @Override - public SSZHasherSerializerResult visitAny(SSZType sszType, Object value) { + public SSZSerializerResult visitAny(SSZType sszType, Object value) { return sszVisitorHost.handleAny(sszType, value, new SSZSimpleSerializer()); } @Override - public SSZHasherSerializerResult visitList( + public SSZSerializerResult visitList( SSZListType descriptor, Object listValue, int startIdx, int len) { return sszVisitorHost.handleSubList( descriptor, listValue, startIdx, len, new SSZSimpleSerializer()); diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java index c6b5fe965..b2cfc4a2a 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java @@ -10,13 +10,13 @@ import org.ethereum.beacon.ssz.visitor.SSZSosDeserializer; import org.ethereum.beacon.ssz.visitor.SSZSosDeserializer.DecodeResult; import org.ethereum.beacon.ssz.visitor.SSZSosSerializer; -import org.ethereum.beacon.ssz.visitor.SSZSosSerializer.SSZSerializerResult; +import org.ethereum.beacon.ssz.visitor.SSZSosSerializer.SerializerResult; import org.ethereum.beacon.ssz.visitor.SSZVisitorHandler; import org.ethereum.beacon.ssz.visitor.SSZVisitorHost; import org.javatuples.Pair; /** SSZ serializer/deserializer */ -public class SSZSerializer implements BytesSerializer, SSZVisitorHandler { +public class SSZSerializer implements BytesSerializer, SSZVisitorHandler { private final SSZVisitorHost sszVisitorHost; private final TypeResolver typeResolver; @@ -39,17 +39,17 @@ public byte[] encode(@Nullable C inputObject, Class inputClazz) return visit(inputObject, inputClazz).getSerializedBody().getArrayUnsafe(); } - private SSZSerializerResult visit(C input, Class clazz) { + private SerializerResult visit(C input, Class clazz) { return visitAny(typeResolver.resolveSSZType(new SSZField(clazz)), input); } @Override - public SSZSerializerResult visitAny(SSZType sszType, Object value) { + public SerializerResult visitAny(SSZType sszType, Object value) { return sszVisitorHost.handleAny(sszType, value, new SSZSosSerializer()); } @Override - public SSZSerializerResult visitList(SSZListType descriptor, Object listValue, int startIdx, int len) { + public SerializerResult visitList(SSZListType descriptor, Object listValue, int startIdx, int len) { return sszVisitorHost.handleSubList(descriptor, listValue, startIdx, len, new SSZSosSerializer()); } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java index d0cb25129..332507691 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java @@ -15,7 +15,7 @@ import org.ethereum.beacon.ssz.incremental.UpdateListener; import org.ethereum.beacon.ssz.type.SSZCompositeType; import org.ethereum.beacon.ssz.type.SSZListType; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZHasherSerializerResult; +import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -49,7 +49,7 @@ public UpdateListener fork() { } public SSZIncrementalHasher( - SSZVisitorHandler serializer, + SSZVisitorHandler serializer, Function hashFunction, int bytesPerChunk) { super(serializer, hashFunction, bytesPerChunk); } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java index a2d07a6fd..b3699acbc 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java @@ -8,7 +8,7 @@ import org.ethereum.beacon.ssz.type.SSZBasicType; import org.ethereum.beacon.ssz.type.SSZCompositeType; import org.ethereum.beacon.ssz.type.SSZListType; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZHasherSerializerResult; +import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -17,12 +17,12 @@ public class SSZSimpleHasher implements SSZVisitor { private final Hash32[] zeroHashes = new Hash32[32]; - final SSZVisitorHandler serializer; + final SSZVisitorHandler serializer; final Function hashFunction; final int bytesPerChunk; public SSZSimpleHasher( - SSZVisitorHandler serializer, + SSZVisitorHandler serializer, Function hashFunction, int bytesPerChunk) { this.serializer = serializer; this.hashFunction = hashFunction; @@ -31,7 +31,7 @@ public SSZSimpleHasher( @Override public MerkleTrie visitBasicValue(SSZBasicType descriptor, Object value) { - SSZHasherSerializerResult sszSerializerResult = serializer.visitAny(descriptor, value); + SSZSerializerResult sszSerializerResult = serializer.visitAny(descriptor, value); return merkleize(pack(sszSerializerResult.serializedBody)); } @@ -43,7 +43,7 @@ public MerkleTrie visitComposite(SSZCompositeType type, Object rawValue, if (type.getChildrenCount(rawValue) == 0) { // empty chunk list } else if (type.isList() && ((SSZListType) type).getElementType().isBasicType()) { - SSZHasherSerializerResult sszSerializerResult = serializer.visitAny(type, rawValue); + SSZSerializerResult sszSerializerResult = serializer.visitAny(type, rawValue); chunks = pack(sszSerializerResult.serializedBody); } else { diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java index ed15781da..58a37ff00 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java @@ -11,18 +11,18 @@ import java.util.ArrayList; import java.util.List; -public class SSZSimpleSerializer implements SSZVisitor { +public class SSZSimpleSerializer implements SSZVisitor { - public static class SSZHasherSerializerResult { + public static class SSZSerializerResult { BytesValue serializedBody; BytesValue serializedLength; - public SSZHasherSerializerResult(BytesValue serializedBody) { + public SSZSerializerResult(BytesValue serializedBody) { this.serializedBody = serializedBody; this.serializedLength = BytesValue.EMPTY; } - public SSZHasherSerializerResult(BytesValue serializedBody, + public SSZSerializerResult(BytesValue serializedBody, BytesValue serializedLength) { this.serializedBody = serializedBody; this.serializedLength = serializedLength; @@ -46,19 +46,19 @@ public boolean isFixedSize() { } @Override - public SSZHasherSerializerResult visitBasicValue(SSZBasicType type, Object value) { + public SSZSerializerResult visitBasicValue(SSZBasicType type, Object value) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); type.getAccessor().encode(value, type.getTypeDescriptor(), baos); BytesValue res = BytesValue.wrap(baos.toByteArray()); if (!type.isFixedSize()) { res = serializeLength(res.size()).concat(res); } - return new SSZHasherSerializerResult(res); + return new SSZSerializerResult(res); } @Override - public SSZHasherSerializerResult visitList(SSZListType type, Object param, - ChildVisitor childVisitor) { + public SSZSerializerResult visitList(SSZListType type, Object param, + ChildVisitor childVisitor) { if (type.isVector()) { if (type.getChildrenCount(param) != type.getVectorLength()) { @@ -70,31 +70,31 @@ public SSZHasherSerializerResult visitList(SSZListType type, Object param, } @Override - public SSZHasherSerializerResult visitSubList(SSZListType type, Object param, - int startIdx, int len, ChildVisitor childVisitor) { + public SSZSerializerResult visitSubList(SSZListType type, Object param, + int startIdx, int len, ChildVisitor childVisitor) { return visitComposite(type, param, childVisitor, startIdx, len); } @Override - public SSZHasherSerializerResult visitComposite(SSZCompositeType type, Object rawValue, - ChildVisitor childVisitor) { + public SSZSerializerResult visitComposite(SSZCompositeType type, Object rawValue, + ChildVisitor childVisitor) { return visitComposite(type, rawValue, childVisitor, 0, type.getChildrenCount(rawValue)); } - private SSZHasherSerializerResult visitComposite(SSZCompositeType type, Object rawValue, - ChildVisitor childVisitor, int startIdx, int len) { + private SSZSerializerResult visitComposite(SSZCompositeType type, Object rawValue, + ChildVisitor childVisitor, int startIdx, int len) { List childSerializations = new ArrayList<>(); boolean fixedSize = type.isFixedSize(); int length = 0; for (int i = startIdx; i < startIdx + len; i++) { - SSZHasherSerializerResult res = childVisitor.apply(i, type.getChild(rawValue, i)); + SSZSerializerResult res = childVisitor.apply(i, type.getChild(rawValue, i)); childSerializations.add(res.serializedLength); childSerializations.add(res.serializedBody); fixedSize &= res.isFixedSize(); length += res.serializedBody.size() + res.serializedLength.size(); } - return new SSZHasherSerializerResult(BytesValue.concat(childSerializations), + return new SSZSerializerResult(BytesValue.concat(childSerializations), fixedSize ? BytesValue.EMPTY : serializeLength(length)); } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosSerializer.java index c9c607074..22dcf5f78 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosSerializer.java @@ -17,22 +17,22 @@ * SSZ serializer with offset-based encoding of variable sized elements */ public class SSZSosSerializer - implements SSZVisitor { + implements SSZVisitor { private static BytesValue serializeLength(long len) { return BytesValues.ofUnsignedIntLittleEndian(len); } @Override - public SSZSerializerResult visitBasicValue(SSZBasicType type, Object value) { + public SerializerResult visitBasicValue(SSZBasicType type, Object value) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); type.getAccessor().encode(value, type.getTypeDescriptor(), baos); - return new SSZSerializerResult( + return new SerializerResult( BytesValue.wrap(baos.toByteArray()), baos.size(), type.isFixedSize()); } @Override - public SSZSerializerResult visitList( - SSZListType type, Object param, ChildVisitor childVisitor) { + public SerializerResult visitList( + SSZListType type, Object param, ChildVisitor childVisitor) { if (type.isVector()) { if (type.getChildrenCount(param) != type.getVectorLength()) { @@ -49,34 +49,34 @@ public SSZSerializerResult visitList( } @Override - public SSZSerializerResult visitSubList( + public SerializerResult visitSubList( SSZListType type, Object param, int startIdx, int len, - ChildVisitor childVisitor) { + ChildVisitor childVisitor) { return visitComposite(type, param, childVisitor, startIdx, len); } @Override - public SSZSerializerResult visitComposite( + public SerializerResult visitComposite( SSZCompositeType type, Object rawValue, - ChildVisitor childVisitor) { + ChildVisitor childVisitor) { return visitComposite(type, rawValue, childVisitor, 0, type.getChildrenCount(rawValue)); } - private SSZSerializerResult visitComposite( + private SerializerResult visitComposite( SSZCompositeType type, Object rawValue, - ChildVisitor childVisitor, + ChildVisitor childVisitor, int startIdx, int len) { - List childSerializations = new ArrayList<>(); + List childSerializations = new ArrayList<>(); boolean fixedSize = type.isFixedSize(); for (int i = startIdx; i < startIdx + len; i++) { - SSZSerializerResult res = childVisitor.apply(i, type.getChild(rawValue, i)); + SerializerResult res = childVisitor.apply(i, type.getChild(rawValue, i)); childSerializations.add(res); } @@ -87,7 +87,7 @@ private SSZSerializerResult visitComposite( BytesValue composite = BytesValue.EMPTY; // Fixed part - for (SSZSerializerResult s : childSerializations) { + for (SerializerResult s : childSerializations) { composite = composite.concat(s.isFixedSize() ? s.serializedBody : serializeLength(currentOffset)); if (!s.isFixedSize()) { @@ -96,22 +96,22 @@ private SSZSerializerResult visitComposite( } // Variable part - for (SSZSerializerResult s : childSerializations) { + for (SerializerResult s : childSerializations) { if (s.isFixedSize()) { continue; } composite = composite.concat(s.serializedBody); } - return new SSZSerializerResult(composite, currentOffset, fixedSize); + return new SerializerResult(composite, currentOffset, fixedSize); } - public static class SSZSerializerResult { + public static class SerializerResult { final BytesValue serializedBody; final int serializedLength; final boolean fixedSize; - public SSZSerializerResult(BytesValue serializedBody, int serializedLength, boolean fixedSize) { + public SerializerResult(BytesValue serializedBody, int serializedLength, boolean fixedSize) { this.serializedBody = serializedBody; this.serializedLength = serializedLength; this.fixedSize = fixedSize; From 2a65af0e025b3722e5bc346182022e1cf2738d06 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 15 May 2019 14:06:31 +0300 Subject: [PATCH 12/15] ssz: rename ssz sos serializers to shorter names --- .../org/ethereum/beacon/ssz/SSZSerializer.java | 14 +++++++------- ...SZSosDeserializer.java => SosDeserializer.java} | 4 ++-- .../{SSZSosSerializer.java => SosSerializer.java} | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) rename ssz/src/main/java/org/ethereum/beacon/ssz/visitor/{SSZSosDeserializer.java => SosDeserializer.java} (97%) rename ssz/src/main/java/org/ethereum/beacon/ssz/visitor/{SSZSosSerializer.java => SosSerializer.java} (95%) diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java index b2cfc4a2a..3549ec8e5 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java @@ -7,10 +7,10 @@ import org.ethereum.beacon.ssz.type.SSZListType; import org.ethereum.beacon.ssz.type.SSZType; import org.ethereum.beacon.ssz.type.TypeResolver; -import org.ethereum.beacon.ssz.visitor.SSZSosDeserializer; -import org.ethereum.beacon.ssz.visitor.SSZSosDeserializer.DecodeResult; -import org.ethereum.beacon.ssz.visitor.SSZSosSerializer; -import org.ethereum.beacon.ssz.visitor.SSZSosSerializer.SerializerResult; +import org.ethereum.beacon.ssz.visitor.SosDeserializer; +import org.ethereum.beacon.ssz.visitor.SosDeserializer.DecodeResult; +import org.ethereum.beacon.ssz.visitor.SosSerializer; +import org.ethereum.beacon.ssz.visitor.SosSerializer.SerializerResult; import org.ethereum.beacon.ssz.visitor.SSZVisitorHandler; import org.ethereum.beacon.ssz.visitor.SSZVisitorHost; import org.javatuples.Pair; @@ -45,12 +45,12 @@ private SerializerResult visit(C input, Class clazz) { @Override public SerializerResult visitAny(SSZType sszType, Object value) { - return sszVisitorHost.handleAny(sszType, value, new SSZSosSerializer()); + return sszVisitorHost.handleAny(sszType, value, new SosSerializer()); } @Override public SerializerResult visitList(SSZListType descriptor, Object listValue, int startIdx, int len) { - return sszVisitorHost.handleSubList(descriptor, listValue, startIdx, len, new SSZSosSerializer()); + return sszVisitorHost.handleSubList(descriptor, listValue, startIdx, len, new SosSerializer()); } /** @@ -64,7 +64,7 @@ public C decode(byte[] data, Class clazz) { DecodeResult decodeResult = sszVisitorHost.handleAny( typeResolver.resolveSSZType(new SSZField(clazz)), Pair.with(Bytes.wrap(data), null), - new SSZSosDeserializer()); + new SosDeserializer()); if (data.length != decodeResult.readBytes) { throw new SSZSerializeException(String.format("Invalid SSZ encoding, calculated data size %s bytes, while provided %s bytes", decodeResult.readBytes, data.length)); } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosDeserializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SosDeserializer.java similarity index 97% rename from ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosDeserializer.java rename to ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SosDeserializer.java index 684e610a3..269e238bf 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosDeserializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SosDeserializer.java @@ -7,7 +7,7 @@ import org.ethereum.beacon.ssz.type.SSZContainerType; import org.ethereum.beacon.ssz.type.SSZListType; import org.ethereum.beacon.ssz.type.SSZType; -import org.ethereum.beacon.ssz.visitor.SSZSosDeserializer.DecodeResult; +import org.ethereum.beacon.ssz.visitor.SosDeserializer.DecodeResult; import org.javatuples.Pair; import java.nio.ByteOrder; @@ -18,7 +18,7 @@ /** * SSZ deserializer with offset-based decoding of variable sized elements */ -public class SSZSosDeserializer implements SSZVisitor> { +public class SosDeserializer implements SSZVisitor> { static final int BYTES_PER_LENGTH_OFFSET = 4; /** diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SosSerializer.java similarity index 95% rename from ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosSerializer.java rename to ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SosSerializer.java index 22dcf5f78..dfe33992d 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSosSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SosSerializer.java @@ -11,13 +11,13 @@ import java.util.ArrayList; import java.util.List; -import static org.ethereum.beacon.ssz.visitor.SSZSosDeserializer.BYTES_PER_LENGTH_OFFSET; +import static org.ethereum.beacon.ssz.visitor.SosDeserializer.BYTES_PER_LENGTH_OFFSET; /** * SSZ serializer with offset-based encoding of variable sized elements */ -public class SSZSosSerializer - implements SSZVisitor { +public class SosSerializer + implements SSZVisitor { private static BytesValue serializeLength(long len) { return BytesValues.ofUnsignedIntLittleEndian(len); } From 713b1d5808d371c3852a1854a8ba76ce3594d428 Mon Sep 17 00:00:00 2001 From: Dmitrii Shmatko Date: Wed, 15 May 2019 16:30:15 +0300 Subject: [PATCH 13/15] ssz: remove old serialization, not used --- .../ethereum/beacon/ssz/HasherSerializer.java | 16 +-- .../ssz/visitor/SSZIncrementalHasher.java | 4 +- .../beacon/ssz/visitor/SSZSimpleHasher.java | 10 +- .../ssz/visitor/SSZSimpleSerializer.java | 104 ------------------ 4 files changed, 15 insertions(+), 119 deletions(-) delete mode 100644 ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java index b66443319..ce9a9972e 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java @@ -5,16 +5,16 @@ import org.ethereum.beacon.ssz.type.SSZListType; import org.ethereum.beacon.ssz.type.SSZType; import org.ethereum.beacon.ssz.type.TypeResolver; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; import org.ethereum.beacon.ssz.visitor.SSZVisitorHandler; import org.ethereum.beacon.ssz.visitor.SSZVisitorHost; +import org.ethereum.beacon.ssz.visitor.SosSerializer; +import org.ethereum.beacon.ssz.visitor.SosSerializer.SerializerResult; import javax.annotation.Nullable; /** SSZ hasher serializer */ public class HasherSerializer - implements BytesSerializer, SSZVisitorHandler { + implements BytesSerializer, SSZVisitorHandler { private final SSZVisitorHost sszVisitorHost; private final TypeResolver typeResolver; @@ -36,20 +36,20 @@ public byte[] encode(@Nullable C inputObject, Class inputClazz) return visit(inputObject, inputClazz).getSerializedBody().getArrayUnsafe(); } - private SSZSerializerResult visit(C input, Class clazz) { + private SerializerResult visit(C input, Class clazz) { return visitAny(typeResolver.resolveSSZType(new SSZField(clazz)), input); } @Override - public SSZSerializerResult visitAny(SSZType sszType, Object value) { - return sszVisitorHost.handleAny(sszType, value, new SSZSimpleSerializer()); + public SerializerResult visitAny(SSZType sszType, Object value) { + return sszVisitorHost.handleAny(sszType, value, new SosSerializer()); } @Override - public SSZSerializerResult visitList( + public SerializerResult visitList( SSZListType descriptor, Object listValue, int startIdx, int len) { return sszVisitorHost.handleSubList( - descriptor, listValue, startIdx, len, new SSZSimpleSerializer()); + descriptor, listValue, startIdx, len, new SosSerializer()); } public C decode(byte[] data, Class clazz) { diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java index 332507691..a4469f045 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java @@ -15,7 +15,7 @@ import org.ethereum.beacon.ssz.incremental.UpdateListener; import org.ethereum.beacon.ssz.type.SSZCompositeType; import org.ethereum.beacon.ssz.type.SSZListType; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; +import org.ethereum.beacon.ssz.visitor.SosSerializer.SerializerResult; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -49,7 +49,7 @@ public UpdateListener fork() { } public SSZIncrementalHasher( - SSZVisitorHandler serializer, + SSZVisitorHandler serializer, Function hashFunction, int bytesPerChunk) { super(serializer, hashFunction, bytesPerChunk); } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java index b3699acbc..43c27bb8a 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java @@ -8,7 +8,7 @@ import org.ethereum.beacon.ssz.type.SSZBasicType; import org.ethereum.beacon.ssz.type.SSZCompositeType; import org.ethereum.beacon.ssz.type.SSZListType; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; +import org.ethereum.beacon.ssz.visitor.SosSerializer.SerializerResult; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -17,12 +17,12 @@ public class SSZSimpleHasher implements SSZVisitor { private final Hash32[] zeroHashes = new Hash32[32]; - final SSZVisitorHandler serializer; + final SSZVisitorHandler serializer; final Function hashFunction; final int bytesPerChunk; public SSZSimpleHasher( - SSZVisitorHandler serializer, + SSZVisitorHandler serializer, Function hashFunction, int bytesPerChunk) { this.serializer = serializer; this.hashFunction = hashFunction; @@ -31,7 +31,7 @@ public SSZSimpleHasher( @Override public MerkleTrie visitBasicValue(SSZBasicType descriptor, Object value) { - SSZSerializerResult sszSerializerResult = serializer.visitAny(descriptor, value); + SerializerResult sszSerializerResult = serializer.visitAny(descriptor, value); return merkleize(pack(sszSerializerResult.serializedBody)); } @@ -43,7 +43,7 @@ public MerkleTrie visitComposite(SSZCompositeType type, Object rawValue, if (type.getChildrenCount(rawValue) == 0) { // empty chunk list } else if (type.isList() && ((SSZListType) type).getElementType().isBasicType()) { - SSZSerializerResult sszSerializerResult = serializer.visitAny(type, rawValue); + SerializerResult sszSerializerResult = serializer.visitAny(type, rawValue); chunks = pack(sszSerializerResult.serializedBody); } else { diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java deleted file mode 100644 index 58a37ff00..000000000 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.ethereum.beacon.ssz.visitor; - -import org.ethereum.beacon.ssz.SSZSerializeException; -import org.ethereum.beacon.ssz.type.SSZBasicType; -import org.ethereum.beacon.ssz.type.SSZCompositeType; -import org.ethereum.beacon.ssz.type.SSZListType; -import tech.pegasys.artemis.util.bytes.BytesValue; -import tech.pegasys.artemis.util.bytes.BytesValues; - -import java.io.ByteArrayOutputStream; -import java.util.ArrayList; -import java.util.List; - -public class SSZSimpleSerializer implements SSZVisitor { - - public static class SSZSerializerResult { - BytesValue serializedBody; - BytesValue serializedLength; - - public SSZSerializerResult(BytesValue serializedBody) { - this.serializedBody = serializedBody; - this.serializedLength = BytesValue.EMPTY; - } - - public SSZSerializerResult(BytesValue serializedBody, - BytesValue serializedLength) { - this.serializedBody = serializedBody; - this.serializedLength = serializedLength; - } - - public BytesValue getSerializedBody() { - return serializedBody; - } - - public BytesValue getSerializedLength() { - return serializedLength; - } - - public BytesValue getSerialized() { - return BytesValue.concat(getSerializedLength(), getSerializedBody()); - } - - public boolean isFixedSize() { - return serializedLength.isEmpty(); - } - } - - @Override - public SSZSerializerResult visitBasicValue(SSZBasicType type, Object value) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - type.getAccessor().encode(value, type.getTypeDescriptor(), baos); - BytesValue res = BytesValue.wrap(baos.toByteArray()); - if (!type.isFixedSize()) { - res = serializeLength(res.size()).concat(res); - } - return new SSZSerializerResult(res); - } - - @Override - public SSZSerializerResult visitList(SSZListType type, Object param, - ChildVisitor childVisitor) { - - if (type.isVector()) { - if (type.getChildrenCount(param) != type.getVectorLength()) { - throw new SSZSerializeException("Vector type length doesn't match actual list length: " - + type.getVectorLength() + " != " + type.getChildrenCount(param) + " for " + type.toStringHelper()); - } - } - return visitComposite(type, param, childVisitor); - } - - @Override - public SSZSerializerResult visitSubList(SSZListType type, Object param, - int startIdx, int len, ChildVisitor childVisitor) { - return visitComposite(type, param, childVisitor, startIdx, len); - } - - @Override - public SSZSerializerResult visitComposite(SSZCompositeType type, Object rawValue, - ChildVisitor childVisitor) { - return visitComposite(type, rawValue, childVisitor, 0, type.getChildrenCount(rawValue)); - } - - private SSZSerializerResult visitComposite(SSZCompositeType type, Object rawValue, - ChildVisitor childVisitor, int startIdx, int len) { - List childSerializations = new ArrayList<>(); - boolean fixedSize = type.isFixedSize(); - int length = 0; - for (int i = startIdx; i < startIdx + len; i++) { - SSZSerializerResult res = childVisitor.apply(i, type.getChild(rawValue, i)); - childSerializations.add(res.serializedLength); - childSerializations.add(res.serializedBody); - fixedSize &= res.isFixedSize(); - length += res.serializedBody.size() + res.serializedLength.size(); - } - - return new SSZSerializerResult(BytesValue.concat(childSerializations), - fixedSize ? BytesValue.EMPTY : serializeLength(length)); - } - - private BytesValue serializeLength(int len) { - return BytesValues.ofUnsignedIntLittleEndian(len); - } -} From 4ecf96e4ae263a5a85167916826a9aa927b19d4b Mon Sep 17 00:00:00 2001 From: Anton Nashatyrev Date: Wed, 15 May 2019 18:11:07 +0300 Subject: [PATCH 14/15] Use SosSerializer for Hasher --- .../ethereum/beacon/ssz/HasherSerializer.java | 58 ------------------- .../org/ethereum/beacon/ssz/SSZBuilder.java | 14 +---- .../ssz/visitor/SSZIncrementalHasher.java | 4 +- .../beacon/ssz/visitor/SSZSimpleHasher.java | 9 +-- .../beacon/ssz/SSZIncrementalTest.java | 14 ++--- 5 files changed, 16 insertions(+), 83 deletions(-) delete mode 100644 ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java deleted file mode 100644 index b66443319..000000000 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/HasherSerializer.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.ethereum.beacon.ssz; - -import net.consensys.cava.ssz.SSZException; -import org.ethereum.beacon.ssz.access.SSZField; -import org.ethereum.beacon.ssz.type.SSZListType; -import org.ethereum.beacon.ssz.type.SSZType; -import org.ethereum.beacon.ssz.type.TypeResolver; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; -import org.ethereum.beacon.ssz.visitor.SSZVisitorHandler; -import org.ethereum.beacon.ssz.visitor.SSZVisitorHost; - -import javax.annotation.Nullable; - -/** SSZ hasher serializer */ -public class HasherSerializer - implements BytesSerializer, SSZVisitorHandler { - - private final SSZVisitorHost sszVisitorHost; - private final TypeResolver typeResolver; - - public HasherSerializer(SSZVisitorHost sszVisitorHost, TypeResolver typeResolver) { - this.sszVisitorHost = sszVisitorHost; - this.typeResolver = typeResolver; - } - - /** - * Serializes input to byte[] data - * - * @param inputObject input value - * @param inputClazz Class of value - * @return SSZ serialization - */ - @Override - public byte[] encode(@Nullable C inputObject, Class inputClazz) { - return visit(inputObject, inputClazz).getSerializedBody().getArrayUnsafe(); - } - - private SSZSerializerResult visit(C input, Class clazz) { - return visitAny(typeResolver.resolveSSZType(new SSZField(clazz)), input); - } - - @Override - public SSZSerializerResult visitAny(SSZType sszType, Object value) { - return sszVisitorHost.handleAny(sszType, value, new SSZSimpleSerializer()); - } - - @Override - public SSZSerializerResult visitList( - SSZListType descriptor, Object listValue, int startIdx, int len) { - return sszVisitorHost.handleSubList( - descriptor, listValue, startIdx, len, new SSZSimpleSerializer()); - } - - public C decode(byte[] data, Class clazz) { - throw new SSZException("Hasher doesn't support decoding!"); - } -} diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZBuilder.java b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZBuilder.java index e0a8a1d3f..f3968a0ad 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZBuilder.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZBuilder.java @@ -264,23 +264,13 @@ public SSZSerializer buildSerializer() { return new SSZSerializer(visitorHost, typeResolver); } - /** - * Finalizes build of {@link HasherSerializer} with builder - * - * @return {@link HasherSerializer} - */ - public HasherSerializer buildHasherSerializer() { - buildCommon(); - return new HasherSerializer(visitorHost, typeResolver); - } - public SSZHasher buildHasher(Function hashFunction) { buildCommon(); SSZVisitor hasherVisitor; if (incrementalHasher) { - hasherVisitor = new SSZIncrementalHasher(buildHasherSerializer(), hashFunction, sszHashBytesPerChunk); + hasherVisitor = new SSZIncrementalHasher(buildSerializer(), hashFunction, sszHashBytesPerChunk); } else { - hasherVisitor = new SSZSimpleHasher(buildHasherSerializer(), hashFunction, sszHashBytesPerChunk); + hasherVisitor = new SSZSimpleHasher(buildSerializer(), hashFunction, sszHashBytesPerChunk); } return new SSZHasher(typeResolver, visitorHost, hasherVisitor); } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java index 332507691..a4469f045 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java @@ -15,7 +15,7 @@ import org.ethereum.beacon.ssz.incremental.UpdateListener; import org.ethereum.beacon.ssz.type.SSZCompositeType; import org.ethereum.beacon.ssz.type.SSZListType; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; +import org.ethereum.beacon.ssz.visitor.SosSerializer.SerializerResult; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -49,7 +49,7 @@ public UpdateListener fork() { } public SSZIncrementalHasher( - SSZVisitorHandler serializer, + SSZVisitorHandler serializer, Function hashFunction, int bytesPerChunk) { super(serializer, hashFunction, bytesPerChunk); } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java index b3699acbc..e263102f6 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java @@ -9,6 +9,7 @@ import org.ethereum.beacon.ssz.type.SSZCompositeType; import org.ethereum.beacon.ssz.type.SSZListType; import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; +import org.ethereum.beacon.ssz.visitor.SosSerializer.SerializerResult; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32; import tech.pegasys.artemis.util.bytes.BytesValue; @@ -17,12 +18,12 @@ public class SSZSimpleHasher implements SSZVisitor { private final Hash32[] zeroHashes = new Hash32[32]; - final SSZVisitorHandler serializer; + final SSZVisitorHandler serializer; final Function hashFunction; final int bytesPerChunk; public SSZSimpleHasher( - SSZVisitorHandler serializer, + SSZVisitorHandler serializer, Function hashFunction, int bytesPerChunk) { this.serializer = serializer; this.hashFunction = hashFunction; @@ -31,7 +32,7 @@ public SSZSimpleHasher( @Override public MerkleTrie visitBasicValue(SSZBasicType descriptor, Object value) { - SSZSerializerResult sszSerializerResult = serializer.visitAny(descriptor, value); + SerializerResult sszSerializerResult = serializer.visitAny(descriptor, value); return merkleize(pack(sszSerializerResult.serializedBody)); } @@ -43,7 +44,7 @@ public MerkleTrie visitComposite(SSZCompositeType type, Object rawValue, if (type.getChildrenCount(rawValue) == 0) { // empty chunk list } else if (type.isList() && ((SSZListType) type).getElementType().isBasicType()) { - SSZSerializerResult sszSerializerResult = serializer.visitAny(type, rawValue); + SerializerResult sszSerializerResult = serializer.visitAny(type, rawValue); chunks = pack(sszSerializerResult.serializedBody); } else { diff --git a/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java b/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java index 58bc5761d..6e9bcc0f6 100644 --- a/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java +++ b/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java @@ -103,7 +103,7 @@ public void testHashIncremental1() throws Exception { TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); + SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, 32); @@ -212,7 +212,7 @@ public void testReadList() { TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); + SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, 32); @@ -350,7 +350,7 @@ public void testPackedList1() { TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); + SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, @@ -441,7 +441,7 @@ private void listRandomTest(WriteList list, Supplier numSuppl TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); + SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, @@ -512,7 +512,7 @@ public void testPackedListRemove1() { TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); + SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, 32); @@ -587,7 +587,7 @@ public void testListRemove() { TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); + SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, 32); @@ -749,7 +749,7 @@ public void testComplexStruct() { TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); - HasherSerializer serializer = new HasherSerializer(visitorHost, typeResolver); + SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, 32); From 6ee88f240afbf123552ba322869448eb28c425cc Mon Sep 17 00:00:00 2001 From: Anton Nashatyrev Date: Wed, 15 May 2019 18:14:05 +0300 Subject: [PATCH 15/15] Resolve merge conflict --- .../java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java | 1 - 1 file changed, 1 deletion(-) diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java index e263102f6..43c27bb8a 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleHasher.java @@ -8,7 +8,6 @@ import org.ethereum.beacon.ssz.type.SSZBasicType; import org.ethereum.beacon.ssz.type.SSZCompositeType; import org.ethereum.beacon.ssz.type.SSZListType; -import org.ethereum.beacon.ssz.visitor.SSZSimpleSerializer.SSZSerializerResult; import org.ethereum.beacon.ssz.visitor.SosSerializer.SerializerResult; import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.Bytes32;