From 52705c850a1b8b97a6baf33a3a7f87ba14a5c3af Mon Sep 17 00:00:00 2001 From: Michal Foksa Date: Wed, 1 Jan 2025 22:50:55 +0100 Subject: [PATCH 01/11] `@Decimal` annotation created. Avro schema for a field annotated with `@Decimal` is created with logical type `decimal` and type either `bytes` (by default) or `fixed`. --- .../dataformat/avro/annotation/Decimal.java | 34 ++++++++ .../dataformat/avro/schema/RecordVisitor.java | 21 +++-- .../dataformat/avro/BigDecimalTest.java | 81 ++++++++++++++++++- 3 files changed, 127 insertions(+), 9 deletions(-) create mode 100644 avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/Decimal.java diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/Decimal.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/Decimal.java new file mode 100644 index 000000000..c3e23ffcc --- /dev/null +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/Decimal.java @@ -0,0 +1,34 @@ +package com.fasterxml.jackson.dataformat.avro.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Instructs the {@link com.fasterxml.jackson.dataformat.avro.schema.AvroSchemaGenerator AvroSchemaGenerator} + * to declare the annotated property's logical type as "decimal" ({@link org.apache.avro.LogicalTypes.Decimal}). + * By default, the Avro type is "bytes" ({@link org.apache.avro.Schema.Type#BYTES}), unless the field is also + * annotated with {@link com.fasterxml.jackson.dataformat.avro.AvroFixedSize}, in which case the Avro type + * will be "fixed" ({@link org.apache.avro.Schema.Type#FIXED}). + *

+ * This annotation is only used during Avro schema generation and does not affect data serialization + * or deserialization. + * + * @since 2.19 + */ +@Target({ElementType.ANNOTATION_TYPE, ElementType.FIELD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface Decimal { + + /** + * Maximum precision of decimals stored in this type. + */ + int precision(); + + /** + * Scale must be zero or a positive integer less than or equal to the precision. + */ + int scale() default 0; + +} diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/RecordVisitor.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/RecordVisitor.java index 8247eee97..8b1419492 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/RecordVisitor.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/RecordVisitor.java @@ -4,6 +4,8 @@ import java.util.List; import java.util.Map; +import com.fasterxml.jackson.dataformat.avro.annotation.Decimal; +import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.Schema.Type; import org.apache.avro.reflect.AvroMeta; @@ -141,17 +143,26 @@ public void optionalProperty(String name, JsonFormatVisitable handler, protected Schema.Field schemaFieldForWriter(BeanProperty prop, boolean optional) throws JsonMappingException { - Schema writerSchema; + Schema writerSchema = null; // Check if schema for property is overridden AvroSchema schemaOverride = prop.getAnnotation(AvroSchema.class); if (schemaOverride != null) { Schema.Parser parser = new Schema.Parser(); writerSchema = parser.parse(schemaOverride.value()); } else { - AvroFixedSize fixedSize = prop.getAnnotation(AvroFixedSize.class); - if (fixedSize != null) { + if (prop.getAnnotation(AvroFixedSize.class) != null) { + AvroFixedSize fixedSize = prop.getAnnotation(AvroFixedSize.class); writerSchema = Schema.createFixed(fixedSize.typeName(), null, fixedSize.typeNamespace(), fixedSize.size()); - } else { + } + if (_visitorWrapper.isLogicalTypesEnabled() && prop.getAnnotation(Decimal.class) != null) { + if (writerSchema == null) { + writerSchema = Schema.create(Type.BYTES); + } + Decimal decimal = prop.getAnnotation(Decimal.class); + writerSchema = LogicalTypes.decimal(decimal.precision(), decimal.scale()) + .addToSchema(writerSchema); + } + if (writerSchema == null) { JsonSerializer ser = null; // 23-Nov-2012, tatu: Ideally shouldn't need to do this but... @@ -204,7 +215,7 @@ protected Schema.Field schemaFieldForWriter(BeanProperty prop, boolean optional) /** * A union schema with a default value must always have the schema branch corresponding to the default value first, or Avro will print a - * warning complaining that the default value is not compatible. If {@code schema} is a {@link Schema.Type#UNION UNION} schema and + * warning complaining that the default value is not compatible. If {@code schema} is a {@link Type#UNION UNION} schema and * {@code defaultValue} is non-{@code null}, this finds the appropriate branch in the union and reorders the union so that it is first. * * @param schema diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java index f8d796aeb..a9553c16a 100644 --- a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java @@ -2,11 +2,85 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.dataformat.avro.annotation.Decimal; +import com.fasterxml.jackson.dataformat.avro.schema.AvroSchemaGenerator; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.junit.Test; import java.math.BigDecimal; +import static org.assertj.core.api.Assertions.assertThat; + public class BigDecimalTest extends AvroTestBase { + private static final AvroMapper MAPPER = new AvroMapper(); + + static class BigDecimalWithDecimalAnnotationToBytesWrapper { + @JsonProperty(required = true) // field is made required only to have simpler avro schema + @Decimal(precision = 10, scale = 2) + public BigDecimal bigDecimalValue; + + public BigDecimalWithDecimalAnnotationToBytesWrapper(BigDecimal bigDecimalValue) { + this.bigDecimalValue = bigDecimalValue; + } + } + + @Test + public void testSchemaCreationOnBigDecimalWithDecimalAnnotationToBytes() throws JsonMappingException { + // GIVEN + AvroSchemaGenerator gen = new AvroSchemaGenerator() + .enableLogicalTypes(); + + // WHEN + MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationToBytesWrapper.class, gen); + final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); + + System.out.println(BigDecimalWithDecimalAnnotationToBytesWrapper.class.getSimpleName() + " schema:\n" + actualSchema.toString(true)); + + // THEN + assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); + + Schema bigDecimalValue = actualSchema.getField("bigDecimalValue").schema(); + assertThat(bigDecimalValue.getType()).isEqualTo(Schema.Type.BYTES); + assertThat(bigDecimalValue.getLogicalType()).isEqualTo(LogicalTypes.decimal(10, 2)); + assertThat(bigDecimalValue.getProp("java-class")).isNull(); + } + + static class BigDecimalWithDecimalAnnotationToFixedWrapper { + @JsonProperty(required = true) // field is made required only to have simpler avro schema + @AvroFixedSize(typeName = "BigDecimalWithDecimalAnnotationToFixedWrapper", size = 10) + @Decimal(precision = 6, scale = 3) + public BigDecimal bigDecimalValue; + + public BigDecimalWithDecimalAnnotationToFixedWrapper(BigDecimal bigDecimalValue) { + this.bigDecimalValue = bigDecimalValue; + } + } + + @Test + public void testSchemaCreationOnBigDecimalWithDecimalAnnotationToFixed() throws JsonMappingException { + // GIVEN + AvroSchemaGenerator gen = new AvroSchemaGenerator() + .enableLogicalTypes(); + + // WHEN + MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationToFixedWrapper.class, gen); + final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); + + System.out.println(BigDecimalWithDecimalAnnotationToFixedWrapper.class.getSimpleName() + " schema:\n" + actualSchema.toString(true)); + + // THEN + assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); + + Schema bigDecimalValue = actualSchema.getField("bigDecimalValue").schema(); + assertThat(bigDecimalValue.getType()).isEqualTo(Schema.Type.FIXED); + assertThat(bigDecimalValue.getFixedSize()).isEqualTo(10); + assertThat(bigDecimalValue.getLogicalType()).isEqualTo(LogicalTypes.decimal(6, 3)); + assertThat(bigDecimalValue.getProp("java-class")).isNull(); + } + public static class NamedAmount { public final String name; public final BigDecimal amount; @@ -20,13 +94,12 @@ public NamedAmount(@JsonProperty("name") String name, } public void testSerializeBigDecimal() throws Exception { - AvroMapper mapper = newMapper(); - AvroSchema schema = mapper.schemaFor(NamedAmount.class); + AvroSchema schema = MAPPER.schemaFor(NamedAmount.class); - byte[] bytes = mapper.writer(schema) + byte[] bytes = MAPPER.writer(schema) .writeValueAsBytes(new NamedAmount("peter", 42.0)); - NamedAmount result = mapper.reader(schema).forType(NamedAmount.class).readValue(bytes); + NamedAmount result = MAPPER.reader(schema).forType(NamedAmount.class).readValue(bytes); assertEquals("peter", result.name); assertEquals(BigDecimal.valueOf(42.0), result.amount); From 4dd982867bb8f9466d83d7f0ae88c20342f8a0a1 Mon Sep 17 00:00:00 2001 From: Michal Foksa Date: Thu, 2 Jan 2025 15:56:46 +0100 Subject: [PATCH 02/11] Serialization of java.math.BigDecimal to Avro `bytes` or `fixed` type added. --- .../avro/ser/NonBSGenericDatumWriter.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/ser/NonBSGenericDatumWriter.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/ser/NonBSGenericDatumWriter.java index 827e8bc23..5eae69333 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/ser/NonBSGenericDatumWriter.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/ser/NonBSGenericDatumWriter.java @@ -6,6 +6,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import org.apache.avro.Conversions.DecimalConversion; import org.apache.avro.Schema; import org.apache.avro.Schema.Type; import org.apache.avro.generic.GenericData; @@ -27,6 +28,8 @@ public class NonBSGenericDatumWriter private final static Class CLS_BIG_DECIMAL = BigDecimal.class; private final static Class CLS_BIG_INTEGER = BigInteger.class; + private final static DecimalConversion BIG_DECIMAL_CONVERSION = new DecimalConversion(); + public NonBSGenericDatumWriter(Schema root) { super(root); } @@ -97,6 +100,11 @@ protected void write(Schema schema, Object datum, Encoder out) throws IOExceptio super.writeWithoutConversion(schema, ByteBuffer.wrap((byte[]) datum), out); return; } + if (datum.getClass() == CLS_BIG_DECIMAL) { + super.writeWithoutConversion(schema, BIG_DECIMAL_CONVERSION.toBytes( + (BigDecimal) datum, schema, schema.getLogicalType()), out); + return; + } break; case FIXED: // One more mismatch to fix @@ -111,6 +119,11 @@ protected void write(Schema schema, Object datum, Encoder out) throws IOExceptio super.writeWithoutConversion(schema, new GenericData.Fixed(schema, (byte[]) datum), out); return; } + if (datum.getClass() == CLS_BIG_DECIMAL) { + super.writeWithoutConversion(schema, BIG_DECIMAL_CONVERSION.toFixed( + (BigDecimal) datum, schema, schema.getLogicalType()), out); + return; + } break; default: From 1816c380fc6886654a42a2a7b47c2195bc966fa4 Mon Sep 17 00:00:00 2001 From: Michal Foksa Date: Thu, 2 Jan 2025 16:10:16 +0100 Subject: [PATCH 03/11] Parsing Avro `bytes` or `fixed` types with logical type decimal into java.util.BigDecimal. --- .../dataformat/avro/annotation/Decimal.java | 5 +- .../dataformat/avro/deser/AvroParserImpl.java | 28 +++ .../avro/deser/AvroReaderFactory.java | 7 + .../dataformat/avro/deser/ScalarDecoder.java | 97 ++++++++++ .../dataformat/avro/BigDecimalTest.java | 179 +++++++++++++++--- 5 files changed, 289 insertions(+), 27 deletions(-) diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/Decimal.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/Decimal.java index c3e23ffcc..682786c50 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/Decimal.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/Decimal.java @@ -6,7 +6,8 @@ import java.lang.annotation.Target; /** - * Instructs the {@link com.fasterxml.jackson.dataformat.avro.schema.AvroSchemaGenerator AvroSchemaGenerator} + * When generate logical types is enabled, annotation instructs the + * {@link com.fasterxml.jackson.dataformat.avro.schema.AvroSchemaGenerator AvroSchemaGenerator} * to declare the annotated property's logical type as "decimal" ({@link org.apache.avro.LogicalTypes.Decimal}). * By default, the Avro type is "bytes" ({@link org.apache.avro.Schema.Type#BYTES}), unless the field is also * annotated with {@link com.fasterxml.jackson.dataformat.avro.AvroFixedSize}, in which case the Avro type @@ -29,6 +30,6 @@ /** * Scale must be zero or a positive integer less than or equal to the precision. */ - int scale() default 0; + int scale(); } diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java index d54893503..b2bd511ef 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java @@ -581,6 +581,34 @@ public long getRemainingElements() public abstract int decodeIndex() throws IOException; public abstract int decodeEnum() throws IOException; + /* + /********************************************************** + /* Methods for AvroReadContext implementations: decimals + /********************************************************** + */ + + public JsonToken decodeBytesDecimal(int scale) throws IOException { + decodeBytes(); + _numberBigDecimal = new BigDecimal(new BigInteger(_binaryValue), scale); + _numTypesValid = NR_BIGDECIMAL; + return JsonToken.VALUE_NUMBER_FLOAT; + } + + public void skipBytesDecimal() throws IOException { + skipBytes(); + } + + public JsonToken decodeFixedDecimal(int scale, int size) throws IOException { + decodeFixed(size); + _numberBigDecimal = new BigDecimal(new BigInteger(_binaryValue), scale); + _numTypesValid = NR_BIGDECIMAL; + return JsonToken.VALUE_NUMBER_FLOAT; + } + + public void skipFixedDecimal(int size) throws IOException { + skipFixed(size); + } + /* /********************************************************** /* Methods for AvroReadContext impls, other diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroReaderFactory.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroReaderFactory.java index b14f0bbd4..818fb7278 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroReaderFactory.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroReaderFactory.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.util.*; +import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import com.fasterxml.jackson.dataformat.avro.deser.ScalarDecoder.*; @@ -56,12 +57,18 @@ public ScalarDecoder createScalarValueDecoder(Schema type) case BOOLEAN: return READER_BOOLEAN; case BYTES: + if (type.getLogicalType() instanceof LogicalTypes.Decimal) { + return new BytesDecimalReader(((LogicalTypes.Decimal) type.getLogicalType()).getScale()); + } return READER_BYTES; case DOUBLE: return READER_DOUBLE; case ENUM: return new EnumDecoder(AvroSchemaHelper.getFullName(type), type.getEnumSymbols()); case FIXED: + if (type.getLogicalType() instanceof LogicalTypes.Decimal) { + return new FixedDecimalReader(((LogicalTypes.Decimal) type.getLogicalType()).getScale(), type.getFixedSize()); + } return new FixedDecoder(type.getFixedSize(), AvroSchemaHelper.getFullName(type)); case FLOAT: return READER_FLOAT; diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/ScalarDecoder.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/ScalarDecoder.java index 9f41bb611..7df76fd0e 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/ScalarDecoder.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/ScalarDecoder.java @@ -1,6 +1,7 @@ package com.fasterxml.jackson.dataformat.avro.deser; import java.io.IOException; +import java.math.BigDecimal; import java.util.List; import com.fasterxml.jackson.core.JsonToken; @@ -546,4 +547,100 @@ public void skipValue(AvroParserImpl parser) throws IOException { } } } + + protected final static class FixedDecimalReader extends ScalarDecoder { + private final int _scale; + private final int _size; + + public FixedDecimalReader(int scale, int size) { + _scale = scale; + _size = size; + } + + @Override + public JsonToken decodeValue(AvroParserImpl parser) throws IOException { + return parser.decodeFixedDecimal(_scale, _size); + } + + @Override + protected void skipValue(AvroParserImpl parser) throws IOException { + parser.skipFixedDecimal(_size); + } + + @Override + public String getTypeId() { + return AvroSchemaHelper.getTypeId(BigDecimal.class); + } + + @Override + public AvroFieldReader asFieldReader(String name, boolean skipper) { + return new FR(name, skipper, getTypeId(), _scale, _size); + } + + private final static class FR extends AvroFieldReader { + private final int _scale; + private final int _size; + public FR(String name, boolean skipper, String typeId, int scale, int size) { + super(name, skipper, typeId); + _scale = scale; + _size = size; + } + + @Override + public JsonToken readValue(AvroReadContext parent, AvroParserImpl parser) throws IOException { + return parser.decodeFixedDecimal(_scale, _size); + } + + @Override + public void skipValue(AvroParserImpl parser) throws IOException { + parser.skipFixedDecimal(_size); + } + } + } + + protected final static class BytesDecimalReader extends ScalarDecoder { + private final int _scale; + + public BytesDecimalReader(int scale) { + _scale = scale; + } + + @Override + public JsonToken decodeValue(AvroParserImpl parser) throws IOException { + return parser.decodeBytesDecimal(_scale); + } + + @Override + protected void skipValue(AvroParserImpl parser) throws IOException { + parser.skipBytesDecimal(); + } + + @Override + public String getTypeId() { + return AvroSchemaHelper.getTypeId(BigDecimal.class); + } + + @Override + public AvroFieldReader asFieldReader(String name, boolean skipper) { + return new FR(name, skipper, getTypeId(), _scale); + } + + private final static class FR extends AvroFieldReader { + private final int _scale; + public FR(String name, boolean skipper, String typeId, int scale) { + super(name, skipper, typeId); + _scale = scale; + } + + @Override + public JsonToken readValue(AvroReadContext parent, AvroParserImpl parser) throws IOException { + return parser.decodeBytesDecimal(_scale); + } + + @Override + public void skipValue(AvroParserImpl parser) throws IOException { + parser.skipFloat(); + } + } + } } diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java index a9553c16a..bf116573c 100644 --- a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java @@ -13,35 +13,55 @@ import static org.assertj.core.api.Assertions.assertThat; -public class BigDecimalTest extends AvroTestBase -{ +public class BigDecimalTest extends AvroTestBase { private static final AvroMapper MAPPER = new AvroMapper(); - static class BigDecimalWithDecimalAnnotationToBytesWrapper { - @JsonProperty(required = true) // field is made required only to have simpler avro schema + static class BigDecimalWithDecimalAnnotationWrapper { + @JsonProperty(required = true) // field is required to have simpler avro schema @Decimal(precision = 10, scale = 2) - public BigDecimal bigDecimalValue; + public final BigDecimal bigDecimalValue; - public BigDecimalWithDecimalAnnotationToBytesWrapper(BigDecimal bigDecimalValue) { + public BigDecimalWithDecimalAnnotationWrapper(BigDecimal bigDecimalValue) { this.bigDecimalValue = bigDecimalValue; } } @Test - public void testSchemaCreationOnBigDecimalWithDecimalAnnotationToBytes() throws JsonMappingException { + public void testSchemaCreation_withLogicalTypesDisabled_onBigDecimalWithDecimalAnnotation() throws JsonMappingException { // GIVEN AvroSchemaGenerator gen = new AvroSchemaGenerator() - .enableLogicalTypes(); + .disableLogicalTypes(); // WHEN - MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationToBytesWrapper.class, gen); + MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationWrapper.class, gen); + // actualSchema = MAPPER.schemaFor(BigDecimalWithDecimalAnnotationWrapper.class) would be enough in this case + // because logical types are disabled by default. final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); - System.out.println(BigDecimalWithDecimalAnnotationToBytesWrapper.class.getSimpleName() + " schema:\n" + actualSchema.toString(true)); + System.out.println(BigDecimalWithDecimalAnnotationWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); // THEN assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); + Schema bigDecimalValue = actualSchema.getField("bigDecimalValue").schema(); + assertThat(bigDecimalValue.getType()).isEqualTo(Schema.Type.STRING); + assertThat(bigDecimalValue.getLogicalType()).isNull(); + assertThat(bigDecimalValue.getProp("java-class")).isEqualTo("java.math.BigDecimal"); + } + + @Test + public void testSchemaCreation_withLogicalTypesEnabled_onBigDecimalWithDecimalAnnotation() throws JsonMappingException { + // GIVEN + AvroSchemaGenerator gen = new AvroSchemaGenerator() + .enableLogicalTypes(); + + // WHEN + MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationWrapper.class, gen); + final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); + + System.out.println(BigDecimalWithDecimalAnnotationWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); + // THEN + assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); Schema bigDecimalValue = actualSchema.getField("bigDecimalValue").schema(); assertThat(bigDecimalValue.getType()).isEqualTo(Schema.Type.BYTES); assertThat(bigDecimalValue.getLogicalType()).isEqualTo(LogicalTypes.decimal(10, 2)); @@ -49,10 +69,10 @@ public void testSchemaCreationOnBigDecimalWithDecimalAnnotationToBytes() throws } static class BigDecimalWithDecimalAnnotationToFixedWrapper { - @JsonProperty(required = true) // field is made required only to have simpler avro schema + @JsonProperty(required = true) // field is required to have simpler avro schema @AvroFixedSize(typeName = "BigDecimalWithDecimalAnnotationToFixedWrapper", size = 10) @Decimal(precision = 6, scale = 3) - public BigDecimal bigDecimalValue; + public final BigDecimal bigDecimalValue; public BigDecimalWithDecimalAnnotationToFixedWrapper(BigDecimal bigDecimalValue) { this.bigDecimalValue = bigDecimalValue; @@ -60,7 +80,7 @@ public BigDecimalWithDecimalAnnotationToFixedWrapper(BigDecimal bigDecimalValue) } @Test - public void testSchemaCreationOnBigDecimalWithDecimalAnnotationToFixed() throws JsonMappingException { + public void testSchemaCreation_withLogicalTypesEnabled_onBigDecimalWithDecimalAnnotationToFixed() throws JsonMappingException { // GIVEN AvroSchemaGenerator gen = new AvroSchemaGenerator() .enableLogicalTypes(); @@ -69,7 +89,7 @@ public void testSchemaCreationOnBigDecimalWithDecimalAnnotationToFixed() throws MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationToFixedWrapper.class, gen); final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); - System.out.println(BigDecimalWithDecimalAnnotationToFixedWrapper.class.getSimpleName() + " schema:\n" + actualSchema.toString(true)); + System.out.println(BigDecimalWithDecimalAnnotationToFixedWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); // THEN assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); @@ -78,30 +98,139 @@ public void testSchemaCreationOnBigDecimalWithDecimalAnnotationToFixed() throws assertThat(bigDecimalValue.getType()).isEqualTo(Schema.Type.FIXED); assertThat(bigDecimalValue.getFixedSize()).isEqualTo(10); assertThat(bigDecimalValue.getLogicalType()).isEqualTo(LogicalTypes.decimal(6, 3)); - assertThat(bigDecimalValue.getProp("java-class")).isNull(); + assertThat(bigDecimalValue.getProp("java-class")).isNull(); } - public static class NamedAmount { + static class BigDecimalAndName { + public final BigDecimal bigDecimalValue; public final String name; - public final BigDecimal amount; @JsonCreator - public NamedAmount(@JsonProperty("name") String name, - @JsonProperty("amount") double amount) { + public BigDecimalAndName( + @JsonProperty("bigDecimalValue") BigDecimal bigDecimalValue, + @JsonProperty("name") String name) { + this.bigDecimalValue = bigDecimalValue; this.name = name; - this.amount = BigDecimal.valueOf(amount); } } - public void testSerializeBigDecimal() throws Exception { - AvroSchema schema = MAPPER.schemaFor(NamedAmount.class); + // By default, BigDecimal is serialized to string + public void testSerialization_toString() throws Exception { + // GIVEN + String schemaString = "{" + + " \"type\" : \"record\"," + + " \"name\" : \"BigDecimalAndName\"," + + " \"namespace\" : \"test\"," + + " \"fields\" : [ {" + + " \"name\" : \"bigDecimalValue\"," + + " \"type\" : {" + + " \"type\" : \"string\"," + + " \"java-class\" : \"java.math.BigDecimal\"" + + " }" + + " }, {" + + " \"name\" : \"name\"," + + " \"type\" : \"string\"" + + " } ]" + + "}"; + + AvroSchema schema = MAPPER.schemaFrom(schemaString); + + // WHEN + // serialize + byte[] bytes = MAPPER.writer(schema) + .writeValueAsBytes(new BigDecimalAndName(BigDecimal.valueOf(42.2), "peter")); + + // deserialize + BigDecimalAndName result = MAPPER.reader(schema) + .forType(BigDecimalAndName.class) + .readValue(bytes); + + // THEN + assertEquals(BigDecimal.valueOf(42.2), result.bigDecimalValue); + assertEquals("peter", result.name); + } + + public void testSerialization_toBytesWithLogicalTypeDecimal() throws Exception { + // GIVEN + String schemaString = "{" + + " \"type\" : \"record\"," + + " \"name\" : \"BigDecimalAndName\"," + + " \"namespace\" : \"test\"," + + " \"fields\" : [ {" + + " \"name\" : \"bigDecimalValue\"," + + " \"type\" : [ \"null\", {" + + " \"type\" : \"bytes\"," + + " \"logicalType\" : \"decimal\"," + + " \"precision\" : 10," + + " \"scale\" : 2" + + " } ]" + + " }, {" + + " \"name\" : \"name\"," + + " \"type\" : [ \"null\", \"string\" ]" + + " } ]" + + "}"; + + AvroSchema schema = MAPPER.schemaFrom(schemaString); + + // WHEN + // serialize + byte[] bytes = MAPPER.writer(schema) + .writeValueAsBytes(new BigDecimalAndName( + new BigDecimal("42.2"), + "peter")); + + // deserialize + BigDecimalAndName result = MAPPER.reader(schema) + .forType(BigDecimalAndName.class) + .readValue(bytes); + + // THEN + // Because scale of decimal logical type is 2, result is with 2 decimal places + assertEquals(new BigDecimal("42.20"), result.bigDecimalValue); + assertEquals("peter", result.name); + } + + public void testSerialization_toFixedWithLogicalTypeDecimal() throws Exception { + // GIVEN + String schemaString = "{" + + " \"type\" : \"record\"," + + " \"name\" : \"BigDecimalAndName\"," + + " \"namespace\" : \"com.fasterxml.jackson.dataformat.avro.BigDecimalTest\"," + + " \"fields\" : [ {" + + " \"name\" : \"bigDecimalValue\"," + + " \"type\" : [ \"null\", {" + + " \"type\" : \"fixed\"," + + " \"name\" : \"BigDecimalValueType\"," + + " \"namespace\" : \"\"," + + " \"size\" : 10," + + " \"logicalType\" : \"decimal\"," + + " \"precision\" : 10," + + " \"scale\" : 2" + + " } ]" + + " }, {" + + " \"name\" : \"name\"," + + " \"type\" : [ \"null\", \"string\" ]" + + " } ]" + + "}"; + + AvroSchema schema = MAPPER.schemaFrom(schemaString); + // WHEN + // serialize byte[] bytes = MAPPER.writer(schema) - .writeValueAsBytes(new NamedAmount("peter", 42.0)); + .writeValueAsBytes(new BigDecimalAndName( + new BigDecimal("42.2"), + "peter")); - NamedAmount result = MAPPER.reader(schema).forType(NamedAmount.class).readValue(bytes); + // deserialize + BigDecimalAndName result = MAPPER.reader(schema) + .forType(BigDecimalAndName.class) + .readValue(bytes); + // THEN + // Because scale of decimal logical type is 2, result is with 2 decimal places + assertEquals(new BigDecimal("42.20"), result.bigDecimalValue); assertEquals("peter", result.name); - assertEquals(BigDecimal.valueOf(42.0), result.amount); } + } From 213fbe322c659686f7f7ca56cb3ea7859ee1392e Mon Sep 17 00:00:00 2001 From: Michal Foksa Date: Fri, 3 Jan 2025 15:49:00 +0100 Subject: [PATCH 04/11] assertEquals replaced with assertThat(actual).isEqualTo(expected); --- .../jackson/dataformat/avro/BigDecimalTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java index bf116573c..d0112b535 100644 --- a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java @@ -146,8 +146,8 @@ public void testSerialization_toString() throws Exception { .readValue(bytes); // THEN - assertEquals(BigDecimal.valueOf(42.2), result.bigDecimalValue); - assertEquals("peter", result.name); + assertThat(result.bigDecimalValue).isEqualTo(BigDecimal.valueOf(42.2)); + assertThat(result.name).isEqualTo("peter"); } public void testSerialization_toBytesWithLogicalTypeDecimal() throws Exception { @@ -186,8 +186,8 @@ public void testSerialization_toBytesWithLogicalTypeDecimal() throws Exception { // THEN // Because scale of decimal logical type is 2, result is with 2 decimal places - assertEquals(new BigDecimal("42.20"), result.bigDecimalValue); - assertEquals("peter", result.name); + assertThat(result.bigDecimalValue).isEqualTo(new BigDecimal("42.20")); + assertThat(result.name).isEqualTo("peter"); } public void testSerialization_toFixedWithLogicalTypeDecimal() throws Exception { @@ -229,8 +229,8 @@ public void testSerialization_toFixedWithLogicalTypeDecimal() throws Exception { // THEN // Because scale of decimal logical type is 2, result is with 2 decimal places - assertEquals(new BigDecimal("42.20"), result.bigDecimalValue); - assertEquals("peter", result.name); + assertThat(result.bigDecimalValue).isEqualTo(new BigDecimal("42.20")); + assertThat(result.name).isEqualTo("peter"); } } From 997ad53a7bdf39f25060b0fb97c6b2a5354271be Mon Sep 17 00:00:00 2001 From: Michal Foksa Date: Fri, 3 Jan 2025 21:30:52 +0100 Subject: [PATCH 05/11] Assert content of serialized bytes array. --- .../dataformat/avro/BigDecimalTest.java | 50 +++++++++++++++---- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java index d0112b535..09f71039b 100644 --- a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java @@ -135,12 +135,21 @@ public void testSerialization_toString() throws Exception { AvroSchema schema = MAPPER.schemaFrom(schemaString); - // WHEN - // serialize + // WHEN - serialize byte[] bytes = MAPPER.writer(schema) .writeValueAsBytes(new BigDecimalAndName(BigDecimal.valueOf(42.2), "peter")); - // deserialize + // THEN + assertThat(bytes).isEqualTo(new byte[]{ + // bigDecimalValue + 0x08, // -> 4 dec - bigDecimalValue property string value length + 0x34, 0x32, 0x2E, 0x32, // -> "42.2" in ASCII + // name + 0x0A, // -> 5 dec - name property string length + 0x70, 0x65, 0x74, 0x65, 0x72 // -> "peter" in ASCII + }); + + // WHEN - deserialize BigDecimalAndName result = MAPPER.reader(schema) .forType(BigDecimalAndName.class) .readValue(bytes); @@ -172,14 +181,24 @@ public void testSerialization_toBytesWithLogicalTypeDecimal() throws Exception { AvroSchema schema = MAPPER.schemaFrom(schemaString); - // WHEN - // serialize + // WHEN - serialize byte[] bytes = MAPPER.writer(schema) .writeValueAsBytes(new BigDecimalAndName( new BigDecimal("42.2"), "peter")); - - // deserialize + // THEN + assertThat(bytes).isEqualTo(new byte[]{ + // bigDecimalValue + 0x02, // -> 1 dec - second bigDecimalValue property type (bytes) + 0x04, // -> 2 dec - bigDecimalValue property bytes length + 0x10, 0x7C, // -> 0x107C -> 4220 dec - it is 42.2 value in scale 2. + // name + 0x02, // 1 dec - second name property type (string) + 0x0A, // -> 5 dec - name property string length + 0x70, 0x65, 0x74, 0x65, 0x72 // -> "peter" in ASCII + }); + + // WHEN - deserialize BigDecimalAndName result = MAPPER.reader(schema) .forType(BigDecimalAndName.class) .readValue(bytes); @@ -215,14 +234,25 @@ public void testSerialization_toFixedWithLogicalTypeDecimal() throws Exception { AvroSchema schema = MAPPER.schemaFrom(schemaString); - // WHEN - // serialize + // WHEN - serialize byte[] bytes = MAPPER.writer(schema) .writeValueAsBytes(new BigDecimalAndName( new BigDecimal("42.2"), "peter")); - // deserialize + // THEN + assertThat(bytes).isEqualTo(new byte[]{ + // bigDecimalValue + 0x02, // -> 1 dec - second bigDecimalValue property type (bytes) + // 10 bytes long fixed value + 0x00 ,0x00 ,0x00 ,0x00 ,0x00 ,0x00 ,0x00 ,0x00 ,0x10 ,0x7C, // -> 0x107C -> 4220 dec - it is 42.2 value in scale 2. + // name + 0x02, // 1 dec - second name property type (string) + 0x0A, // -> 5 dec - name property string length + 0x70, 0x65, 0x74, 0x65, 0x72 // -> "peter" in ASCII + }); + + // WHEN - deserialize BigDecimalAndName result = MAPPER.reader(schema) .forType(BigDecimalAndName.class) .readValue(bytes); From 57e63b1c515c732a67f571e26b31c7eb52ba84f3 Mon Sep 17 00:00:00 2001 From: Michal Foksa Date: Fri, 3 Jan 2025 21:35:34 +0100 Subject: [PATCH 06/11] Serialization-deserialization tests moved into BigDecimal_serialization_and_deserializationTest.java --- ...erialization_and_deserializationTest.java} | 93 +------------------ 1 file changed, 1 insertion(+), 92 deletions(-) rename avro/src/test/java/com/fasterxml/jackson/dataformat/avro/{BigDecimalTest.java => BigDecimal_serialization_and_deserializationTest.java} (60%) diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimal_serialization_and_deserializationTest.java similarity index 60% rename from avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java rename to avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimal_serialization_and_deserializationTest.java index 09f71039b..97e534314 100644 --- a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimalTest.java +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimal_serialization_and_deserializationTest.java @@ -2,105 +2,14 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.dataformat.avro.annotation.Decimal; -import com.fasterxml.jackson.dataformat.avro.schema.AvroSchemaGenerator; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.junit.Test; import java.math.BigDecimal; import static org.assertj.core.api.Assertions.assertThat; -public class BigDecimalTest extends AvroTestBase { +public class BigDecimal_serialization_and_deserializationTest extends AvroTestBase { private static final AvroMapper MAPPER = new AvroMapper(); - static class BigDecimalWithDecimalAnnotationWrapper { - @JsonProperty(required = true) // field is required to have simpler avro schema - @Decimal(precision = 10, scale = 2) - public final BigDecimal bigDecimalValue; - - public BigDecimalWithDecimalAnnotationWrapper(BigDecimal bigDecimalValue) { - this.bigDecimalValue = bigDecimalValue; - } - } - - @Test - public void testSchemaCreation_withLogicalTypesDisabled_onBigDecimalWithDecimalAnnotation() throws JsonMappingException { - // GIVEN - AvroSchemaGenerator gen = new AvroSchemaGenerator() - .disableLogicalTypes(); - - // WHEN - MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationWrapper.class, gen); - // actualSchema = MAPPER.schemaFor(BigDecimalWithDecimalAnnotationWrapper.class) would be enough in this case - // because logical types are disabled by default. - final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); - - System.out.println(BigDecimalWithDecimalAnnotationWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); - - // THEN - assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); - Schema bigDecimalValue = actualSchema.getField("bigDecimalValue").schema(); - assertThat(bigDecimalValue.getType()).isEqualTo(Schema.Type.STRING); - assertThat(bigDecimalValue.getLogicalType()).isNull(); - assertThat(bigDecimalValue.getProp("java-class")).isEqualTo("java.math.BigDecimal"); - } - - @Test - public void testSchemaCreation_withLogicalTypesEnabled_onBigDecimalWithDecimalAnnotation() throws JsonMappingException { - // GIVEN - AvroSchemaGenerator gen = new AvroSchemaGenerator() - .enableLogicalTypes(); - - // WHEN - MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationWrapper.class, gen); - final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); - - System.out.println(BigDecimalWithDecimalAnnotationWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); - - // THEN - assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); - Schema bigDecimalValue = actualSchema.getField("bigDecimalValue").schema(); - assertThat(bigDecimalValue.getType()).isEqualTo(Schema.Type.BYTES); - assertThat(bigDecimalValue.getLogicalType()).isEqualTo(LogicalTypes.decimal(10, 2)); - assertThat(bigDecimalValue.getProp("java-class")).isNull(); - } - - static class BigDecimalWithDecimalAnnotationToFixedWrapper { - @JsonProperty(required = true) // field is required to have simpler avro schema - @AvroFixedSize(typeName = "BigDecimalWithDecimalAnnotationToFixedWrapper", size = 10) - @Decimal(precision = 6, scale = 3) - public final BigDecimal bigDecimalValue; - - public BigDecimalWithDecimalAnnotationToFixedWrapper(BigDecimal bigDecimalValue) { - this.bigDecimalValue = bigDecimalValue; - } - } - - @Test - public void testSchemaCreation_withLogicalTypesEnabled_onBigDecimalWithDecimalAnnotationToFixed() throws JsonMappingException { - // GIVEN - AvroSchemaGenerator gen = new AvroSchemaGenerator() - .enableLogicalTypes(); - - // WHEN - MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationToFixedWrapper.class, gen); - final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); - - System.out.println(BigDecimalWithDecimalAnnotationToFixedWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); - - // THEN - assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); - - Schema bigDecimalValue = actualSchema.getField("bigDecimalValue").schema(); - assertThat(bigDecimalValue.getType()).isEqualTo(Schema.Type.FIXED); - assertThat(bigDecimalValue.getFixedSize()).isEqualTo(10); - assertThat(bigDecimalValue.getLogicalType()).isEqualTo(LogicalTypes.decimal(6, 3)); - assertThat(bigDecimalValue.getProp("java-class")).isNull(); - } - static class BigDecimalAndName { public final BigDecimal bigDecimalValue; public final String name; From 879afff7049a532becbe81a032912ea1cdfdf33b Mon Sep 17 00:00:00 2001 From: Michal Foksa Date: Fri, 3 Jan 2025 21:38:36 +0100 Subject: [PATCH 07/11] Schema creation tests moved into BigDecimal_schemaCreationTest.java --- .../avro/BigDecimal_schemaCreationTest.java | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimal_schemaCreationTest.java diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimal_schemaCreationTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimal_schemaCreationTest.java new file mode 100644 index 000000000..a366c5e52 --- /dev/null +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimal_schemaCreationTest.java @@ -0,0 +1,102 @@ +package com.fasterxml.jackson.dataformat.avro; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.dataformat.avro.annotation.Decimal; +import com.fasterxml.jackson.dataformat.avro.schema.AvroSchemaGenerator; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.junit.Test; + +import java.math.BigDecimal; + +import static org.assertj.core.api.Assertions.assertThat; + +public class BigDecimal_schemaCreationTest extends AvroTestBase { + private static final AvroMapper MAPPER = new AvroMapper(); + + static class BigDecimalWithDecimalAnnotationWrapper { + @JsonProperty(required = true) // field is required to have simpler avro schema + @Decimal(precision = 10, scale = 2) + public final BigDecimal bigDecimalValue; + + public BigDecimalWithDecimalAnnotationWrapper(BigDecimal bigDecimalValue) { + this.bigDecimalValue = bigDecimalValue; + } + } + + @Test + public void testSchemaCreation_withLogicalTypesDisabled_onBigDecimalWithDecimalAnnotation() throws JsonMappingException { + // GIVEN + AvroSchemaGenerator gen = new AvroSchemaGenerator() + .disableLogicalTypes(); + + // WHEN + MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationWrapper.class, gen); + // actualSchema = MAPPER.schemaFor(BigDecimalWithDecimalAnnotationWrapper.class) would be enough in this case + // because logical types are disabled by default. + final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); + + System.out.println(BigDecimalWithDecimalAnnotationWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); + + // THEN + assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); + Schema bigDecimalValue = actualSchema.getField("bigDecimalValue").schema(); + assertThat(bigDecimalValue.getType()).isEqualTo(Schema.Type.STRING); + assertThat(bigDecimalValue.getLogicalType()).isNull(); + assertThat(bigDecimalValue.getProp("java-class")).isEqualTo("java.math.BigDecimal"); + } + + @Test + public void testSchemaCreation_withLogicalTypesEnabled_onBigDecimalWithDecimalAnnotation() throws JsonMappingException { + // GIVEN + AvroSchemaGenerator gen = new AvroSchemaGenerator() + .enableLogicalTypes(); + + // WHEN + MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationWrapper.class, gen); + final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); + + System.out.println(BigDecimalWithDecimalAnnotationWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); + + // THEN + assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); + Schema bigDecimalValue = actualSchema.getField("bigDecimalValue").schema(); + assertThat(bigDecimalValue.getType()).isEqualTo(Schema.Type.BYTES); + assertThat(bigDecimalValue.getLogicalType()).isEqualTo(LogicalTypes.decimal(10, 2)); + assertThat(bigDecimalValue.getProp("java-class")).isNull(); + } + + static class BigDecimalWithDecimalAnnotationToFixedWrapper { + @JsonProperty(required = true) // field is required to have simpler avro schema + @AvroFixedSize(typeName = "BigDecimalWithDecimalAnnotationToFixedWrapper", size = 10) + @Decimal(precision = 6, scale = 3) + public final BigDecimal bigDecimalValue; + + public BigDecimalWithDecimalAnnotationToFixedWrapper(BigDecimal bigDecimalValue) { + this.bigDecimalValue = bigDecimalValue; + } + } + + @Test + public void testSchemaCreation_withLogicalTypesEnabled_onBigDecimalWithDecimalAnnotationToFixed() throws JsonMappingException { + // GIVEN + AvroSchemaGenerator gen = new AvroSchemaGenerator() + .enableLogicalTypes(); + + // WHEN + MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationToFixedWrapper.class, gen); + final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); + + System.out.println(BigDecimalWithDecimalAnnotationToFixedWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); + + // THEN + assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); + + Schema bigDecimalValue = actualSchema.getField("bigDecimalValue").schema(); + assertThat(bigDecimalValue.getType()).isEqualTo(Schema.Type.FIXED); + assertThat(bigDecimalValue.getFixedSize()).isEqualTo(10); + assertThat(bigDecimalValue.getLogicalType()).isEqualTo(LogicalTypes.decimal(6, 3)); + assertThat(bigDecimalValue.getProp("java-class")).isNull(); + } +} From 0e9f56237685ff53b4db9b23fd5099d1ee522157 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Fri, 3 Jan 2025 21:30:57 -0800 Subject: [PATCH 08/11] Minor streamlining --- .../dataformat/avro/schema/RecordVisitor.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/RecordVisitor.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/RecordVisitor.java index 8b1419492..715bab834 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/RecordVisitor.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/RecordVisitor.java @@ -4,7 +4,6 @@ import java.util.List; import java.util.Map; -import com.fasterxml.jackson.dataformat.avro.annotation.Decimal; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.Schema.Type; @@ -17,6 +16,7 @@ import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.ser.BeanPropertyWriter; import com.fasterxml.jackson.dataformat.avro.AvroFixedSize; +import com.fasterxml.jackson.dataformat.avro.annotation.Decimal; import com.fasterxml.jackson.dataformat.avro.ser.CustomEncodingSerializer; public class RecordVisitor @@ -150,17 +150,19 @@ protected Schema.Field schemaFieldForWriter(BeanProperty prop, boolean optional) Schema.Parser parser = new Schema.Parser(); writerSchema = parser.parse(schemaOverride.value()); } else { - if (prop.getAnnotation(AvroFixedSize.class) != null) { - AvroFixedSize fixedSize = prop.getAnnotation(AvroFixedSize.class); + AvroFixedSize fixedSize = prop.getAnnotation(AvroFixedSize.class); + if (fixedSize != null) { writerSchema = Schema.createFixed(fixedSize.typeName(), null, fixedSize.typeNamespace(), fixedSize.size()); } - if (_visitorWrapper.isLogicalTypesEnabled() && prop.getAnnotation(Decimal.class) != null) { - if (writerSchema == null) { - writerSchema = Schema.create(Type.BYTES); - } + if (_visitorWrapper.isLogicalTypesEnabled()) { Decimal decimal = prop.getAnnotation(Decimal.class); - writerSchema = LogicalTypes.decimal(decimal.precision(), decimal.scale()) - .addToSchema(writerSchema); + if (decimal != null) { + if (writerSchema == null) { + writerSchema = Schema.create(Type.BYTES); + } + writerSchema = LogicalTypes.decimal(decimal.precision(), decimal.scale()) + .addToSchema(writerSchema); + } } if (writerSchema == null) { JsonSerializer ser = null; @@ -169,9 +171,7 @@ protected Schema.Field schemaFieldForWriter(BeanProperty prop, boolean optional) if (prop instanceof BeanPropertyWriter) { BeanPropertyWriter bpw = (BeanPropertyWriter) prop; ser = bpw.getSerializer(); - /* - * 2-Mar-2017, bryan: AvroEncode annotation expects to have the schema used directly - */ + // 2-Mar-2017, bryan: AvroEncode annotation expects to have the schema used directly optional = optional && !(ser instanceof CustomEncodingSerializer); // Don't modify schema } final SerializerProvider prov = getProvider(); From 00bcdc5e2d4b3ce0eb4b20ace95713b0f7e8e2e0 Mon Sep 17 00:00:00 2001 From: Michal Foksa Date: Sat, 4 Jan 2025 16:41:55 +0100 Subject: [PATCH 09/11] @Decimal annotation renamed to @AvroDecinal --- .../{Decimal.java => AvroDecimal.java} | 2 +- .../dataformat/avro/schema/RecordVisitor.java | 8 ++--- .../avro/BigDecimal_schemaCreationTest.java | 36 +++++++++---------- 3 files changed, 23 insertions(+), 23 deletions(-) rename avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/{Decimal.java => AvroDecimal.java} (97%) diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/Decimal.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/AvroDecimal.java similarity index 97% rename from avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/Decimal.java rename to avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/AvroDecimal.java index 682786c50..5205db285 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/Decimal.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/annotation/AvroDecimal.java @@ -20,7 +20,7 @@ */ @Target({ElementType.ANNOTATION_TYPE, ElementType.FIELD}) @Retention(RetentionPolicy.RUNTIME) -public @interface Decimal { +public @interface AvroDecimal { /** * Maximum precision of decimals stored in this type. diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/RecordVisitor.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/RecordVisitor.java index 715bab834..8aa8f9b32 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/RecordVisitor.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/RecordVisitor.java @@ -16,7 +16,7 @@ import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.ser.BeanPropertyWriter; import com.fasterxml.jackson.dataformat.avro.AvroFixedSize; -import com.fasterxml.jackson.dataformat.avro.annotation.Decimal; +import com.fasterxml.jackson.dataformat.avro.annotation.AvroDecimal; import com.fasterxml.jackson.dataformat.avro.ser.CustomEncodingSerializer; public class RecordVisitor @@ -155,12 +155,12 @@ protected Schema.Field schemaFieldForWriter(BeanProperty prop, boolean optional) writerSchema = Schema.createFixed(fixedSize.typeName(), null, fixedSize.typeNamespace(), fixedSize.size()); } if (_visitorWrapper.isLogicalTypesEnabled()) { - Decimal decimal = prop.getAnnotation(Decimal.class); - if (decimal != null) { + AvroDecimal avroDecimal = prop.getAnnotation(AvroDecimal.class); + if (avroDecimal != null) { if (writerSchema == null) { writerSchema = Schema.create(Type.BYTES); } - writerSchema = LogicalTypes.decimal(decimal.precision(), decimal.scale()) + writerSchema = LogicalTypes.decimal(avroDecimal.precision(), avroDecimal.scale()) .addToSchema(writerSchema); } } diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimal_schemaCreationTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimal_schemaCreationTest.java index a366c5e52..48d8722e3 100644 --- a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimal_schemaCreationTest.java +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/BigDecimal_schemaCreationTest.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.dataformat.avro.annotation.Decimal; +import com.fasterxml.jackson.dataformat.avro.annotation.AvroDecimal; import com.fasterxml.jackson.dataformat.avro.schema.AvroSchemaGenerator; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; @@ -15,29 +15,29 @@ public class BigDecimal_schemaCreationTest extends AvroTestBase { private static final AvroMapper MAPPER = new AvroMapper(); - static class BigDecimalWithDecimalAnnotationWrapper { + static class BigDecimalWithAvroDecimalAnnotationWrapper { @JsonProperty(required = true) // field is required to have simpler avro schema - @Decimal(precision = 10, scale = 2) + @AvroDecimal(precision = 10, scale = 2) public final BigDecimal bigDecimalValue; - public BigDecimalWithDecimalAnnotationWrapper(BigDecimal bigDecimalValue) { + public BigDecimalWithAvroDecimalAnnotationWrapper(BigDecimal bigDecimalValue) { this.bigDecimalValue = bigDecimalValue; } } @Test - public void testSchemaCreation_withLogicalTypesDisabled_onBigDecimalWithDecimalAnnotation() throws JsonMappingException { + public void testSchemaCreation_withLogicalTypesDisabled_onBigDecimalWithAvroDecimalAnnotation() throws JsonMappingException { // GIVEN AvroSchemaGenerator gen = new AvroSchemaGenerator() .disableLogicalTypes(); // WHEN - MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationWrapper.class, gen); - // actualSchema = MAPPER.schemaFor(BigDecimalWithDecimalAnnotationWrapper.class) would be enough in this case + MAPPER.acceptJsonFormatVisitor(BigDecimalWithAvroDecimalAnnotationWrapper.class, gen); + // actualSchema = MAPPER.schemaFor(BigDecimalWithAvroDecimalAnnotationWrapper.class) would be enough in this case // because logical types are disabled by default. final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); - System.out.println(BigDecimalWithDecimalAnnotationWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); + System.out.println(BigDecimalWithAvroDecimalAnnotationWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); // THEN assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); @@ -48,16 +48,16 @@ public void testSchemaCreation_withLogicalTypesDisabled_onBigDecimalWithDecimalA } @Test - public void testSchemaCreation_withLogicalTypesEnabled_onBigDecimalWithDecimalAnnotation() throws JsonMappingException { + public void testSchemaCreation_withLogicalTypesEnabled_onBigDecimalWithAvroDecimalAnnotation() throws JsonMappingException { // GIVEN AvroSchemaGenerator gen = new AvroSchemaGenerator() .enableLogicalTypes(); // WHEN - MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationWrapper.class, gen); + MAPPER.acceptJsonFormatVisitor(BigDecimalWithAvroDecimalAnnotationWrapper.class, gen); final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); - System.out.println(BigDecimalWithDecimalAnnotationWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); + System.out.println(BigDecimalWithAvroDecimalAnnotationWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); // THEN assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); @@ -67,28 +67,28 @@ public void testSchemaCreation_withLogicalTypesEnabled_onBigDecimalWithDecimalAn assertThat(bigDecimalValue.getProp("java-class")).isNull(); } - static class BigDecimalWithDecimalAnnotationToFixedWrapper { + static class BigDecimalWithAvroDecimalAnnotationToFixedWrapper { @JsonProperty(required = true) // field is required to have simpler avro schema - @AvroFixedSize(typeName = "BigDecimalWithDecimalAnnotationToFixedWrapper", size = 10) - @Decimal(precision = 6, scale = 3) + @AvroFixedSize(typeName = "BigDecimalWithAvroDecimalAnnotationToFixedWrapper", size = 10) + @AvroDecimal(precision = 6, scale = 3) public final BigDecimal bigDecimalValue; - public BigDecimalWithDecimalAnnotationToFixedWrapper(BigDecimal bigDecimalValue) { + public BigDecimalWithAvroDecimalAnnotationToFixedWrapper(BigDecimal bigDecimalValue) { this.bigDecimalValue = bigDecimalValue; } } @Test - public void testSchemaCreation_withLogicalTypesEnabled_onBigDecimalWithDecimalAnnotationToFixed() throws JsonMappingException { + public void testSchemaCreation_withLogicalTypesEnabled_onBigDecimalWithAvroDecimalAnnotationToFixed() throws JsonMappingException { // GIVEN AvroSchemaGenerator gen = new AvroSchemaGenerator() .enableLogicalTypes(); // WHEN - MAPPER.acceptJsonFormatVisitor(BigDecimalWithDecimalAnnotationToFixedWrapper.class, gen); + MAPPER.acceptJsonFormatVisitor(BigDecimalWithAvroDecimalAnnotationToFixedWrapper.class, gen); final Schema actualSchema = gen.getGeneratedSchema().getAvroSchema(); - System.out.println(BigDecimalWithDecimalAnnotationToFixedWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); + System.out.println(BigDecimalWithAvroDecimalAnnotationToFixedWrapper.class.getSimpleName() + " schema:" + actualSchema.toString(true)); // THEN assertThat(actualSchema.getField("bigDecimalValue")).isNotNull(); From bae034645d63719b99baaab1807d7e68b94c5c3f Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Sat, 4 Jan 2025 15:13:53 -0800 Subject: [PATCH 10/11] Update release notes --- release-notes/CREDITS-2.x | 15 +++++++++++++++ release-notes/VERSION-2.x | 6 ++++++ 2 files changed, 21 insertions(+) diff --git a/release-notes/CREDITS-2.x b/release-notes/CREDITS-2.x index 20b109095..82fd2be1c 100644 --- a/release-notes/CREDITS-2.x +++ b/release-notes/CREDITS-2.x @@ -219,12 +219,17 @@ Michal Foksa (MichalFoksa@github) (2.13.0) * Contributed #290: (avro) Generate logicalType switch (2.13.0) +* Contributed fix for #308: (avro) Incorrect serialization for `LogicalType.Decimal` + (Java `BigDecimal`) * Contributed #310: (avro) Avro schema generation: allow override namespace with new `@AvroNamespace` annotation (2.14.0) * Contributed #494: Avro Schema generation: allow mapping Java Enum properties to Avro String values (2.18.0) +* Contributed fix for #535: (avro) AvroSchemaGenerator: logicalType(s) never set + for non-date classes + (2.19.0) * Contributed #536: (avro) Add Logical Type support for `java.util.UUID` (2.19.0) @@ -357,3 +362,13 @@ Robert Noack (@mr-robert) Knut Wannheden (@knutwannheden) * Contributed #518: Should not read past end for CBOR string values (2.18.1) + +Idan Sheinberg (@sheinbergon) + * Reported #308: (avro) Incorrect serialization for `LogicalType.Decimal` (Java + `BigDecimal`) + (2.19.0) + +Cormac Redmond (@credmond) + * Reported #535: (avro) AvroSchemaGenerator: logicalType(s) never set for + non-Date classes + (2.19.0) diff --git a/release-notes/VERSION-2.x b/release-notes/VERSION-2.x index c60c55636..8f770bcc2 100644 --- a/release-notes/VERSION-2.x +++ b/release-notes/VERSION-2.x @@ -16,6 +16,12 @@ Active maintainers: 2.19.0 (not yet released) +#308: (avro) Incorrect serialization for `LogicalType.Decimal` (Java `BigDecimal`) + (reported by Idan S) + (fix contributed by Michal F) +#535: (avro) AvroSchemaGenerator: logicalType(s) never set for non-date classes + (reported by Cormac R) + (fix contributed by Michal F) #536: (avro) Add Logical Type support for `java.util.UUID` (contributed by Michal F) From 3be3c21bc730a98c0d80721995de03aca935de81 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Sat, 4 Jan 2025 15:22:04 -0800 Subject: [PATCH 11/11] Minor javadoc/comment adds --- .../jackson/dataformat/avro/deser/AvroParserImpl.java | 4 ++++ .../jackson/dataformat/avro/deser/ScalarDecoder.java | 6 ++++++ .../dataformat/avro/ser/NonBSGenericDatumWriter.java | 1 + 3 files changed, 11 insertions(+) diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java index b2bd511ef..a0c3393dc 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/AvroParserImpl.java @@ -587,6 +587,7 @@ public long getRemainingElements() /********************************************************** */ + // @since 2.19 public JsonToken decodeBytesDecimal(int scale) throws IOException { decodeBytes(); _numberBigDecimal = new BigDecimal(new BigInteger(_binaryValue), scale); @@ -594,10 +595,12 @@ public JsonToken decodeBytesDecimal(int scale) throws IOException { return JsonToken.VALUE_NUMBER_FLOAT; } + // @since 2.19 public void skipBytesDecimal() throws IOException { skipBytes(); } + // @since 2.19 public JsonToken decodeFixedDecimal(int scale, int size) throws IOException { decodeFixed(size); _numberBigDecimal = new BigDecimal(new BigInteger(_binaryValue), scale); @@ -605,6 +608,7 @@ public JsonToken decodeFixedDecimal(int scale, int size) throws IOException { return JsonToken.VALUE_NUMBER_FLOAT; } + // @since 2.19 public void skipFixedDecimal(int size) throws IOException { skipFixed(size); } diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/ScalarDecoder.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/ScalarDecoder.java index 7df76fd0e..1c99c51bd 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/ScalarDecoder.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/deser/ScalarDecoder.java @@ -548,6 +548,9 @@ public void skipValue(AvroParserImpl parser) throws IOException { } } + /** + * @since 2.19 + */ protected final static class FixedDecimalReader extends ScalarDecoder { private final int _scale; private final int _size; @@ -598,6 +601,9 @@ public void skipValue(AvroParserImpl parser) throws IOException { } } + /** + * @since 2.19 + */ protected final static class BytesDecimalReader extends ScalarDecoder { private final int _scale; diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/ser/NonBSGenericDatumWriter.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/ser/NonBSGenericDatumWriter.java index 5eae69333..076fb1190 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/ser/NonBSGenericDatumWriter.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/ser/NonBSGenericDatumWriter.java @@ -28,6 +28,7 @@ public class NonBSGenericDatumWriter private final static Class CLS_BIG_DECIMAL = BigDecimal.class; private final static Class CLS_BIG_INTEGER = BigInteger.class; + // @since 2.19 private final static DecimalConversion BIG_DECIMAL_CONVERSION = new DecimalConversion(); public NonBSGenericDatumWriter(Schema root) {