From f3ea4490f1eedf6de7a7063f4fc625262b2267f0 Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Fri, 6 Jul 2018 15:24:42 +0200 Subject: [PATCH 01/18] Added end-to-end test that reproduces union with logical types problem --- lang/java/compiler/pom.xml | 19 ++++++++++ .../TestEndToEndJavaCodeGeneration.java | 37 +++++++++++++++++++ .../avro/union_with_logical_types.avsc | 12 ++++++ 3 files changed, 68 insertions(+) create mode 100644 lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestEndToEndJavaCodeGeneration.java create mode 100644 lang/java/compiler/src/test/resources/avro/union_with_logical_types.avsc diff --git a/lang/java/compiler/pom.xml b/lang/java/compiler/pom.xml index ee260c7f14b..2078c3dbabd 100644 --- a/lang/java/compiler/pom.xml +++ b/lang/java/compiler/pom.xml @@ -113,6 +113,25 @@ + + org.apache.avro + avro-maven-plugin + 1.9.0-SNAPSHOT + + + generate-test-sources + + schema + protocol + idl-protocol + + + ${project.basedir}/src/test/resources/avro + + + + + diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestEndToEndJavaCodeGeneration.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestEndToEndJavaCodeGeneration.java new file mode 100644 index 00000000000..ac4189d44e3 --- /dev/null +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestEndToEndJavaCodeGeneration.java @@ -0,0 +1,37 @@ +package org.apache.avro.compiler.specific; + +import org.apache.avro.compiler.testdata.UnionWithLogicalTypes; +import org.joda.time.LocalDate; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; + +public class TestEndToEndJavaCodeGeneration { + + @Test + public void testWithNullValues() throws IOException { + UnionWithLogicalTypes instanceOfGeneratedClass = UnionWithLogicalTypes.newBuilder() + .setDateOrNull(null) + .setStringOrNull("hello") + .build(); + final ByteBuffer bb = instanceOfGeneratedClass.toByteBuffer(); + final UnionWithLogicalTypes copy = UnionWithLogicalTypes.fromByteBuffer(bb); + Assert.assertEquals(instanceOfGeneratedClass.getDateOrNull(), copy.getDateOrNull()); + Assert.assertEquals(instanceOfGeneratedClass.getStringOrNull(), copy.getStringOrNull()); + } + + @Test + public void testNonNullValues() throws IOException { + UnionWithLogicalTypes instanceOfGeneratedClass = UnionWithLogicalTypes.newBuilder() + .setDateOrNull(LocalDate.now()) + .setStringOrNull("hello") + .build(); + final ByteBuffer bb = instanceOfGeneratedClass.toByteBuffer(); + final UnionWithLogicalTypes copy = UnionWithLogicalTypes.fromByteBuffer(bb); + Assert.assertEquals(instanceOfGeneratedClass.getDateOrNull(), copy.getDateOrNull()); + Assert.assertEquals(instanceOfGeneratedClass.getStringOrNull(), copy.getStringOrNull()); + } + +} diff --git a/lang/java/compiler/src/test/resources/avro/union_with_logical_types.avsc b/lang/java/compiler/src/test/resources/avro/union_with_logical_types.avsc new file mode 100644 index 00000000000..98fda363f7c --- /dev/null +++ b/lang/java/compiler/src/test/resources/avro/union_with_logical_types.avsc @@ -0,0 +1,12 @@ +{"namespace": "org.apache.avro.compiler.testdata", + "type": "record", + "name": "UnionWithLogicalTypes", + "doc" : "Test unions with logical types in generated Java classes", + "fields": [ + {"name": "stringOrNull", "type": ["null", {"type": "string", "java-class": "java.lang.String"}], "default": null}, + {"name": "dateOrNull", "type": ["null", {"type": "int", "logicalType": "date"}], "default": null} + ] +} + + + From 2b745742f11e8c823155232cf431942e31f0751c Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Mon, 6 Aug 2018 14:27:23 +0200 Subject: [PATCH 02/18] Adding required conversions to SpecificData in generated class (same as in SpecificCompiler) --- .../avro/compiler/specific/templates/java/classic/record.vm | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm index 045c7e1ef3b..f68a6a51b89 100644 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm @@ -39,6 +39,12 @@ public class ${this.mangle($schema.getName())}#if ($schema.isError()) extends or public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } private static SpecificData MODEL$ = new SpecificData(); + static { + MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.DateConversion()); + MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimeConversion()); + MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimestampConversion()); + MODEL$.addLogicalTypeConversion(new org.apache.avro.Conversions.DecimalConversion()); + } #if (!$schema.isError()) private static final BinaryMessageEncoder<${this.mangle($schema.getName())}> ENCODER = From be3d3fbd0b774a6fec1e8ce5357010edf3a61fcc Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Mon, 6 Aug 2018 15:47:36 +0200 Subject: [PATCH 03/18] Added test with BigDecimal --- .../src/test/resources/avro/union_with_logical_types.avsc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lang/java/compiler/src/test/resources/avro/union_with_logical_types.avsc b/lang/java/compiler/src/test/resources/avro/union_with_logical_types.avsc index 98fda363f7c..66dee40ea51 100644 --- a/lang/java/compiler/src/test/resources/avro/union_with_logical_types.avsc +++ b/lang/java/compiler/src/test/resources/avro/union_with_logical_types.avsc @@ -4,7 +4,8 @@ "doc" : "Test unions with logical types in generated Java classes", "fields": [ {"name": "stringOrNull", "type": ["null", {"type": "string", "java-class": "java.lang.String"}], "default": null}, - {"name": "dateOrNull", "type": ["null", {"type": "int", "logicalType": "date"}], "default": null} + {"name": "dateOrNull", "type": ["null", {"type": "int", "logicalType": "date"}], "default": null}, + {"name": "decimalOrNull", "type": ["null", {"type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 2}], "default": null} ] } From f251c63778252637223e2911e03e0565e0b8858a Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Mon, 6 Aug 2018 15:48:36 +0200 Subject: [PATCH 04/18] Added test with BigDecimal --- lang/java/compiler/pom.xml | 1 + .../specific/TestEndToEndJavaCodeGeneration.java | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lang/java/compiler/pom.xml b/lang/java/compiler/pom.xml index 2078c3dbabd..2e1179e4252 100644 --- a/lang/java/compiler/pom.xml +++ b/lang/java/compiler/pom.xml @@ -127,6 +127,7 @@ ${project.basedir}/src/test/resources/avro + true diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestEndToEndJavaCodeGeneration.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestEndToEndJavaCodeGeneration.java index ac4189d44e3..051fced457d 100644 --- a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestEndToEndJavaCodeGeneration.java +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestEndToEndJavaCodeGeneration.java @@ -6,6 +6,7 @@ import org.junit.Test; import java.io.IOException; +import java.math.BigDecimal; import java.nio.ByteBuffer; public class TestEndToEndJavaCodeGeneration { @@ -23,7 +24,7 @@ public void testWithNullValues() throws IOException { } @Test - public void testNonNullValues() throws IOException { + public void testDate() throws IOException { UnionWithLogicalTypes instanceOfGeneratedClass = UnionWithLogicalTypes.newBuilder() .setDateOrNull(LocalDate.now()) .setStringOrNull("hello") @@ -34,4 +35,16 @@ public void testNonNullValues() throws IOException { Assert.assertEquals(instanceOfGeneratedClass.getStringOrNull(), copy.getStringOrNull()); } + @Test + public void testDecimal() throws IOException { + UnionWithLogicalTypes instanceOfGeneratedClass = UnionWithLogicalTypes.newBuilder() + .setStringOrNull("hello") + .setDecimalOrNull(BigDecimal.valueOf(123, 2)) + .build(); + final ByteBuffer bb = instanceOfGeneratedClass.toByteBuffer(); + final UnionWithLogicalTypes copy = UnionWithLogicalTypes.fromByteBuffer(bb); + Assert.assertEquals(instanceOfGeneratedClass.getDecimalOrNull(), copy.getDecimalOrNull()); + Assert.assertEquals(instanceOfGeneratedClass.getStringOrNull(), copy.getStringOrNull()); + } + } From 5e9917bf7a5d8f98f34123736084b19430884492 Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Thu, 9 Aug 2018 16:03:23 +0200 Subject: [PATCH 05/18] Introduced customizable conversions in compiler and Maven plugin. --- lang/java/codegen-test/pom.xml | 82 +++++++++++++++++++ .../CustomConversion.java | 30 +++++++ .../TestEndToEndJavaCodeGeneration.java | 7 +- .../avro/union_with_logical_types.avsc | 2 +- lang/java/compiler/pom.xml | 20 ----- .../compiler/specific/SpecificCompiler.java | 23 ++++++ .../specific/templates/java/classic/record.vm | 7 +- .../apache/avro/mojo/AbstractAvroMojo.java | 8 ++ .../org/apache/avro/mojo/IDLProtocolMojo.java | 3 + .../org/apache/avro/mojo/ProtocolMojo.java | 3 + .../java/org/apache/avro/mojo/SchemaMojo.java | 3 + lang/java/pom.xml | 1 + .../avro/examples/baseball/Player.java | 6 ++ .../src/test/compiler/output/Player.java | 6 ++ 14 files changed, 172 insertions(+), 29 deletions(-) create mode 100644 lang/java/codegen-test/pom.xml create mode 100644 lang/java/codegen-test/src/main/java/org.apache.avro.codegentest/CustomConversion.java rename lang/java/{compiler/src/test/java/org/apache/avro/compiler/specific => codegen-test/src/test/java/org/apache/avro/codegentest}/TestEndToEndJavaCodeGeneration.java (93%) rename lang/java/{compiler => codegen-test}/src/test/resources/avro/union_with_logical_types.avsc (90%) diff --git a/lang/java/codegen-test/pom.xml b/lang/java/codegen-test/pom.xml new file mode 100644 index 00000000000..9cb3dcb9a41 --- /dev/null +++ b/lang/java/codegen-test/pom.xml @@ -0,0 +1,82 @@ + + + + 4.0.0 + + + avro-parent + org.apache.avro + 1.9.0-SNAPSHOT + ../ + + + avro-codegen-test + + Apache Avro Codegen Test + jar + http://avro.apache.org + Tests generated Avro Specific Java API + + + + org.apache.avro + avro-maven-plugin + ${project.version} + + + generate-test-sources + + schema + protocol + idl-protocol + + + ${project.basedir}/src/test/resources/avro + true + org.apache.avro.codegentest.CustomConversion + + + + + + org.apache.avro + avro-codegen-test + ${project.version} + + + + + + + + + + ${project.groupId} + avro + ${project.version} + + + ${project.groupId} + avro-compiler + ${project.version} + + + + diff --git a/lang/java/codegen-test/src/main/java/org.apache.avro.codegentest/CustomConversion.java b/lang/java/codegen-test/src/main/java/org.apache.avro.codegentest/CustomConversion.java new file mode 100644 index 00000000000..bbd93be261b --- /dev/null +++ b/lang/java/codegen-test/src/main/java/org.apache.avro.codegentest/CustomConversion.java @@ -0,0 +1,30 @@ +package org.apache.avro.codegentest; + +import org.apache.avro.Conversion; +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; + +import java.time.LocalDate; + +public class CustomConversion extends Conversion { + + @Override + public Class getConvertedType() { + return LocalDate.class; + } + + @Override + public String getLogicalTypeName() { + return "date"; + } + + @Override + public LocalDate fromInt(Integer value, Schema schema, LogicalType type) { + return LocalDate.ofEpochDay(value); + } + + @Override + public Integer toInt(LocalDate value, Schema schema, LogicalType type) { + return (int) value.toEpochDay(); + } +} diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestEndToEndJavaCodeGeneration.java b/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java similarity index 93% rename from lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestEndToEndJavaCodeGeneration.java rename to lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java index 051fced457d..b151b940712 100644 --- a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestEndToEndJavaCodeGeneration.java +++ b/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java @@ -1,13 +1,13 @@ -package org.apache.avro.compiler.specific; +package org.apache.avro.codegentest; -import org.apache.avro.compiler.testdata.UnionWithLogicalTypes; -import org.joda.time.LocalDate; +import org.apache.avro.godegentest.testdata.UnionWithLogicalTypes; import org.junit.Assert; import org.junit.Test; import java.io.IOException; import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.time.LocalDate; public class TestEndToEndJavaCodeGeneration { @@ -46,5 +46,4 @@ public void testDecimal() throws IOException { Assert.assertEquals(instanceOfGeneratedClass.getDecimalOrNull(), copy.getDecimalOrNull()); Assert.assertEquals(instanceOfGeneratedClass.getStringOrNull(), copy.getStringOrNull()); } - } diff --git a/lang/java/compiler/src/test/resources/avro/union_with_logical_types.avsc b/lang/java/codegen-test/src/test/resources/avro/union_with_logical_types.avsc similarity index 90% rename from lang/java/compiler/src/test/resources/avro/union_with_logical_types.avsc rename to lang/java/codegen-test/src/test/resources/avro/union_with_logical_types.avsc index 66dee40ea51..f4e46173c17 100644 --- a/lang/java/compiler/src/test/resources/avro/union_with_logical_types.avsc +++ b/lang/java/codegen-test/src/test/resources/avro/union_with_logical_types.avsc @@ -1,4 +1,4 @@ -{"namespace": "org.apache.avro.compiler.testdata", +{"namespace": "org.apache.avro.godegentest.testdata", "type": "record", "name": "UnionWithLogicalTypes", "doc" : "Test unions with logical types in generated Java classes", diff --git a/lang/java/compiler/pom.xml b/lang/java/compiler/pom.xml index 2e1179e4252..ee260c7f14b 100644 --- a/lang/java/compiler/pom.xml +++ b/lang/java/compiler/pom.xml @@ -113,26 +113,6 @@ - - org.apache.avro - avro-maven-plugin - 1.9.0-SNAPSHOT - - - generate-test-sources - - schema - protocol - idl-protocol - - - ${project.basedir}/src/test/resources/avro - true - - - - - diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java index 575462e7366..4c3dad466f7 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java @@ -36,6 +36,7 @@ import org.apache.avro.Conversion; import org.apache.avro.Conversions; +import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; import org.apache.avro.data.Jsr310TimeConversions; import org.apache.avro.data.TimeConversions; @@ -272,6 +273,28 @@ public DateTimeLogicalTypeImplementation getDateTimeLogicalTypeImplementation() return dateTimeLogicalTypeImplementation; } + public void addCustomConversion(String conversionClass) { + try { + final Conversion conversion = (Conversion) Class.forName(conversionClass).newInstance(); + specificData.addLogicalTypeConversion(conversion); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Failed to load conversion class " + conversionClass + ", is it on the classpath?", e); + } catch (IllegalAccessException | InstantiationException e) { + throw new RuntimeException("Failed to instantiate conversion class " + conversionClass, e); + } + } + + public Collection getUsedConversionClasses(Schema schema) { + Map usedConversions = new LinkedHashMap<>(); + for (Field field : schema.getFields()) { + final LogicalType logicalType = field.schema().getLogicalType(); + if (logicalType != null && !usedConversions.containsKey(logicalType.getName())) { + usedConversions.put(logicalType.getName(), specificData.getConversionFor(logicalType).getClass().getCanonicalName()); + } + } + return usedConversions.values(); + } + private static String logChuteName = null; private void initializeVelocity() { diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm index f68a6a51b89..deaaa6bbdd7 100644 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm @@ -40,10 +40,9 @@ public class ${this.mangle($schema.getName())}#if ($schema.isError()) extends or private static SpecificData MODEL$ = new SpecificData(); static { - MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.DateConversion()); - MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimeConversion()); - MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimestampConversion()); - MODEL$.addLogicalTypeConversion(new org.apache.avro.Conversions.DecimalConversion()); +#foreach ($conversion in $this.getUsedConversionClasses($schema)) + MODEL$.addLogicalTypeConversion(new ${conversion}()); +#end } #if (!$schema.isError()) diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java index 3ddb0629b76..5e912ebb9bf 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java @@ -122,6 +122,14 @@ public abstract class AbstractAvroMojo extends AbstractMojo { */ protected boolean createSetters; + /** + * A set of fully qualified class names of custom {@link org.apache.avro.Conversion} implementations to add to the compiler. + * The classes must be on the classpath at compile time and whenever the Java objects are serialized. + * + * @parameter property="customConversions" + */ + protected String[] customConversions = new String[0]; + /** * Determines whether or not to use Java classes for decimal types * diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java index 3ee7fcb0028..de9503f3172 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java @@ -94,6 +94,9 @@ protected void doCompile(String filename, File sourceDirectory, File outputDirec compiler.setFieldVisibility(getFieldVisibility()); compiler.setCreateSetters(createSetters); compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); + for (String customConversion : customConversions) { + compiler.addCustomConversion(customConversion); + } compiler.compileToDestination(null, outputDirectory); } catch (ParseException e) { throw new IOException(e); diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java index 25ee56ba16f..08a08fb151a 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java @@ -62,6 +62,9 @@ protected void doCompile(String filename, File sourceDirectory, File outputDirec compiler.setFieldVisibility(getFieldVisibility()); compiler.setCreateSetters(createSetters); compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); + for (String customConversion : customConversions) { + compiler.addCustomConversion(customConversion); + } compiler.compileToDestination(src, outputDirectory); } diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java index ab9d0eb29ef..fc7c12fc28c 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java @@ -79,6 +79,9 @@ protected void doCompile(String filename, File sourceDirectory, File outputDirec compiler.setFieldVisibility(getFieldVisibility()); compiler.setCreateSetters(createSetters); compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); + for (String customConversion : customConversions) { + compiler.addCustomConversion(customConversion); + } compiler.setOutputCharacterEncoding(project.getProperties().getProperty("project.build.sourceEncoding")); compiler.compileToDestination(src, outputDirectory); } diff --git a/lang/java/pom.xml b/lang/java/pom.xml index 471c34044b2..a3fb85ddf6e 100644 --- a/lang/java/pom.xml +++ b/lang/java/pom.xml @@ -97,6 +97,7 @@ thrift archetypes grpc + codegen-test diff --git a/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java b/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java index 569f4271126..a7e876a1287 100644 --- a/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java +++ b/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java @@ -18,6 +18,12 @@ public class Player extends org.apache.avro.specific.SpecificRecordBase implemen public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } private static SpecificData MODEL$ = new SpecificData(); + static { + MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.DateConversion()); + MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimeConversion()); + MODEL$.addLogicalTypeConversion(new org.apache.avro.Conversions.DecimalConversion()); + MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimestampConversion()); + } private static final BinaryMessageEncoder ENCODER = new BinaryMessageEncoder(MODEL$, SCHEMA$); diff --git a/lang/java/tools/src/test/compiler/output/Player.java b/lang/java/tools/src/test/compiler/output/Player.java index 5bbb3b01850..4df71b01968 100644 --- a/lang/java/tools/src/test/compiler/output/Player.java +++ b/lang/java/tools/src/test/compiler/output/Player.java @@ -18,6 +18,12 @@ public class Player extends org.apache.avro.specific.SpecificRecordBase implemen public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } private static SpecificData MODEL$ = new SpecificData(); + static { + MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.DateConversion()); + MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimeConversion()); + MODEL$.addLogicalTypeConversion(new org.apache.avro.Conversions.DecimalConversion()); + MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimestampConversion()); + } private static final BinaryMessageEncoder ENCODER = new BinaryMessageEncoder(MODEL$, SCHEMA$); From e587ab50d45a08da77e0697caff115181c1f6595 Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Fri, 10 Aug 2018 11:40:42 +0200 Subject: [PATCH 06/18] Fixed bug --- .../java/org/apache/avro/generic/GenericData.java | 4 ++++ .../TestEndToEndJavaCodeGeneration.java | 4 ++++ .../avro/compiler/specific/SpecificCompiler.java | 15 +++++++++------ .../specific/templates/java/classic/record.vm | 7 +++++-- .../avro/examples/baseball/Player.java | 6 ------ .../tools/src/test/compiler/output/Player.java | 6 ------ 6 files changed, 22 insertions(+), 20 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java index 8b8a2961f08..c9e551c06cc 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java +++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java @@ -105,6 +105,10 @@ public GenericData(ClassLoader classLoader) { private Map, Map>> conversionsByClass = new IdentityHashMap<>(); + public Map> getConversions() { + return conversions; + } + /** * Registers the given conversion to be used when reading and writing with * this data model. diff --git a/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java b/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java index b151b940712..a966115c48b 100644 --- a/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java +++ b/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java @@ -1,5 +1,6 @@ package org.apache.avro.codegentest; +import org.apache.avro.compiler.specific.SpecificCompiler; import org.apache.avro.godegentest.testdata.UnionWithLogicalTypes; import org.junit.Assert; import org.junit.Test; @@ -37,6 +38,9 @@ public void testDate() throws IOException { @Test public void testDecimal() throws IOException { + final SpecificCompiler specificCompiler = new SpecificCompiler(UnionWithLogicalTypes.SCHEMA$); + specificCompiler.addCustomConversion(CustomConversion.class.getCanonicalName()); + specificCompiler.getUsedConversionClasses(UnionWithLogicalTypes.SCHEMA$); UnionWithLogicalTypes instanceOfGeneratedClass = UnionWithLogicalTypes.newBuilder() .setStringOrNull("hello") .setDecimalOrNull(BigDecimal.valueOf(123, 2)) diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java index 4c3dad466f7..0c53ba2acb6 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java @@ -36,7 +36,6 @@ import org.apache.avro.Conversion; import org.apache.avro.Conversions; -import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; import org.apache.avro.data.Jsr310TimeConversions; import org.apache.avro.data.TimeConversions; @@ -285,14 +284,18 @@ public void addCustomConversion(String conversionClass) { } public Collection getUsedConversionClasses(Schema schema) { - Map usedConversions = new LinkedHashMap<>(); + LinkedHashMap> classnameToConversion = new LinkedHashMap<>(); + for (Conversion conversion : specificData.getConversions().values()) { + classnameToConversion.put(conversion.getConvertedType().getCanonicalName(), conversion); + } + Collection result = new ArrayList<>(); for (Field field : schema.getFields()) { - final LogicalType logicalType = field.schema().getLogicalType(); - if (logicalType != null && !usedConversions.containsKey(logicalType.getName())) { - usedConversions.put(logicalType.getName(), specificData.getConversionFor(logicalType).getClass().getCanonicalName()); + final String className = javaType(field.schema()); + if (classnameToConversion.containsKey(className)) { + result.add(classnameToConversion.get(className).getClass().getCanonicalName()); } } - return usedConversions.values(); + return result; } private static String logChuteName = null; diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm index deaaa6bbdd7..17e165946b6 100644 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm @@ -39,11 +39,14 @@ public class ${this.mangle($schema.getName())}#if ($schema.isError()) extends or public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } private static SpecificData MODEL$ = new SpecificData(); - static { -#foreach ($conversion in $this.getUsedConversionClasses($schema)) +#set ($usedConversions = $this.getUsedConversionClasses($schema)) +#if (!$usedConversions.isEmpty()) +static { +#foreach ($conversion in $usedConversions) MODEL$.addLogicalTypeConversion(new ${conversion}()); #end } +#end #if (!$schema.isError()) private static final BinaryMessageEncoder<${this.mangle($schema.getName())}> ENCODER = diff --git a/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java b/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java index a7e876a1287..569f4271126 100644 --- a/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java +++ b/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java @@ -18,12 +18,6 @@ public class Player extends org.apache.avro.specific.SpecificRecordBase implemen public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } private static SpecificData MODEL$ = new SpecificData(); - static { - MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.DateConversion()); - MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimeConversion()); - MODEL$.addLogicalTypeConversion(new org.apache.avro.Conversions.DecimalConversion()); - MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimestampConversion()); - } private static final BinaryMessageEncoder ENCODER = new BinaryMessageEncoder(MODEL$, SCHEMA$); diff --git a/lang/java/tools/src/test/compiler/output/Player.java b/lang/java/tools/src/test/compiler/output/Player.java index 4df71b01968..5bbb3b01850 100644 --- a/lang/java/tools/src/test/compiler/output/Player.java +++ b/lang/java/tools/src/test/compiler/output/Player.java @@ -18,12 +18,6 @@ public class Player extends org.apache.avro.specific.SpecificRecordBase implemen public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } private static SpecificData MODEL$ = new SpecificData(); - static { - MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.DateConversion()); - MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimeConversion()); - MODEL$.addLogicalTypeConversion(new org.apache.avro.Conversions.DecimalConversion()); - MODEL$.addLogicalTypeConversion(new org.apache.avro.data.TimeConversions.TimestampConversion()); - } private static final BinaryMessageEncoder ENCODER = new BinaryMessageEncoder(MODEL$, SCHEMA$); From 50875e322f3f5bc83c0311302b96a189c6092e67 Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Mon, 13 Aug 2018 13:54:10 +0200 Subject: [PATCH 07/18] Fixed Maven plugin classpath --- lang/java/codegen-test-deps/pom.xml | 45 +++++++++++++++++++ .../CustomConversion.java | 0 lang/java/codegen-test/pom.xml | 11 ++++- .../TestEndToEndJavaCodeGeneration.java | 4 +- .../avro/union_with_logical_types.avsc | 2 +- .../compiler/specific/SpecificCompiler.java | 8 ++-- .../apache/avro/mojo/AbstractAvroMojo.java | 44 ++++++++++++++++++ .../org/apache/avro/mojo/IDLProtocolMojo.java | 29 +++--------- .../org/apache/avro/mojo/ProtocolMojo.java | 15 ++++++- .../java/org/apache/avro/mojo/SchemaMojo.java | 14 +++++- lang/java/pom.xml | 1 + 11 files changed, 136 insertions(+), 37 deletions(-) create mode 100644 lang/java/codegen-test-deps/pom.xml rename lang/java/{codegen-test => codegen-test-deps}/src/main/java/org.apache.avro.codegentest/CustomConversion.java (100%) diff --git a/lang/java/codegen-test-deps/pom.xml b/lang/java/codegen-test-deps/pom.xml new file mode 100644 index 00000000000..ebbd620094d --- /dev/null +++ b/lang/java/codegen-test-deps/pom.xml @@ -0,0 +1,45 @@ + + + + 4.0.0 + + + avro-parent + org.apache.avro + 1.9.0-SNAPSHOT + ../ + + + avro-codegen-test-deps + + Apache Avro Codegen Test dependencies + jar + http://avro.apache.org + Contains dependencies for the maven plugin used in avro-codegen-test + + + + ${project.groupId} + avro + ${project.version} + + + + diff --git a/lang/java/codegen-test/src/main/java/org.apache.avro.codegentest/CustomConversion.java b/lang/java/codegen-test-deps/src/main/java/org.apache.avro.codegentest/CustomConversion.java similarity index 100% rename from lang/java/codegen-test/src/main/java/org.apache.avro.codegentest/CustomConversion.java rename to lang/java/codegen-test-deps/src/main/java/org.apache.avro.codegentest/CustomConversion.java diff --git a/lang/java/codegen-test/pom.xml b/lang/java/codegen-test/pom.xml index 9cb3dcb9a41..3cb00d119a6 100644 --- a/lang/java/codegen-test/pom.xml +++ b/lang/java/codegen-test/pom.xml @@ -48,7 +48,9 @@ idl-protocol - ${project.basedir}/src/test/resources/avro + String + ${project.basedir}/src/test/resources/avro + ${project.build.directory}/generated-test-sources/java true org.apache.avro.codegentest.CustomConversion @@ -57,7 +59,7 @@ org.apache.avro - avro-codegen-test + avro-codegen-test-deps ${project.version} @@ -77,6 +79,11 @@ avro-compiler ${project.version} + + ${project.groupId} + avro-codegen-test-deps + ${project.version} + diff --git a/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java b/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java index a966115c48b..81c21d9c23f 100644 --- a/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java +++ b/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java @@ -1,7 +1,7 @@ package org.apache.avro.codegentest; import org.apache.avro.compiler.specific.SpecificCompiler; -import org.apache.avro.godegentest.testdata.UnionWithLogicalTypes; +import org.apache.avro.codegentest.testdata.UnionWithLogicalTypes; import org.junit.Assert; import org.junit.Test; @@ -39,7 +39,7 @@ public void testDate() throws IOException { @Test public void testDecimal() throws IOException { final SpecificCompiler specificCompiler = new SpecificCompiler(UnionWithLogicalTypes.SCHEMA$); - specificCompiler.addCustomConversion(CustomConversion.class.getCanonicalName()); + specificCompiler.addCustomConversion(CustomConversion.class); specificCompiler.getUsedConversionClasses(UnionWithLogicalTypes.SCHEMA$); UnionWithLogicalTypes instanceOfGeneratedClass = UnionWithLogicalTypes.newBuilder() .setStringOrNull("hello") diff --git a/lang/java/codegen-test/src/test/resources/avro/union_with_logical_types.avsc b/lang/java/codegen-test/src/test/resources/avro/union_with_logical_types.avsc index f4e46173c17..99e698e8ba1 100644 --- a/lang/java/codegen-test/src/test/resources/avro/union_with_logical_types.avsc +++ b/lang/java/codegen-test/src/test/resources/avro/union_with_logical_types.avsc @@ -1,4 +1,4 @@ -{"namespace": "org.apache.avro.godegentest.testdata", +{"namespace": "org.apache.avro.codegentest.testdata", "type": "record", "name": "UnionWithLogicalTypes", "doc" : "Test unions with logical types in generated Java classes", diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java index 0c53ba2acb6..6c0f7704ff9 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java @@ -272,13 +272,11 @@ public DateTimeLogicalTypeImplementation getDateTimeLogicalTypeImplementation() return dateTimeLogicalTypeImplementation; } - public void addCustomConversion(String conversionClass) { + public void addCustomConversion(Class conversionClass) { try { - final Conversion conversion = (Conversion) Class.forName(conversionClass).newInstance(); + final Conversion conversion = (Conversion)conversionClass.newInstance(); specificData.addLogicalTypeConversion(conversion); - } catch (ClassNotFoundException e) { - throw new RuntimeException("Failed to load conversion class " + conversionClass + ", is it on the classpath?", e); - } catch (IllegalAccessException | InstantiationException e) { + } catch (IllegalAccessException | InstantiationException e) { throw new RuntimeException("Failed to instantiate conversion class " + conversionClass, e); } } diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java index 5e912ebb9bf..7064dd91f30 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java @@ -20,10 +20,16 @@ import java.io.File; import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import org.apache.avro.compiler.specific.SpecificCompiler; import org.apache.avro.compiler.specific.SpecificCompiler.DateTimeLogicalTypeImplementation; +import org.apache.maven.artifact.DependencyResolutionRequiredException; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.project.MavenProject; @@ -271,6 +277,44 @@ protected DateTimeLogicalTypeImplementation getDateTimeLogicalTypeImplementation protected abstract void doCompile(String filename, File sourceDirectory, File outputDirectory) throws IOException; + protected URLClassLoader createClassLoader2() throws DependencyResolutionRequiredException, MalformedURLException { + List urls = appendElements(project.getRuntimeClasspathElements()); + urls.addAll(appendElements(project.getTestClasspathElements())); + return new URLClassLoader(urls.toArray(new URL[urls.size()]), + Thread.currentThread().getContextClassLoader()); + } + + private List appendElements(List runtimeClasspathElements) throws MalformedURLException { + List runtimeUrls = new ArrayList<>(); + if (runtimeClasspathElements != null) { + for (Object runtimeClasspathElement : runtimeClasspathElements) { + String element = (String) runtimeClasspathElement; + runtimeUrls.add(new File(element).toURI().toURL()); + } + } + return runtimeUrls; + } + + protected URLClassLoader createClassLoaderIncludingCurrentProjectClasses(File sourceDirectory) throws MalformedURLException, DependencyResolutionRequiredException { + List runtimeClasspathElements = project.getRuntimeClasspathElements(); + List runtimeUrls = new ArrayList<>(); + + // Add the source directory of avro files to the classpath so that + // imports can refer to other idl files as classpath resources + runtimeUrls.add(sourceDirectory.toURI().toURL()); + + // If runtimeClasspathElements is not empty values add its values to Idl path. + if (runtimeClasspathElements != null && !runtimeClasspathElements.isEmpty()) { + for (Object runtimeClasspathElement : runtimeClasspathElements) { + String element = (String) runtimeClasspathElement; + runtimeUrls.add(new File(element).toURI().toURL()); + } + } + + return new URLClassLoader + (runtimeUrls.toArray(new URL[0]), Thread.currentThread().getContextClassLoader()); + } + protected abstract String[] getIncludes(); protected abstract String[] getTestIncludes(); diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java index de9503f3172..98f0fe41081 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java @@ -20,10 +20,7 @@ import java.io.File; import java.io.IOException; -import java.net.URL; import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.List; import org.apache.avro.Protocol; import org.apache.avro.compiler.idl.Idl; @@ -63,27 +60,11 @@ public class IDLProtocolMojo extends AbstractAvroMojo { @Override protected void doCompile(String filename, File sourceDirectory, File outputDirectory) throws IOException { try { - @SuppressWarnings("rawtypes") - List runtimeClasspathElements = project.getRuntimeClasspathElements(); Idl parser; + @SuppressWarnings("rawtypes") - List runtimeUrls = new ArrayList<>(); - - // Add the source directory of avro files to the classpath so that - // imports can refer to other idl files as classpath resources - runtimeUrls.add(sourceDirectory.toURI().toURL()); - - // If runtimeClasspathElements is not empty values add its values to Idl path. - if (runtimeClasspathElements != null && !runtimeClasspathElements.isEmpty()) { - for (Object runtimeClasspathElement : runtimeClasspathElements) { - String element = (String) runtimeClasspathElement; - runtimeUrls.add(new File(element).toURI().toURL()); - } - } - - URLClassLoader projPathLoader = new URLClassLoader - (runtimeUrls.toArray(new URL[0]), Thread.currentThread().getContextClassLoader()); - parser = new Idl(new File(sourceDirectory, filename), projPathLoader); + URLClassLoader projPathLoader = createClassLoaderIncludingCurrentProjectClasses(sourceDirectory); + parser = new Idl(new File(sourceDirectory, filename), projPathLoader); Protocol p = parser.CompilationUnit(); String json = p.toString(true); @@ -95,13 +76,15 @@ protected void doCompile(String filename, File sourceDirectory, File outputDirec compiler.setCreateSetters(createSetters); compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); for (String customConversion : customConversions) { - compiler.addCustomConversion(customConversion); + compiler.addCustomConversion(projPathLoader.loadClass(customConversion)); } compiler.compileToDestination(null, outputDirectory); } catch (ParseException e) { throw new IOException(e); } catch (DependencyResolutionRequiredException drre) { throw new IOException(drre); + } catch (ClassNotFoundException e) { + throw new IOException(e); } } diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java index 08a08fb151a..01ef7d6ac57 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java @@ -22,15 +22,18 @@ import java.io.File; import java.io.IOException; +import java.net.URLClassLoader; import org.apache.avro.Protocol; import org.apache.avro.compiler.specific.SpecificCompiler; +import org.apache.maven.artifact.DependencyResolutionRequiredException; /** * Generate Java classes and interfaces from Avro protocol files (.avpr) * * @goal protocol * @phase generate-sources + * @requiresDependencyResolution runtime * @threadSafe */ public class ProtocolMojo extends AbstractAvroMojo { @@ -62,8 +65,16 @@ protected void doCompile(String filename, File sourceDirectory, File outputDirec compiler.setFieldVisibility(getFieldVisibility()); compiler.setCreateSetters(createSetters); compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); - for (String customConversion : customConversions) { - compiler.addCustomConversion(customConversion); + final URLClassLoader classLoader; + try { + classLoader = createClassLoader2(); + for (String customConversion : customConversions) { + compiler.addCustomConversion(classLoader.loadClass(customConversion)); + } + } catch (DependencyResolutionRequiredException e) { + throw new IOException(e); + } catch (ClassNotFoundException e) { + throw new IOException(e); } compiler.compileToDestination(src, outputDirectory); } diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java index fc7c12fc28c..2190e305578 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java @@ -22,15 +22,18 @@ import java.io.File; import java.io.IOException; +import java.net.URLClassLoader; import org.apache.avro.Schema; import org.apache.avro.compiler.specific.SpecificCompiler; +import org.apache.maven.artifact.DependencyResolutionRequiredException; /** * Generate Java classes from Avro schema files (.avsc) * * @goal schema * @phase generate-sources + * @requiresDependencyResolution runtime+test * @threadSafe */ public class SchemaMojo extends AbstractAvroMojo { @@ -79,8 +82,15 @@ protected void doCompile(String filename, File sourceDirectory, File outputDirec compiler.setFieldVisibility(getFieldVisibility()); compiler.setCreateSetters(createSetters); compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); - for (String customConversion : customConversions) { - compiler.addCustomConversion(customConversion); + try { + final URLClassLoader classLoader = createClassLoader2(); + for (String customConversion : customConversions) { + compiler.addCustomConversion(classLoader.loadClass(customConversion)); + } + } catch (ClassNotFoundException e) { + throw new IOException(e); + } catch (DependencyResolutionRequiredException e) { + throw new IOException(e); } compiler.setOutputCharacterEncoding(project.getProperties().getProperty("project.build.sourceEncoding")); compiler.compileToDestination(src, outputDirectory); diff --git a/lang/java/pom.xml b/lang/java/pom.xml index a3fb85ddf6e..9fc3a7eb88f 100644 --- a/lang/java/pom.xml +++ b/lang/java/pom.xml @@ -98,6 +98,7 @@ archetypes grpc codegen-test + codegen-test-deps From 3fb2a10b4f45eafd533f48d7272686bdecc7e1d5 Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Mon, 20 Aug 2018 10:54:12 +0200 Subject: [PATCH 08/18] Get the correct SpecificData whenever possible, to get the right conversions --- .../apache/avro/specific/SpecificData.java | 50 +++++++++++++++++++ .../avro/specific/SpecificDatumReader.java | 4 +- .../avro/specific/SpecificDatumWriter.java | 4 +- .../specific/SpecificRecordBuilderBase.java | 6 +-- .../TestEndToEndJavaCodeGeneration.java | 45 +++++++++++++---- 5 files changed, 91 insertions(+), 18 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java index 44de5c43408..77e867038ae 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java +++ b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java @@ -17,6 +17,7 @@ */ package org.apache.avro.specific; +import java.lang.reflect.Field; import java.util.Arrays; import java.util.HashSet; import java.util.Map; @@ -122,6 +123,55 @@ public DatumWriter createDatumWriter(Schema schema) { /** Return the singleton instance. */ public static SpecificData get() { return INSTANCE; } + /** + * For RECORD type schemas, this method returns the SpecificData instance of the class associated with the schema, + * in order to get the right conversions for any logical types used. + * + * @param reader the reader schema + * @return the SpecificData associated with the schema's class, or the default instance. + */ + public static SpecificData getForSchema(Schema reader) { + if (reader.getType() == Type.RECORD) { + final String className = getClassName(reader); + if (className != null) { + final Class clazz; + try { + clazz = Class.forName(className); + return getForClass(clazz); + } catch (ClassNotFoundException e) { + return SpecificData.get(); + } + } + } + return SpecificData.get(); + } + + /** + * If the given class is assignable to {@link SpecificRecordBase}, this method returns the SpecificData instance + * from the field {@code MODEL$}, in order to get the correct {@link org.apache.avro.Conversion} instances for the class. + * Falls back to the default instance {@link SpecificData#get()} for other classes or if the field is not found. + * + * @param c A class + * @param . + * @return The SpecificData from the SpecificRecordBase instance, or the default SpecificData instance. + */ + public static SpecificData getForClass(Class c) { + if (SpecificRecordBase.class.isAssignableFrom(c)) { + final Field specificDataField; + try { + specificDataField = c.getDeclaredField("MODEL$"); + specificDataField.setAccessible(true); + return (SpecificData) specificDataField.get(null); + } catch (NoSuchFieldException e) { + // Return default instance + return SpecificData.get(); + } catch (IllegalAccessException e) { + throw new AvroRuntimeException(e); + } + } + return SpecificData.get(); + } + @Override protected boolean isEnum(Object datum) { return datum instanceof Enum || super.isEnum(datum); diff --git a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java index 29c989b7831..f2c227b95f7 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java @@ -39,12 +39,12 @@ public SpecificDatumReader(Class c) { /** Construct where the writer's and reader's schemas are the same. */ public SpecificDatumReader(Schema schema) { - this(schema, schema, SpecificData.get()); + this(schema, schema, SpecificData.getForSchema(schema)); } /** Construct given writer's and reader's schema. */ public SpecificDatumReader(Schema writer, Schema reader) { - this(writer, reader, SpecificData.get()); + this(writer, reader, SpecificData.getForSchema(reader)); } /** Construct given writer's schema, reader's schema, and a {@link diff --git a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumWriter.java b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumWriter.java index 1204f4955e0..69f190ccd80 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumWriter.java +++ b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumWriter.java @@ -32,11 +32,11 @@ public SpecificDatumWriter() { } public SpecificDatumWriter(Class c) { - super(SpecificData.get().getSchema(c), SpecificData.get()); + super(SpecificData.get().getSchema(c), SpecificData.getForClass(c)); } public SpecificDatumWriter(Schema schema) { - super(schema, SpecificData.get()); + super(schema, SpecificData.getForSchema(schema)); } public SpecificDatumWriter(Schema root, SpecificData specificData) { diff --git a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificRecordBuilderBase.java b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificRecordBuilderBase.java index ecf3c34dcc0..a8d220b5085 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificRecordBuilderBase.java +++ b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificRecordBuilderBase.java @@ -32,7 +32,7 @@ abstract public class SpecificRecordBuilderBase * @param schema the schema associated with the record class. */ protected SpecificRecordBuilderBase(Schema schema) { - super(schema, SpecificData.get()); + super(schema, SpecificData.getForSchema(schema)); } /** @@ -40,7 +40,7 @@ protected SpecificRecordBuilderBase(Schema schema) { * @param other SpecificRecordBuilderBase instance to copy. */ protected SpecificRecordBuilderBase(SpecificRecordBuilderBase other) { - super(other, SpecificData.get()); + super(other, other.data()); } /** @@ -48,6 +48,6 @@ protected SpecificRecordBuilderBase(SpecificRecordBuilderBase other) { * @param other the record instance to copy. */ protected SpecificRecordBuilderBase(T other) { - super(other.getSchema(), SpecificData.get()); + super(other.getSchema(), SpecificData.getForSchema(other.getSchema())); } } diff --git a/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java b/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java index 81c21d9c23f..7b38964a335 100644 --- a/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java +++ b/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java @@ -1,13 +1,17 @@ package org.apache.avro.codegentest; -import org.apache.avro.compiler.specific.SpecificCompiler; import org.apache.avro.codegentest.testdata.UnionWithLogicalTypes; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificDatumWriter; import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigDecimal; -import java.nio.ByteBuffer; import java.time.LocalDate; public class TestEndToEndJavaCodeGeneration { @@ -18,8 +22,8 @@ public void testWithNullValues() throws IOException { .setDateOrNull(null) .setStringOrNull("hello") .build(); - final ByteBuffer bb = instanceOfGeneratedClass.toByteBuffer(); - final UnionWithLogicalTypes copy = UnionWithLogicalTypes.fromByteBuffer(bb); + final byte[] serialized = serialize(instanceOfGeneratedClass); + final UnionWithLogicalTypes copy = deserialize(serialized); Assert.assertEquals(instanceOfGeneratedClass.getDateOrNull(), copy.getDateOrNull()); Assert.assertEquals(instanceOfGeneratedClass.getStringOrNull(), copy.getStringOrNull()); } @@ -30,24 +34,43 @@ public void testDate() throws IOException { .setDateOrNull(LocalDate.now()) .setStringOrNull("hello") .build(); - final ByteBuffer bb = instanceOfGeneratedClass.toByteBuffer(); - final UnionWithLogicalTypes copy = UnionWithLogicalTypes.fromByteBuffer(bb); + final byte[] serialized = serialize(instanceOfGeneratedClass); + final UnionWithLogicalTypes copy = deserialize(serialized); Assert.assertEquals(instanceOfGeneratedClass.getDateOrNull(), copy.getDateOrNull()); Assert.assertEquals(instanceOfGeneratedClass.getStringOrNull(), copy.getStringOrNull()); } @Test public void testDecimal() throws IOException { - final SpecificCompiler specificCompiler = new SpecificCompiler(UnionWithLogicalTypes.SCHEMA$); - specificCompiler.addCustomConversion(CustomConversion.class); - specificCompiler.getUsedConversionClasses(UnionWithLogicalTypes.SCHEMA$); UnionWithLogicalTypes instanceOfGeneratedClass = UnionWithLogicalTypes.newBuilder() .setStringOrNull("hello") .setDecimalOrNull(BigDecimal.valueOf(123, 2)) .build(); - final ByteBuffer bb = instanceOfGeneratedClass.toByteBuffer(); - final UnionWithLogicalTypes copy = UnionWithLogicalTypes.fromByteBuffer(bb); + final byte[] serialized = serialize(instanceOfGeneratedClass); + final UnionWithLogicalTypes copy = deserialize(serialized); Assert.assertEquals(instanceOfGeneratedClass.getDecimalOrNull(), copy.getDecimalOrNull()); Assert.assertEquals(instanceOfGeneratedClass.getStringOrNull(), copy.getStringOrNull()); } + + private byte[] serialize(UnionWithLogicalTypes object) { + SpecificDatumWriter datumWriter = new SpecificDatumWriter<>(UnionWithLogicalTypes.getClassSchema()); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try { + datumWriter.write(object, EncoderFactory.get().directBinaryEncoder(outputStream, null)); + return outputStream.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private UnionWithLogicalTypes deserialize(byte[] bytes) { + SpecificDatumReader datumReader = new SpecificDatumReader<>(UnionWithLogicalTypes.getClassSchema(), UnionWithLogicalTypes.getClassSchema()); + try { + final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + return datumReader.read(null, DecoderFactory.get().directBinaryDecoder(byteArrayInputStream, null)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } From e5ed84f7501eb4e45a6c3c6c533cbd6fa7c140cc Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Mon, 20 Aug 2018 11:12:08 +0200 Subject: [PATCH 09/18] No need to expose the map of conversions so expose only the values. --- .../src/main/java/org/apache/avro/generic/GenericData.java | 4 ++-- .../org/apache/avro/compiler/specific/SpecificCompiler.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java index c9e551c06cc..797b76d0da2 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java +++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java @@ -105,8 +105,8 @@ public GenericData(ClassLoader classLoader) { private Map, Map>> conversionsByClass = new IdentityHashMap<>(); - public Map> getConversions() { - return conversions; + public Collection> getConversions() { + return conversions.values(); } /** diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java index 6c0f7704ff9..ed6abbd6e9f 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java @@ -283,7 +283,7 @@ public void addCustomConversion(Class conversionClass) { public Collection getUsedConversionClasses(Schema schema) { LinkedHashMap> classnameToConversion = new LinkedHashMap<>(); - for (Conversion conversion : specificData.getConversions().values()) { + for (Conversion conversion : specificData.getConversions()) { classnameToConversion.put(conversion.getConvertedType().getCanonicalName(), conversion); } Collection result = new ArrayList<>(); From 64f407b5c22dbf2dbd1a146bf6ffc85743e6f787 Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Tue, 21 Aug 2018 10:20:59 +0200 Subject: [PATCH 10/18] Better tests --- .../CustomConversion.java | 30 ------ .../TestEndToEndJavaCodeGeneration.java | 76 -------------- .../avro/union_with_logical_types.avsc | 13 --- .../compiler/specific/SpecificCompiler.java | 17 ++-- .../specific/templates/java/classic/record.vm | 11 --- .../specific/TestSpecificCompiler.java | 26 ++--- .../codegen-test/pom.xml | 8 +- .../codegentest/TestCustomConversion.java | 80 +++++++++++++++ .../codegentest/TestNullableLogicalTypes.java | 77 +++++++++++++++ .../resources/avro/custom_conversion.avsc | 12 +++ .../avro/nullable_logical_types.avsc | 11 +++ lang/java/integration-test/pom.xml | 99 +++++++++++++++++++ .../test-custom-conversions}/pom.xml | 4 +- .../CustomDecimal.java | 60 +++++++++++ .../CustomDecimalConversion.java | 52 ++++++++++ lang/java/mapred/pom.xml | 12 +++ lang/java/pom.xml | 3 +- 17 files changed, 431 insertions(+), 160 deletions(-) delete mode 100644 lang/java/codegen-test-deps/src/main/java/org.apache.avro.codegentest/CustomConversion.java delete mode 100644 lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java delete mode 100644 lang/java/codegen-test/src/test/resources/avro/union_with_logical_types.avsc rename lang/java/{ => integration-test}/codegen-test/pom.xml (93%) create mode 100644 lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestCustomConversion.java create mode 100644 lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestNullableLogicalTypes.java create mode 100644 lang/java/integration-test/codegen-test/src/test/resources/avro/custom_conversion.avsc create mode 100644 lang/java/integration-test/codegen-test/src/test/resources/avro/nullable_logical_types.avsc create mode 100644 lang/java/integration-test/pom.xml rename lang/java/{codegen-test-deps => integration-test/test-custom-conversions}/pom.xml (93%) create mode 100644 lang/java/integration-test/test-custom-conversions/src/main/java/org.apache.avro.codegentest/CustomDecimal.java create mode 100644 lang/java/integration-test/test-custom-conversions/src/main/java/org.apache.avro.codegentest/CustomDecimalConversion.java diff --git a/lang/java/codegen-test-deps/src/main/java/org.apache.avro.codegentest/CustomConversion.java b/lang/java/codegen-test-deps/src/main/java/org.apache.avro.codegentest/CustomConversion.java deleted file mode 100644 index bbd93be261b..00000000000 --- a/lang/java/codegen-test-deps/src/main/java/org.apache.avro.codegentest/CustomConversion.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.apache.avro.codegentest; - -import org.apache.avro.Conversion; -import org.apache.avro.LogicalType; -import org.apache.avro.Schema; - -import java.time.LocalDate; - -public class CustomConversion extends Conversion { - - @Override - public Class getConvertedType() { - return LocalDate.class; - } - - @Override - public String getLogicalTypeName() { - return "date"; - } - - @Override - public LocalDate fromInt(Integer value, Schema schema, LogicalType type) { - return LocalDate.ofEpochDay(value); - } - - @Override - public Integer toInt(LocalDate value, Schema schema, LogicalType type) { - return (int) value.toEpochDay(); - } -} diff --git a/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java b/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java deleted file mode 100644 index 7b38964a335..00000000000 --- a/lang/java/codegen-test/src/test/java/org/apache/avro/codegentest/TestEndToEndJavaCodeGeneration.java +++ /dev/null @@ -1,76 +0,0 @@ -package org.apache.avro.codegentest; - -import org.apache.avro.codegentest.testdata.UnionWithLogicalTypes; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.specific.SpecificDatumReader; -import org.apache.avro.specific.SpecificDatumWriter; -import org.junit.Assert; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.math.BigDecimal; -import java.time.LocalDate; - -public class TestEndToEndJavaCodeGeneration { - - @Test - public void testWithNullValues() throws IOException { - UnionWithLogicalTypes instanceOfGeneratedClass = UnionWithLogicalTypes.newBuilder() - .setDateOrNull(null) - .setStringOrNull("hello") - .build(); - final byte[] serialized = serialize(instanceOfGeneratedClass); - final UnionWithLogicalTypes copy = deserialize(serialized); - Assert.assertEquals(instanceOfGeneratedClass.getDateOrNull(), copy.getDateOrNull()); - Assert.assertEquals(instanceOfGeneratedClass.getStringOrNull(), copy.getStringOrNull()); - } - - @Test - public void testDate() throws IOException { - UnionWithLogicalTypes instanceOfGeneratedClass = UnionWithLogicalTypes.newBuilder() - .setDateOrNull(LocalDate.now()) - .setStringOrNull("hello") - .build(); - final byte[] serialized = serialize(instanceOfGeneratedClass); - final UnionWithLogicalTypes copy = deserialize(serialized); - Assert.assertEquals(instanceOfGeneratedClass.getDateOrNull(), copy.getDateOrNull()); - Assert.assertEquals(instanceOfGeneratedClass.getStringOrNull(), copy.getStringOrNull()); - } - - @Test - public void testDecimal() throws IOException { - UnionWithLogicalTypes instanceOfGeneratedClass = UnionWithLogicalTypes.newBuilder() - .setStringOrNull("hello") - .setDecimalOrNull(BigDecimal.valueOf(123, 2)) - .build(); - final byte[] serialized = serialize(instanceOfGeneratedClass); - final UnionWithLogicalTypes copy = deserialize(serialized); - Assert.assertEquals(instanceOfGeneratedClass.getDecimalOrNull(), copy.getDecimalOrNull()); - Assert.assertEquals(instanceOfGeneratedClass.getStringOrNull(), copy.getStringOrNull()); - } - - private byte[] serialize(UnionWithLogicalTypes object) { - SpecificDatumWriter datumWriter = new SpecificDatumWriter<>(UnionWithLogicalTypes.getClassSchema()); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try { - datumWriter.write(object, EncoderFactory.get().directBinaryEncoder(outputStream, null)); - return outputStream.toByteArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private UnionWithLogicalTypes deserialize(byte[] bytes) { - SpecificDatumReader datumReader = new SpecificDatumReader<>(UnionWithLogicalTypes.getClassSchema(), UnionWithLogicalTypes.getClassSchema()); - try { - final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); - return datumReader.read(null, DecoderFactory.get().directBinaryDecoder(byteArrayInputStream, null)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/lang/java/codegen-test/src/test/resources/avro/union_with_logical_types.avsc b/lang/java/codegen-test/src/test/resources/avro/union_with_logical_types.avsc deleted file mode 100644 index 99e698e8ba1..00000000000 --- a/lang/java/codegen-test/src/test/resources/avro/union_with_logical_types.avsc +++ /dev/null @@ -1,13 +0,0 @@ -{"namespace": "org.apache.avro.codegentest.testdata", - "type": "record", - "name": "UnionWithLogicalTypes", - "doc" : "Test unions with logical types in generated Java classes", - "fields": [ - {"name": "stringOrNull", "type": ["null", {"type": "string", "java-class": "java.lang.String"}], "default": null}, - {"name": "dateOrNull", "type": ["null", {"type": "int", "logicalType": "date"}], "default": null}, - {"name": "decimalOrNull", "type": ["null", {"type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 2}], "default": null} - ] -} - - - diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java index ed6abbd6e9f..4e3250d895a 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java @@ -286,7 +286,7 @@ public Collection getUsedConversionClasses(Schema schema) { for (Conversion conversion : specificData.getConversions()) { classnameToConversion.put(conversion.getConvertedType().getCanonicalName(), conversion); } - Collection result = new ArrayList<>(); + Collection result = new HashSet<>(); for (Field field : schema.getFields()) { final String className = javaType(field.schema()); if (classnameToConversion.containsKey(className)) { @@ -746,14 +746,13 @@ public String conversionInstance(Schema schema) { return "null"; } - if (LogicalTypes.date().equals(schema.getLogicalType())) { - return "DATE_CONVERSION"; - } else if (LogicalTypes.timeMillis().equals(schema.getLogicalType())) { - return "TIME_CONVERSION"; - } else if (LogicalTypes.timestampMillis().equals(schema.getLogicalType())) { - return "TIMESTAMP_CONVERSION"; - } else if (LogicalTypes.Decimal.class.equals(schema.getLogicalType().getClass())) { - return enableDecimalLogicalType ? "DECIMAL_CONVERSION" : "null"; + if (LogicalTypes.Decimal.class.equals(schema.getLogicalType().getClass()) && !enableDecimalLogicalType) { + return "null"; + } + + final Conversion conversion = specificData.getConversionFor(schema.getLogicalType()); + if (conversion != null) { + return "new " + conversion.getClass().getCanonicalName() + "()"; } return "null"; diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm index 17e165946b6..980d6828086 100644 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm @@ -169,17 +169,6 @@ static { } #if ($this.hasLogicalTypeField($schema)) - protected static final org.apache.avro.Conversions.DecimalConversion DECIMAL_CONVERSION = new org.apache.avro.Conversions.DecimalConversion(); -#if ($this.getDateTimeLogicalTypeImplementation().name() == "JODA") - protected static final org.apache.avro.data.TimeConversions.DateConversion DATE_CONVERSION = new org.apache.avro.data.TimeConversions.DateConversion(); - protected static final org.apache.avro.data.TimeConversions.TimeConversion TIME_CONVERSION = new org.apache.avro.data.TimeConversions.TimeConversion(); - protected static final org.apache.avro.data.TimeConversions.TimestampConversion TIMESTAMP_CONVERSION = new org.apache.avro.data.TimeConversions.TimestampConversion(); -#elseif ($this.getDateTimeLogicalTypeImplementation().name() == "JSR310") - protected static final org.apache.avro.data.Jsr310TimeConversions.DateConversion DATE_CONVERSION = new org.apache.avro.data.Jsr310TimeConversions.DateConversion(); - protected static final org.apache.avro.data.Jsr310TimeConversions.TimeMillisConversion TIME_CONVERSION = new org.apache.avro.data.Jsr310TimeConversions.TimeMillisConversion(); - protected static final org.apache.avro.data.Jsr310TimeConversions.TimestampMillisConversion TIMESTAMP_CONVERSION = new org.apache.avro.data.Jsr310TimeConversions.TimestampMillisConversion(); -#end - private static final org.apache.avro.Conversion[] conversions = new org.apache.avro.Conversion[] { #foreach ($field in $schema.getFields()) diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java index b5b3a0ab29d..65a2865bb62 100644 --- a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java @@ -582,12 +582,12 @@ public void testConversionInstanceWithDecimalLogicalTypeDisabled() throws Except Schema uuidSchema = LogicalTypes.uuid() .addToSchema(Schema.create(Schema.Type.STRING)); - Assert.assertEquals("Should use DATE_CONVERSION for date type", - "DATE_CONVERSION", compiler.conversionInstance(dateSchema)); - Assert.assertEquals("Should use TIME_CONVERSION for time type", - "TIME_CONVERSION", compiler.conversionInstance(timeSchema)); - Assert.assertEquals("Should use TIMESTAMP_CONVERSION for date type", - "TIMESTAMP_CONVERSION", compiler.conversionInstance(timestampSchema)); + Assert.assertEquals("Should use date conversion for date type", + "new org.apache.avro.data.TimeConversions.DateConversion()", compiler.conversionInstance(dateSchema)); + Assert.assertEquals("Should use time conversion for time type", + "new org.apache.avro.data.TimeConversions.TimeConversion()", compiler.conversionInstance(timeSchema)); + Assert.assertEquals("Should use timestamp conversion for date type", + "new org.apache.avro.data.TimeConversions.TimestampConversion()", compiler.conversionInstance(timestampSchema)); Assert.assertEquals("Should use null for decimal if the flag is off", "null", compiler.conversionInstance(decimalSchema)); Assert.assertEquals("Should use null for decimal if the flag is off", @@ -611,14 +611,14 @@ public void testConversionInstanceWithDecimalLogicalTypeEnabled() throws Excepti Schema uuidSchema = LogicalTypes.uuid() .addToSchema(Schema.create(Schema.Type.STRING)); - Assert.assertEquals("Should use DATE_CONVERSION for date type", - "DATE_CONVERSION", compiler.conversionInstance(dateSchema)); - Assert.assertEquals("Should use TIME_CONVERSION for time type", - "TIME_CONVERSION", compiler.conversionInstance(timeSchema)); - Assert.assertEquals("Should use TIMESTAMP_CONVERSION for date type", - "TIMESTAMP_CONVERSION", compiler.conversionInstance(timestampSchema)); + Assert.assertEquals("Should use date conversion for date type", + "new org.apache.avro.data.TimeConversions.DateConversion()", compiler.conversionInstance(dateSchema)); + Assert.assertEquals("Should use time conversion for time type", + "new org.apache.avro.data.TimeConversions.TimeConversion()", compiler.conversionInstance(timeSchema)); + Assert.assertEquals("Should use timestamp conversion for date type", + "new org.apache.avro.data.TimeConversions.TimestampConversion()", compiler.conversionInstance(timestampSchema)); Assert.assertEquals("Should use null for decimal if the flag is off", - "DECIMAL_CONVERSION", compiler.conversionInstance(decimalSchema)); + "new org.apache.avro.Conversions.DecimalConversion()", compiler.conversionInstance(decimalSchema)); Assert.assertEquals("Should use null for decimal if the flag is off", "null", compiler.conversionInstance(uuidSchema)); } diff --git a/lang/java/codegen-test/pom.xml b/lang/java/integration-test/codegen-test/pom.xml similarity index 93% rename from lang/java/codegen-test/pom.xml rename to lang/java/integration-test/codegen-test/pom.xml index 3cb00d119a6..6fb9c54eafe 100644 --- a/lang/java/codegen-test/pom.xml +++ b/lang/java/integration-test/codegen-test/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - avro-parent + avro-integration-test org.apache.avro 1.9.0-SNAPSHOT ../ @@ -52,14 +52,14 @@ ${project.basedir}/src/test/resources/avro ${project.build.directory}/generated-test-sources/java true - org.apache.avro.codegentest.CustomConversion + org.apache.avro.codegentest.CustomDecimalConversion org.apache.avro - avro-codegen-test-deps + avro-test-custom-conversions ${project.version} @@ -81,7 +81,7 @@ ${project.groupId} - avro-codegen-test-deps + avro-test-custom-conversions ${project.version} diff --git a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestCustomConversion.java b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestCustomConversion.java new file mode 100644 index 00000000000..81bdc281038 --- /dev/null +++ b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestCustomConversion.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.avro.codegentest; + +import org.apache.avro.codegentest.testdata.LogicalTypesWithCustomConversion; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificDatumWriter; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigInteger; + +public class TestCustomConversion { + + @Test + public void testNullValues() throws IOException { + LogicalTypesWithCustomConversion instanceOfGeneratedClass = LogicalTypesWithCustomConversion.newBuilder() + .setNonNullCustomField(new CustomDecimal(BigInteger.valueOf(100), 2)) + .build(); + final byte[] serialized = serialize(instanceOfGeneratedClass); + final LogicalTypesWithCustomConversion copy = deserialize(serialized); + Assert.assertEquals(instanceOfGeneratedClass.getNonNullCustomField(), copy.getNonNullCustomField()); + Assert.assertEquals(instanceOfGeneratedClass.getNullableCustomField(), copy.getNullableCustomField()); + } + + @Test + public void testNonNullValues() throws IOException { + LogicalTypesWithCustomConversion instanceOfGeneratedClass = LogicalTypesWithCustomConversion.newBuilder() + .setNonNullCustomField(new CustomDecimal(BigInteger.valueOf(100), 2)) + .setNullableCustomField(new CustomDecimal(BigInteger.valueOf(3000), 2)) + .build(); + final byte[] serialized = serialize(instanceOfGeneratedClass); + final LogicalTypesWithCustomConversion copy = deserialize(serialized); + Assert.assertEquals(instanceOfGeneratedClass.getNullableCustomField(), copy.getNullableCustomField()); + Assert.assertEquals(instanceOfGeneratedClass.getNonNullCustomField(), copy.getNonNullCustomField()); + } + + private byte[] serialize(LogicalTypesWithCustomConversion object) { + SpecificDatumWriter datumWriter = new SpecificDatumWriter<>(LogicalTypesWithCustomConversion.getClassSchema()); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try { + datumWriter.write(object, EncoderFactory.get().directBinaryEncoder(outputStream, null)); + return outputStream.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private LogicalTypesWithCustomConversion deserialize(byte[] bytes) { + SpecificDatumReader datumReader = new SpecificDatumReader<>(LogicalTypesWithCustomConversion.getClassSchema(), LogicalTypesWithCustomConversion.getClassSchema()); + try { + final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + return datumReader.read(null, DecoderFactory.get().directBinaryDecoder(byteArrayInputStream, null)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestNullableLogicalTypes.java b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestNullableLogicalTypes.java new file mode 100644 index 00000000000..375ba86cb38 --- /dev/null +++ b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestNullableLogicalTypes.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.avro.codegentest; + +import org.apache.avro.codegentest.testdata.NullableLogicalTypes; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificDatumWriter; +import org.joda.time.LocalDate; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +public class TestNullableLogicalTypes { + + @Test + public void testWithNullValues() throws IOException { + NullableLogicalTypes instanceOfGeneratedClass = NullableLogicalTypes.newBuilder() + .setNullableDate(null) + .build(); + final byte[] serialized = serialize(instanceOfGeneratedClass); + final NullableLogicalTypes copy = deserialize(serialized); + Assert.assertEquals(instanceOfGeneratedClass.getNullableDate(), copy.getNullableDate()); + } + + @Test + public void testDate() throws IOException { + NullableLogicalTypes instanceOfGeneratedClass = NullableLogicalTypes.newBuilder() + .setNullableDate(LocalDate.now()) + .build(); + final byte[] serialized = serialize(instanceOfGeneratedClass); + final NullableLogicalTypes copy = deserialize(serialized); + Assert.assertEquals(instanceOfGeneratedClass.getNullableDate(), copy.getNullableDate()); + } + + private byte[] serialize(NullableLogicalTypes object) { + SpecificDatumWriter datumWriter = new SpecificDatumWriter<>(NullableLogicalTypes.getClassSchema()); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try { + datumWriter.write(object, EncoderFactory.get().directBinaryEncoder(outputStream, null)); + return outputStream.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private NullableLogicalTypes deserialize(byte[] bytes) { + SpecificDatumReader datumReader = new SpecificDatumReader<>(NullableLogicalTypes.getClassSchema(), NullableLogicalTypes.getClassSchema()); + try { + final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + return datumReader.read(null, DecoderFactory.get().directBinaryDecoder(byteArrayInputStream, null)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/lang/java/integration-test/codegen-test/src/test/resources/avro/custom_conversion.avsc b/lang/java/integration-test/codegen-test/src/test/resources/avro/custom_conversion.avsc new file mode 100644 index 00000000000..ff33c39fa65 --- /dev/null +++ b/lang/java/integration-test/codegen-test/src/test/resources/avro/custom_conversion.avsc @@ -0,0 +1,12 @@ +{"namespace": "org.apache.avro.codegentest.testdata", + "type": "record", + "name": "LogicalTypesWithCustomConversion", + "doc" : "Test unions with logical types in generated Java classes", + "fields": [ + {"name": "nullableCustomField", "type": ["null", {"type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 2}], "default": null}, + {"name": "nonNullCustomField", "type": {"type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 2}} + ] +} + + + diff --git a/lang/java/integration-test/codegen-test/src/test/resources/avro/nullable_logical_types.avsc b/lang/java/integration-test/codegen-test/src/test/resources/avro/nullable_logical_types.avsc new file mode 100644 index 00000000000..0133133b459 --- /dev/null +++ b/lang/java/integration-test/codegen-test/src/test/resources/avro/nullable_logical_types.avsc @@ -0,0 +1,11 @@ +{"namespace": "org.apache.avro.codegentest.testdata", + "type": "record", + "name": "NullableLogicalTypes", + "doc" : "Test unions with logical types in generated Java classes", + "fields": [ + {"name": "nullableDate", "type": ["null", {"type": "int", "logicalType": "date"}], "default": null} + ] +} + + + diff --git a/lang/java/integration-test/pom.xml b/lang/java/integration-test/pom.xml new file mode 100644 index 00000000000..226a0dcbf91 --- /dev/null +++ b/lang/java/integration-test/pom.xml @@ -0,0 +1,99 @@ + + + + 4.0.0 + + + avro-parent + org.apache.avro + 1.9.0-SNAPSHOT + ../ + + + avro-integration-test + Avro Integration Tests + Integration tests for code generation or other things that are hard to test within the modules without creating circular Maven dependencies. + http://avro.apache.org/ + pom + + + codegen-test + test-custom-conversions + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire-plugin.version} + + false + + + + org.apache.maven.plugins + maven-compiler-plugin + ${compiler-plugin.version} + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${checkstyle-plugin.version} + + true + checkstyle.xml + + + + checkstyle-check + test + + check + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${jar-plugin.version} + + + + test-jar + + + + + + + + + + + + + diff --git a/lang/java/codegen-test-deps/pom.xml b/lang/java/integration-test/test-custom-conversions/pom.xml similarity index 93% rename from lang/java/codegen-test-deps/pom.xml rename to lang/java/integration-test/test-custom-conversions/pom.xml index ebbd620094d..7bac7ae4298 100644 --- a/lang/java/codegen-test-deps/pom.xml +++ b/lang/java/integration-test/test-custom-conversions/pom.xml @@ -21,13 +21,13 @@ 4.0.0 - avro-parent + avro-integration-test org.apache.avro 1.9.0-SNAPSHOT ../ - avro-codegen-test-deps + avro-test-custom-conversions Apache Avro Codegen Test dependencies jar diff --git a/lang/java/integration-test/test-custom-conversions/src/main/java/org.apache.avro.codegentest/CustomDecimal.java b/lang/java/integration-test/test-custom-conversions/src/main/java/org.apache.avro.codegentest/CustomDecimal.java new file mode 100644 index 00000000000..d19d52d46e8 --- /dev/null +++ b/lang/java/integration-test/test-custom-conversions/src/main/java/org.apache.avro.codegentest/CustomDecimal.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.avro.codegentest; + +import java.math.BigDecimal; +import java.math.BigInteger; + +/** + * Wraps a BigDecimal just to demonstrate that it is possible to use custom implementation classes with custom conversions. + */ +public class CustomDecimal { + + private final BigDecimal internalValue; + + public CustomDecimal(BigInteger value, int scale) { + internalValue = new BigDecimal(value, scale); + } + + public byte[] toByteArray(int scale) { + final BigDecimal correctlyScaledValue; + if (scale != internalValue.scale()) { + correctlyScaledValue = internalValue.setScale(scale, BigDecimal.ROUND_HALF_UP); + } else { + correctlyScaledValue = internalValue; + } + return correctlyScaledValue.unscaledValue().toByteArray(); + + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CustomDecimal that = (CustomDecimal) o; + + return internalValue.equals(that.internalValue); + } + + @Override + public int hashCode() { + return internalValue.hashCode(); + } +} diff --git a/lang/java/integration-test/test-custom-conversions/src/main/java/org.apache.avro.codegentest/CustomDecimalConversion.java b/lang/java/integration-test/test-custom-conversions/src/main/java/org.apache.avro.codegentest/CustomDecimalConversion.java new file mode 100644 index 00000000000..7c200ad0b27 --- /dev/null +++ b/lang/java/integration-test/test-custom-conversions/src/main/java/org.apache.avro.codegentest/CustomDecimalConversion.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.avro.codegentest; + +import org.apache.avro.Conversion; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; + +import java.math.BigInteger; +import java.nio.ByteBuffer; + +public class CustomDecimalConversion extends Conversion { + + @Override + public Class getConvertedType() { + return CustomDecimal.class; + } + + @Override + public String getLogicalTypeName() { + return "decimal"; + } + + public CustomDecimal fromBytes(ByteBuffer value, Schema schema, LogicalType type) { + int scale = ((LogicalTypes.Decimal)type).getScale(); + byte[] bytes = value.get(new byte[value.remaining()]).array(); + return new CustomDecimal(new BigInteger(bytes), scale); + } + + public ByteBuffer toBytes(CustomDecimal value, Schema schema, LogicalType type) { + int scale = ((LogicalTypes.Decimal)type).getScale(); + return ByteBuffer.wrap(value.toByteArray(scale)); + } + +} diff --git a/lang/java/mapred/pom.xml b/lang/java/mapred/pom.xml index e836ba66aa8..67742c478e0 100644 --- a/lang/java/mapred/pom.xml +++ b/lang/java/mapred/pom.xml @@ -48,6 +48,18 @@ + + org.apache.maven.plugins + maven-jar-plugin + ${jar-plugin.version} + + + + test-jar + + + + ${project.groupId} avro-maven-plugin diff --git a/lang/java/pom.xml b/lang/java/pom.xml index 9fc3a7eb88f..e368f781802 100644 --- a/lang/java/pom.xml +++ b/lang/java/pom.xml @@ -97,8 +97,7 @@ thrift archetypes grpc - codegen-test - codegen-test-deps + integration-test From 4be369bc77c419b2d07841ccbafdb8e3f9a772b9 Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Tue, 21 Aug 2018 15:15:48 +0200 Subject: [PATCH 11/18] Default values and conversions --- .../apache/avro/data/RecordBuilderBase.java | 32 +------ .../specific/templates/java/classic/record.vm | 8 -- .../TestLogicalTypesWithDefaults.java | 95 +++++++++++++++++++ .../logical_types_with_default_values.avsc | 12 +++ 4 files changed, 110 insertions(+), 37 deletions(-) create mode 100644 lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestLogicalTypesWithDefaults.java create mode 100644 lang/java/integration-test/codegen-test/src/test/resources/avro/logical_types_with_default_values.avsc diff --git a/lang/java/avro/src/main/java/org/apache/avro/data/RecordBuilderBase.java b/lang/java/avro/src/main/java/org/apache/avro/data/RecordBuilderBase.java index 8805ba5a71f..9fb4f72c8ae 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/data/RecordBuilderBase.java +++ b/lang/java/avro/src/main/java/org/apache/avro/data/RecordBuilderBase.java @@ -17,19 +17,16 @@ */ package org.apache.avro.data; -import java.io.IOException; -import java.util.Arrays; - import org.apache.avro.AvroRuntimeException; -import org.apache.avro.Conversion; -import org.apache.avro.Conversions; -import org.apache.avro.LogicalType; import org.apache.avro.Schema; import org.apache.avro.Schema.Field; import org.apache.avro.Schema.Type; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.IndexedRecord; +import java.io.IOException; +import java.util.Arrays; + /** Abstract base class for RecordBuilder implementations. Not thread-safe. */ public abstract class RecordBuilderBase implements RecordBuilder { @@ -138,29 +135,6 @@ protected Object defaultValue(Field field) throws IOException { return data.deepCopy(field.schema(), data.getDefaultValue(field)); } - /** - * Gets the default value of the given field, if any. Pass in a conversion - * to convert data to logical type class. Please make sure the schema does - * have a logical type, otherwise an exception would be thrown out. - * @param field the field whose default value should be retrieved. - * @param conversion the tool to convert data to logical type class - * @return the default value associated with the given field, - * or null if none is specified in the schema. - * @throws IOException - */ - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected Object defaultValue(Field field, Conversion conversion) throws IOException { - Schema schema = field.schema(); - LogicalType logicalType = schema.getLogicalType(); - Object rawDefaultValue = data.deepCopy(schema, data.getDefaultValue(field)); - if (conversion == null || logicalType == null) { - return rawDefaultValue; - } else { - return Conversions.convertToLogicalType(rawDefaultValue, schema, - logicalType, conversion); - } - } - @Override public int hashCode() { final int prime = 31; diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm index 980d6828086..387fa21b5ab 100644 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm @@ -452,19 +452,11 @@ static { throw e; } } else { -#if ($this.hasLogicalTypeField($schema)) - record.${this.mangle($field.name(), $schema.isError())} = fieldSetFlags()[$field.pos()] ? this.${this.mangle($field.name(), $schema.isError())} : #if(${this.javaType($field.schema())} != "java.lang.Object")(${this.javaType($field.schema())})#{end} defaultValue(fields()[$field.pos()], record.getConversion($field.pos())); -#else record.${this.mangle($field.name(), $schema.isError())} = fieldSetFlags()[$field.pos()] ? this.${this.mangle($field.name(), $schema.isError())} : #if(${this.javaType($field.schema())} != "java.lang.Object")(${this.javaType($field.schema())})#{end} defaultValue(fields()[$field.pos()]); -#end } -#else -#if ($this.hasLogicalTypeField($schema)) - record.${this.mangle($field.name(), $schema.isError())} = fieldSetFlags()[$field.pos()] ? this.${this.mangle($field.name(), $schema.isError())} : #if(${this.javaType($field.schema())} != "java.lang.Object")(${this.javaType($field.schema())})#{end} defaultValue(fields()[$field.pos()], record.getConversion($field.pos())); #else record.${this.mangle($field.name(), $schema.isError())} = fieldSetFlags()[$field.pos()] ? this.${this.mangle($field.name(), $schema.isError())} : #if(${this.javaType($field.schema())} != "java.lang.Object")(${this.javaType($field.schema())})#{end} defaultValue(fields()[$field.pos()]); #end -#end #end return record; } catch (org.apache.avro.AvroMissingFieldException e) { diff --git a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestLogicalTypesWithDefaults.java b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestLogicalTypesWithDefaults.java new file mode 100644 index 00000000000..4c7e42224f3 --- /dev/null +++ b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestLogicalTypesWithDefaults.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.avro.codegentest; + +import org.apache.avro.codegentest.testdata.LogicalTypesWithDefaults; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificDatumWriter; +import org.joda.time.LocalDate; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +public class TestLogicalTypesWithDefaults { + + LocalDate DEFAULT_VALUE = LocalDate.parse("1973-05-19"); + + @Test + public void testDefaultValueOfNullableField() throws IOException { + LogicalTypesWithDefaults instanceOfGeneratedClass = LogicalTypesWithDefaults.newBuilder() + .setNonNullDate(LocalDate.now()) + .build(); + final byte[] serialized = serialize(instanceOfGeneratedClass); + final LogicalTypesWithDefaults copy = deserialize(serialized); + Assert.assertEquals(DEFAULT_VALUE, instanceOfGeneratedClass.getNullableDate()); + Assert.assertEquals(instanceOfGeneratedClass.getNullableDate(), copy.getNullableDate()); + Assert.assertEquals(instanceOfGeneratedClass.getNonNullDate(), copy.getNonNullDate()); + } + + @Test + public void testDefaultValueOfNonNullField() throws IOException { + LogicalTypesWithDefaults instanceOfGeneratedClass = LogicalTypesWithDefaults.newBuilder() + .setNullableDate(LocalDate.now()) + .build(); + final byte[] serialized = serialize(instanceOfGeneratedClass); + final LogicalTypesWithDefaults copy = deserialize(serialized); + Assert.assertEquals(DEFAULT_VALUE, instanceOfGeneratedClass.getNonNullDate()); + Assert.assertEquals(instanceOfGeneratedClass.getNullableDate(), copy.getNullableDate()); + Assert.assertEquals(instanceOfGeneratedClass.getNonNullDate(), copy.getNonNullDate()); + } + + @Test + public void testWithValues() throws IOException { + LogicalTypesWithDefaults instanceOfGeneratedClass = LogicalTypesWithDefaults.newBuilder() + .setNullableDate(LocalDate.now()) + .setNonNullDate(LocalDate.now()) + .build(); + final byte[] serialized = serialize(instanceOfGeneratedClass); + final LogicalTypesWithDefaults copy = deserialize(serialized); + Assert.assertEquals(instanceOfGeneratedClass.getNullableDate(), copy.getNullableDate()); + Assert.assertEquals(instanceOfGeneratedClass.getNonNullDate(), copy.getNonNullDate()); + } + + private byte[] serialize(LogicalTypesWithDefaults object) { + SpecificDatumWriter datumWriter = new SpecificDatumWriter<>(LogicalTypesWithDefaults.getClassSchema()); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try { + datumWriter.write(object, EncoderFactory.get().directBinaryEncoder(outputStream, null)); + return outputStream.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private LogicalTypesWithDefaults deserialize(byte[] bytes) { + SpecificDatumReader datumReader = new SpecificDatumReader<>(LogicalTypesWithDefaults.getClassSchema(), LogicalTypesWithDefaults.getClassSchema()); + try { + final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + return datumReader.read(null, DecoderFactory.get().directBinaryDecoder(byteArrayInputStream, null)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/lang/java/integration-test/codegen-test/src/test/resources/avro/logical_types_with_default_values.avsc b/lang/java/integration-test/codegen-test/src/test/resources/avro/logical_types_with_default_values.avsc new file mode 100644 index 00000000000..d164b0a65a3 --- /dev/null +++ b/lang/java/integration-test/codegen-test/src/test/resources/avro/logical_types_with_default_values.avsc @@ -0,0 +1,12 @@ +{"namespace": "org.apache.avro.codegentest.testdata", + "type": "record", + "name": "LogicalTypesWithDefaults", + "doc" : "Test logical types and default values in generated Java classes", + "fields": [ + {"name": "nullableDate", "type": [{"type": "int", "logicalType": "date"}, "null"], "default": 1234}, + {"name": "nonNullDate", "type": {"type": "int", "logicalType": "date"}, "default": 1234} + ] +} + + + From 08a40fcd551c7ed9decf2ac643a1742b7ae451f5 Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Tue, 21 Aug 2018 16:23:30 +0200 Subject: [PATCH 12/18] Cleanup of some changes in Maven plugin --- .../apache/avro/mojo/AbstractAvroMojo.java | 22 +--------------- .../org/apache/avro/mojo/IDLProtocolMojo.java | 25 ++++++++++++++++--- .../org/apache/avro/mojo/ProtocolMojo.java | 2 +- .../java/org/apache/avro/mojo/SchemaMojo.java | 2 +- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java index 7064dd91f30..4c49e3730ac 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/AbstractAvroMojo.java @@ -277,7 +277,7 @@ protected DateTimeLogicalTypeImplementation getDateTimeLogicalTypeImplementation protected abstract void doCompile(String filename, File sourceDirectory, File outputDirectory) throws IOException; - protected URLClassLoader createClassLoader2() throws DependencyResolutionRequiredException, MalformedURLException { + protected URLClassLoader createClassLoader() throws DependencyResolutionRequiredException, MalformedURLException { List urls = appendElements(project.getRuntimeClasspathElements()); urls.addAll(appendElements(project.getTestClasspathElements())); return new URLClassLoader(urls.toArray(new URL[urls.size()]), @@ -295,26 +295,6 @@ private List appendElements(List runtimeClasspathElements) throws Malformed return runtimeUrls; } - protected URLClassLoader createClassLoaderIncludingCurrentProjectClasses(File sourceDirectory) throws MalformedURLException, DependencyResolutionRequiredException { - List runtimeClasspathElements = project.getRuntimeClasspathElements(); - List runtimeUrls = new ArrayList<>(); - - // Add the source directory of avro files to the classpath so that - // imports can refer to other idl files as classpath resources - runtimeUrls.add(sourceDirectory.toURI().toURL()); - - // If runtimeClasspathElements is not empty values add its values to Idl path. - if (runtimeClasspathElements != null && !runtimeClasspathElements.isEmpty()) { - for (Object runtimeClasspathElement : runtimeClasspathElements) { - String element = (String) runtimeClasspathElement; - runtimeUrls.add(new File(element).toURI().toURL()); - } - } - - return new URLClassLoader - (runtimeUrls.toArray(new URL[0]), Thread.currentThread().getContextClassLoader()); - } - protected abstract String[] getIncludes(); protected abstract String[] getTestIncludes(); diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java index 98f0fe41081..6e1fe8caf6c 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/IDLProtocolMojo.java @@ -20,7 +20,10 @@ import java.io.File; import java.io.IOException; +import java.net.URL; import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.List; import org.apache.avro.Protocol; import org.apache.avro.compiler.idl.Idl; @@ -60,11 +63,27 @@ public class IDLProtocolMojo extends AbstractAvroMojo { @Override protected void doCompile(String filename, File sourceDirectory, File outputDirectory) throws IOException { try { - Idl parser; @SuppressWarnings("rawtypes") + List runtimeClasspathElements = project.getRuntimeClasspathElements(); + Idl parser; + + List runtimeUrls = new ArrayList<>(); + + // Add the source directory of avro files to the classpath so that + // imports can refer to other idl files as classpath resources + runtimeUrls.add(sourceDirectory.toURI().toURL()); + + // If runtimeClasspathElements is not empty values add its values to Idl path. + if (runtimeClasspathElements != null && !runtimeClasspathElements.isEmpty()) { + for (Object runtimeClasspathElement : runtimeClasspathElements) { + String element = (String) runtimeClasspathElement; + runtimeUrls.add(new File(element).toURI().toURL()); + } + } - URLClassLoader projPathLoader = createClassLoaderIncludingCurrentProjectClasses(sourceDirectory); - parser = new Idl(new File(sourceDirectory, filename), projPathLoader); + URLClassLoader projPathLoader = new URLClassLoader + (runtimeUrls.toArray(new URL[0]), Thread.currentThread().getContextClassLoader()); + parser = new Idl(new File(sourceDirectory, filename), projPathLoader); Protocol p = parser.CompilationUnit(); String json = p.toString(true); diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java index 01ef7d6ac57..7dd769c7946 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/ProtocolMojo.java @@ -67,7 +67,7 @@ protected void doCompile(String filename, File sourceDirectory, File outputDirec compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); final URLClassLoader classLoader; try { - classLoader = createClassLoader2(); + classLoader = createClassLoader(); for (String customConversion : customConversions) { compiler.addCustomConversion(classLoader.loadClass(customConversion)); } diff --git a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java index 2190e305578..953feeade17 100644 --- a/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java +++ b/lang/java/maven-plugin/src/main/java/org/apache/avro/mojo/SchemaMojo.java @@ -83,7 +83,7 @@ protected void doCompile(String filename, File sourceDirectory, File outputDirec compiler.setCreateSetters(createSetters); compiler.setEnableDecimalLogicalType(enableDecimalLogicalType); try { - final URLClassLoader classLoader = createClassLoader2(); + final URLClassLoader classLoader = createClassLoader(); for (String customConversion : customConversions) { compiler.addCustomConversion(classLoader.loadClass(customConversion)); } From e142230fa06e5bb2a42f32cc1d3203a258f9d2af Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Wed, 22 Aug 2018 14:19:41 +0200 Subject: [PATCH 13/18] Fixed equals() for classes with nested logical types. Improved tests --- .../avro/specific/SpecificRecordBase.java | 13 +++-- .../specific/templates/java/classic/record.vm | 1 + .../AbstractSpecificRecordTest.java | 47 +++++++++++++++++++ .../codegentest/TestCustomConversion.java | 41 ++-------------- .../TestLogicalTypesWithDefaults.java | 47 ++----------------- .../codegentest/TestNullableLogicalTypes.java | 38 ++------------- .../CustomDecimal.java | 7 ++- .../avro/examples/baseball/Player.java | 1 + .../src/test/compiler/output/Player.java | 1 + 9 files changed, 76 insertions(+), 120 deletions(-) create mode 100644 lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java diff --git a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificRecordBase.java b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificRecordBase.java index 1902cbc52be..2a7b9697e43 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificRecordBase.java +++ b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificRecordBase.java @@ -35,6 +35,11 @@ public abstract class SpecificRecordBase public abstract Object get(int field); public abstract void put(int field, Object value); + public SpecificData getSpecificData() { + // Default implementation for backwards compatibility, overridden in generated code + return SpecificData.get(); + } + public Conversion getConversion(int field) { // for backward-compatibility. no older specific classes have conversions. return null; @@ -59,22 +64,22 @@ public boolean equals(Object that) { if (that == this) return true; // identical object if (!(that instanceof SpecificRecord)) return false; // not a record if (this.getClass() != that.getClass()) return false; // not same schema - return SpecificData.get().compare(this, that, this.getSchema(), true) == 0; + return getSpecificData().compare(this, that, this.getSchema(), true) == 0; } @Override public int hashCode() { - return SpecificData.get().hashCode(this, this.getSchema()); + return getSpecificData().hashCode(this, this.getSchema()); } @Override public int compareTo(SpecificRecord that) { - return SpecificData.get().compare(this, that, this.getSchema()); + return getSpecificData().compare(this, that, this.getSchema()); } @Override public String toString() { - return SpecificData.get().toString(this); + return getSpecificData().toString(this); } @Override diff --git a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm index 387fa21b5ab..d5c05a6331d 100644 --- a/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm +++ b/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm @@ -155,6 +155,7 @@ static { #end #end + public org.apache.avro.specific.SpecificData getSpecificData() { return MODEL$; } public org.apache.avro.Schema getSchema() { return SCHEMA$; } // Used by DatumWriter. Applications should not call. public java.lang.Object get(int field$) { diff --git a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java new file mode 100644 index 00000000000..728d1e1d9cd --- /dev/null +++ b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java @@ -0,0 +1,47 @@ +package org.apache.avro.codegentest; + +import org.apache.avro.Schema; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.avro.specific.SpecificRecordBase; +import org.junit.Assert; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +abstract class AbstractSpecificRecordTest { + + void verifySerDeAndStandardMethods(T original) { + final byte[] serialized = serialize(original); + final T copy = deserialize(serialized, original.getSchema()); + Assert.assertEquals(original, copy); + // In addition to equals() tested above, make sure the other methods that use SpecificData work as intended + Assert.assertEquals(0, original.compareTo(copy)); + Assert.assertEquals(original.hashCode(), copy.hashCode()); + Assert.assertEquals(original.toString(), copy.toString()); + } + + private byte[] serialize(T object) { + SpecificDatumWriter datumWriter = new SpecificDatumWriter<>(object.getSchema()); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try { + datumWriter.write(object, EncoderFactory.get().directBinaryEncoder(outputStream, null)); + return outputStream.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private T deserialize(byte[] bytes, Schema schema) { + SpecificDatumReader datumReader = new SpecificDatumReader<>(schema, schema); + try { + final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + return datumReader.read(null, DecoderFactory.get().directBinaryDecoder(byteArrayInputStream, null)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestCustomConversion.java b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestCustomConversion.java index 81bdc281038..55e60a224b0 100644 --- a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestCustomConversion.java +++ b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestCustomConversion.java @@ -19,29 +19,19 @@ package org.apache.avro.codegentest; import org.apache.avro.codegentest.testdata.LogicalTypesWithCustomConversion; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.specific.SpecificDatumReader; -import org.apache.avro.specific.SpecificDatumWriter; -import org.junit.Assert; import org.junit.Test; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigInteger; -public class TestCustomConversion { +public class TestCustomConversion extends AbstractSpecificRecordTest { @Test public void testNullValues() throws IOException { LogicalTypesWithCustomConversion instanceOfGeneratedClass = LogicalTypesWithCustomConversion.newBuilder() .setNonNullCustomField(new CustomDecimal(BigInteger.valueOf(100), 2)) .build(); - final byte[] serialized = serialize(instanceOfGeneratedClass); - final LogicalTypesWithCustomConversion copy = deserialize(serialized); - Assert.assertEquals(instanceOfGeneratedClass.getNonNullCustomField(), copy.getNonNullCustomField()); - Assert.assertEquals(instanceOfGeneratedClass.getNullableCustomField(), copy.getNullableCustomField()); + verifySerDeAndStandardMethods(instanceOfGeneratedClass); } @Test @@ -50,31 +40,6 @@ public void testNonNullValues() throws IOException { .setNonNullCustomField(new CustomDecimal(BigInteger.valueOf(100), 2)) .setNullableCustomField(new CustomDecimal(BigInteger.valueOf(3000), 2)) .build(); - final byte[] serialized = serialize(instanceOfGeneratedClass); - final LogicalTypesWithCustomConversion copy = deserialize(serialized); - Assert.assertEquals(instanceOfGeneratedClass.getNullableCustomField(), copy.getNullableCustomField()); - Assert.assertEquals(instanceOfGeneratedClass.getNonNullCustomField(), copy.getNonNullCustomField()); + verifySerDeAndStandardMethods(instanceOfGeneratedClass); } - - private byte[] serialize(LogicalTypesWithCustomConversion object) { - SpecificDatumWriter datumWriter = new SpecificDatumWriter<>(LogicalTypesWithCustomConversion.getClassSchema()); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try { - datumWriter.write(object, EncoderFactory.get().directBinaryEncoder(outputStream, null)); - return outputStream.toByteArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private LogicalTypesWithCustomConversion deserialize(byte[] bytes) { - SpecificDatumReader datumReader = new SpecificDatumReader<>(LogicalTypesWithCustomConversion.getClassSchema(), LogicalTypesWithCustomConversion.getClassSchema()); - try { - final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); - return datumReader.read(null, DecoderFactory.get().directBinaryDecoder(byteArrayInputStream, null)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } diff --git a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestLogicalTypesWithDefaults.java b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestLogicalTypesWithDefaults.java index 4c7e42224f3..c2d2d6d2f7c 100644 --- a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestLogicalTypesWithDefaults.java +++ b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestLogicalTypesWithDefaults.java @@ -19,32 +19,22 @@ package org.apache.avro.codegentest; import org.apache.avro.codegentest.testdata.LogicalTypesWithDefaults; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.specific.SpecificDatumReader; -import org.apache.avro.specific.SpecificDatumWriter; import org.joda.time.LocalDate; import org.junit.Assert; import org.junit.Test; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -public class TestLogicalTypesWithDefaults { +public class TestLogicalTypesWithDefaults extends AbstractSpecificRecordTest { - LocalDate DEFAULT_VALUE = LocalDate.parse("1973-05-19"); + private static final LocalDate DEFAULT_VALUE = LocalDate.parse("1973-05-19"); @Test public void testDefaultValueOfNullableField() throws IOException { LogicalTypesWithDefaults instanceOfGeneratedClass = LogicalTypesWithDefaults.newBuilder() .setNonNullDate(LocalDate.now()) .build(); - final byte[] serialized = serialize(instanceOfGeneratedClass); - final LogicalTypesWithDefaults copy = deserialize(serialized); - Assert.assertEquals(DEFAULT_VALUE, instanceOfGeneratedClass.getNullableDate()); - Assert.assertEquals(instanceOfGeneratedClass.getNullableDate(), copy.getNullableDate()); - Assert.assertEquals(instanceOfGeneratedClass.getNonNullDate(), copy.getNonNullDate()); + verifySerDeAndStandardMethods(instanceOfGeneratedClass); } @Test @@ -52,11 +42,8 @@ public void testDefaultValueOfNonNullField() throws IOException { LogicalTypesWithDefaults instanceOfGeneratedClass = LogicalTypesWithDefaults.newBuilder() .setNullableDate(LocalDate.now()) .build(); - final byte[] serialized = serialize(instanceOfGeneratedClass); - final LogicalTypesWithDefaults copy = deserialize(serialized); Assert.assertEquals(DEFAULT_VALUE, instanceOfGeneratedClass.getNonNullDate()); - Assert.assertEquals(instanceOfGeneratedClass.getNullableDate(), copy.getNullableDate()); - Assert.assertEquals(instanceOfGeneratedClass.getNonNullDate(), copy.getNonNullDate()); + verifySerDeAndStandardMethods(instanceOfGeneratedClass); } @Test @@ -65,31 +52,7 @@ public void testWithValues() throws IOException { .setNullableDate(LocalDate.now()) .setNonNullDate(LocalDate.now()) .build(); - final byte[] serialized = serialize(instanceOfGeneratedClass); - final LogicalTypesWithDefaults copy = deserialize(serialized); - Assert.assertEquals(instanceOfGeneratedClass.getNullableDate(), copy.getNullableDate()); - Assert.assertEquals(instanceOfGeneratedClass.getNonNullDate(), copy.getNonNullDate()); - } - - private byte[] serialize(LogicalTypesWithDefaults object) { - SpecificDatumWriter datumWriter = new SpecificDatumWriter<>(LogicalTypesWithDefaults.getClassSchema()); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try { - datumWriter.write(object, EncoderFactory.get().directBinaryEncoder(outputStream, null)); - return outputStream.toByteArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private LogicalTypesWithDefaults deserialize(byte[] bytes) { - SpecificDatumReader datumReader = new SpecificDatumReader<>(LogicalTypesWithDefaults.getClassSchema(), LogicalTypesWithDefaults.getClassSchema()); - try { - final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); - return datumReader.read(null, DecoderFactory.get().directBinaryDecoder(byteArrayInputStream, null)); - } catch (IOException e) { - throw new RuntimeException(e); - } + verifySerDeAndStandardMethods(instanceOfGeneratedClass); } } diff --git a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestNullableLogicalTypes.java b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestNullableLogicalTypes.java index 375ba86cb38..3a44174dcef 100644 --- a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestNullableLogicalTypes.java +++ b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestNullableLogicalTypes.java @@ -19,28 +19,19 @@ package org.apache.avro.codegentest; import org.apache.avro.codegentest.testdata.NullableLogicalTypes; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.specific.SpecificDatumReader; -import org.apache.avro.specific.SpecificDatumWriter; import org.joda.time.LocalDate; -import org.junit.Assert; import org.junit.Test; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -public class TestNullableLogicalTypes { +public class TestNullableLogicalTypes extends AbstractSpecificRecordTest { @Test public void testWithNullValues() throws IOException { NullableLogicalTypes instanceOfGeneratedClass = NullableLogicalTypes.newBuilder() .setNullableDate(null) .build(); - final byte[] serialized = serialize(instanceOfGeneratedClass); - final NullableLogicalTypes copy = deserialize(serialized); - Assert.assertEquals(instanceOfGeneratedClass.getNullableDate(), copy.getNullableDate()); + verifySerDeAndStandardMethods(instanceOfGeneratedClass); } @Test @@ -48,30 +39,7 @@ public void testDate() throws IOException { NullableLogicalTypes instanceOfGeneratedClass = NullableLogicalTypes.newBuilder() .setNullableDate(LocalDate.now()) .build(); - final byte[] serialized = serialize(instanceOfGeneratedClass); - final NullableLogicalTypes copy = deserialize(serialized); - Assert.assertEquals(instanceOfGeneratedClass.getNullableDate(), copy.getNullableDate()); - } - - private byte[] serialize(NullableLogicalTypes object) { - SpecificDatumWriter datumWriter = new SpecificDatumWriter<>(NullableLogicalTypes.getClassSchema()); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try { - datumWriter.write(object, EncoderFactory.get().directBinaryEncoder(outputStream, null)); - return outputStream.toByteArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private NullableLogicalTypes deserialize(byte[] bytes) { - SpecificDatumReader datumReader = new SpecificDatumReader<>(NullableLogicalTypes.getClassSchema(), NullableLogicalTypes.getClassSchema()); - try { - final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); - return datumReader.read(null, DecoderFactory.get().directBinaryDecoder(byteArrayInputStream, null)); - } catch (IOException e) { - throw new RuntimeException(e); - } + verifySerDeAndStandardMethods(instanceOfGeneratedClass); } } diff --git a/lang/java/integration-test/test-custom-conversions/src/main/java/org.apache.avro.codegentest/CustomDecimal.java b/lang/java/integration-test/test-custom-conversions/src/main/java/org.apache.avro.codegentest/CustomDecimal.java index d19d52d46e8..1d4f40c7e57 100644 --- a/lang/java/integration-test/test-custom-conversions/src/main/java/org.apache.avro.codegentest/CustomDecimal.java +++ b/lang/java/integration-test/test-custom-conversions/src/main/java/org.apache.avro.codegentest/CustomDecimal.java @@ -24,7 +24,7 @@ /** * Wraps a BigDecimal just to demonstrate that it is possible to use custom implementation classes with custom conversions. */ -public class CustomDecimal { +public class CustomDecimal implements Comparable { private final BigDecimal internalValue; @@ -57,4 +57,9 @@ public boolean equals(Object o) { public int hashCode() { return internalValue.hashCode(); } + + @Override + public int compareTo(CustomDecimal o) { + return this.internalValue.compareTo(o.internalValue); + } } diff --git a/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java b/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java index 569f4271126..f04f21614b4 100644 --- a/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java +++ b/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java @@ -89,6 +89,7 @@ public Player(java.lang.Integer number, java.lang.String first_name, java.lang.S this.position = position; } + public org.apache.avro.specific.SpecificData getSpecificData() { return MODEL$; } public org.apache.avro.Schema getSchema() { return SCHEMA$; } // Used by DatumWriter. Applications should not call. public java.lang.Object get(int field$) { diff --git a/lang/java/tools/src/test/compiler/output/Player.java b/lang/java/tools/src/test/compiler/output/Player.java index 5bbb3b01850..02fac3881d6 100644 --- a/lang/java/tools/src/test/compiler/output/Player.java +++ b/lang/java/tools/src/test/compiler/output/Player.java @@ -89,6 +89,7 @@ public Player(java.lang.Integer number, java.lang.CharSequence first_name, java. this.position = position; } + public org.apache.avro.specific.SpecificData getSpecificData() { return MODEL$; } public org.apache.avro.Schema getSchema() { return SCHEMA$; } // Used by DatumWriter. Applications should not call. public java.lang.Object get(int field$) { From dccb147c64743a8dd4789c2e3fb3668f03989039 Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Sat, 6 Oct 2018 13:17:41 +0200 Subject: [PATCH 14/18] Added missing copyright statement --- .../AbstractSpecificRecordTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java index 728d1e1d9cd..8a0b2bb796d 100644 --- a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java +++ b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.avro.codegentest; import org.apache.avro.Schema; From 6a06fdeb4778116d38bd6e81fc200d95bbb24b04 Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Sat, 6 Oct 2018 13:19:04 +0200 Subject: [PATCH 15/18] Fixed compile error after rebase --- .../TestRecordWithJsr310LogicalTypes.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lang/java/avro/src/test/java/org/apache/avro/specific/TestRecordWithJsr310LogicalTypes.java b/lang/java/avro/src/test/java/org/apache/avro/specific/TestRecordWithJsr310LogicalTypes.java index c42b3ca0d5c..fd3f9e45d35 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/specific/TestRecordWithJsr310LogicalTypes.java +++ b/lang/java/avro/src/test/java/org/apache/avro/specific/TestRecordWithJsr310LogicalTypes.java @@ -854,16 +854,16 @@ public TestRecordWithJsr310LogicalTypes.Builder clearDec() { public TestRecordWithJsr310LogicalTypes build() { try { TestRecordWithJsr310LogicalTypes record = new TestRecordWithJsr310LogicalTypes(); - record.b = fieldSetFlags()[0] ? this.b : (java.lang.Boolean) defaultValue(fields()[0], record.getConversion(0)); - record.i32 = fieldSetFlags()[1] ? this.i32 : (java.lang.Integer) defaultValue(fields()[1], record.getConversion(1)); - record.i64 = fieldSetFlags()[2] ? this.i64 : (java.lang.Long) defaultValue(fields()[2], record.getConversion(2)); - record.f32 = fieldSetFlags()[3] ? this.f32 : (java.lang.Float) defaultValue(fields()[3], record.getConversion(3)); - record.f64 = fieldSetFlags()[4] ? this.f64 : (java.lang.Double) defaultValue(fields()[4], record.getConversion(4)); - record.s = fieldSetFlags()[5] ? this.s : (java.lang.CharSequence) defaultValue(fields()[5], record.getConversion(5)); - record.d = fieldSetFlags()[6] ? this.d : (java.time.LocalDate) defaultValue(fields()[6], record.getConversion(6)); - record.t = fieldSetFlags()[7] ? this.t : (java.time.LocalTime) defaultValue(fields()[7], record.getConversion(7)); - record.ts = fieldSetFlags()[8] ? this.ts : (java.time.Instant) defaultValue(fields()[8], record.getConversion(8)); - record.dec = fieldSetFlags()[9] ? this.dec : (java.math.BigDecimal) defaultValue(fields()[9], record.getConversion(9)); + record.b = fieldSetFlags()[0] ? this.b : (java.lang.Boolean) defaultValue(fields()[0]); + record.i32 = fieldSetFlags()[1] ? this.i32 : (java.lang.Integer) defaultValue(fields()[1]); + record.i64 = fieldSetFlags()[2] ? this.i64 : (java.lang.Long) defaultValue(fields()[2]); + record.f32 = fieldSetFlags()[3] ? this.f32 : (java.lang.Float) defaultValue(fields()[3]); + record.f64 = fieldSetFlags()[4] ? this.f64 : (java.lang.Double) defaultValue(fields()[4]); + record.s = fieldSetFlags()[5] ? this.s : (java.lang.CharSequence) defaultValue(fields()[5]); + record.d = fieldSetFlags()[6] ? this.d : (java.time.LocalDate) defaultValue(fields()[6]); + record.t = fieldSetFlags()[7] ? this.t : (java.time.LocalTime) defaultValue(fields()[7]); + record.ts = fieldSetFlags()[8] ? this.ts : (java.time.Instant) defaultValue(fields()[8]); + record.dec = fieldSetFlags()[9] ? this.dec : (java.math.BigDecimal) defaultValue(fields()[9]); return record; } catch (java.lang.Exception e) { throw new org.apache.avro.AvroRuntimeException(e); From a2fb8d934c5a0ca5233df399fc871aea0e63ff6e Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Fri, 9 Nov 2018 11:21:30 +0100 Subject: [PATCH 16/18] Fixed problem with logical types in nested records. --- .../compiler/specific/SpecificCompiler.java | 44 +++++++- .../specific/TestSpecificCompiler.java | 106 ++++++++++++++++++ .../AbstractSpecificRecordTest.java | 3 +- .../codegentest/TestNestedLogicalTypes.java | 67 +++++++++++ .../avro/nested_logical_types_array.avsc | 26 +++++ .../avro/nested_logical_types_map.avsc | 26 +++++ .../avro/nested_logical_types_record.avsc | 23 ++++ .../avro/nested_logical_types_union.avsc | 23 ++++ .../avro/nullable_logical_types_array.avsc | 16 +++ 9 files changed, 331 insertions(+), 3 deletions(-) create mode 100644 lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestNestedLogicalTypes.java create mode 100644 lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_array.avsc create mode 100644 lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_map.avsc create mode 100644 lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_record.avsc create mode 100644 lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_union.avsc create mode 100644 lang/java/integration-test/codegen-test/src/test/resources/avro/nullable_logical_types_array.avsc diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java index 4e3250d895a..f854c40ef42 100644 --- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java +++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java @@ -287,8 +287,7 @@ public Collection getUsedConversionClasses(Schema schema) { classnameToConversion.put(conversion.getConvertedType().getCanonicalName(), conversion); } Collection result = new HashSet<>(); - for (Field field : schema.getFields()) { - final String className = javaType(field.schema()); + for (String className : getClassNamesOfPrimitiveFields(schema)) { if (classnameToConversion.containsKey(className)) { result.add(classnameToConversion.get(className).getClass().getCanonicalName()); } @@ -296,6 +295,47 @@ public Collection getUsedConversionClasses(Schema schema) { return result; } + private Set getClassNamesOfPrimitiveFields(Schema schema) { + Set result = new HashSet<>(); + getClassNamesOfPrimitiveFields(schema, result, new HashSet<>()); + return result; + } + + private void getClassNamesOfPrimitiveFields(Schema schema, Set result, Set seenSchemas) { + if (seenSchemas.contains(schema)) { + return; + } + seenSchemas.add(schema); + switch (schema.getType()) { + case RECORD: + for (Schema.Field field : schema.getFields()) { + getClassNamesOfPrimitiveFields(field.schema(), result, seenSchemas); + } + break; + case MAP: + getClassNamesOfPrimitiveFields(schema.getValueType(), result, seenSchemas); + break; + case ARRAY: + getClassNamesOfPrimitiveFields(schema.getElementType(), result, seenSchemas); + break; + case UNION: + for (Schema s : schema.getTypes()) + getClassNamesOfPrimitiveFields(s, result, seenSchemas); + break; + case ENUM: + case FIXED: + case NULL: + break; + case STRING: case BYTES: + case INT: case LONG: + case FLOAT: case DOUBLE: + case BOOLEAN: + result.add(javaType(schema)); + break; + default: throw new RuntimeException("Unknown type: "+schema); + } + } + private static String logChuteName = null; private void initializeVelocity() { diff --git a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java index 65a2865bb62..6be84eeb9e3 100644 --- a/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java +++ b/lang/java/compiler/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java @@ -490,6 +490,42 @@ public void testJavaUnboxJsr310DateTime() throws Exception { "java.time.Instant", compiler.javaUnbox(timestampSchema)); } + @Test + public void testNullableLogicalTypesJavaUnboxDecimalTypesEnabled() throws Exception { + SpecificCompiler compiler = createCompiler(); + compiler.setEnableDecimalLogicalType(true); + + // Nullable types should return boxed types instead of primitive types + Schema nullableDecimalSchema1 = Schema.createUnion( + Schema.create(Schema.Type.NULL), LogicalTypes.decimal(9,2) + .addToSchema(Schema.create(Schema.Type.BYTES))); + Schema nullableDecimalSchema2 = Schema.createUnion( + LogicalTypes.decimal(9,2) + .addToSchema(Schema.create(Schema.Type.BYTES)), Schema.create(Schema.Type.NULL)); + Assert.assertEquals("Should return boxed type", + compiler.javaUnbox(nullableDecimalSchema1), "java.math.BigDecimal"); + Assert.assertEquals("Should return boxed type", + compiler.javaUnbox(nullableDecimalSchema2), "java.math.BigDecimal"); + } + + @Test + public void testNullableLogicalTypesJavaUnboxDecimalTypesDisabled() throws Exception { + SpecificCompiler compiler = createCompiler(); + compiler.setEnableDecimalLogicalType(false); + + // Since logical decimal types are disabled, a ByteBuffer is expected. + Schema nullableDecimalSchema1 = Schema.createUnion( + Schema.create(Schema.Type.NULL), LogicalTypes.decimal(9,2) + .addToSchema(Schema.create(Schema.Type.BYTES))); + Schema nullableDecimalSchema2 = Schema.createUnion( + LogicalTypes.decimal(9,2) + .addToSchema(Schema.create(Schema.Type.BYTES)), Schema.create(Schema.Type.NULL)); + Assert.assertEquals("Should return boxed type", + compiler.javaUnbox(nullableDecimalSchema1), "java.nio.ByteBuffer"); + Assert.assertEquals("Should return boxed type", + compiler.javaUnbox(nullableDecimalSchema2), "java.nio.ByteBuffer"); + } + @Test public void testNullableTypesJavaUnbox() throws Exception { SpecificCompiler compiler = createCompiler(); @@ -542,6 +578,76 @@ public void testNullableTypesJavaUnbox() throws Exception { compiler.javaUnbox(nullableBooleanSchema2), "java.lang.Boolean"); } + @Test + public void testGetUsedConversionClassesForNullableLogicalTypes() throws Exception { + SpecificCompiler compiler = createCompiler(); + compiler.setEnableDecimalLogicalType(true); + + Schema nullableDecimal1 = Schema.createUnion( + Schema.create(Schema.Type.NULL), LogicalTypes.decimal(9,2) + .addToSchema(Schema.create(Schema.Type.BYTES))); + Schema schemaWithNullableDecimal1 = Schema.createRecord("WithNullableDecimal", "", "", false, Collections.singletonList(new Schema.Field("decimal", nullableDecimal1, "", null))); + + final Collection usedConversionClasses = compiler.getUsedConversionClasses(schemaWithNullableDecimal1); + Assert.assertEquals(1, usedConversionClasses.size()); + Assert.assertEquals("org.apache.avro.Conversions.DecimalConversion", usedConversionClasses.iterator().next()); + } + + @Test + public void testGetUsedConversionClassesForNullableLogicalTypesInNestedRecord() throws Exception { + SpecificCompiler compiler = createCompiler(); + + final Schema schema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"NestedLogicalTypesRecord\",\"namespace\":\"org.apache.avro.codegentest.testdata\",\"doc\":\"Test nested types with logical types in generated Java classes\",\"fields\":[{\"name\":\"nestedRecord\",\"type\":{\"type\":\"record\",\"name\":\"NestedRecord\",\"fields\":[{\"name\":\"nullableDateField\",\"type\":[\"null\",{\"type\":\"int\",\"logicalType\":\"date\"}]}]}}]}"); + + final Collection usedConversionClasses = compiler.getUsedConversionClasses(schema); + Assert.assertEquals(1, usedConversionClasses.size()); + Assert.assertEquals("org.apache.avro.data.TimeConversions.DateConversion", usedConversionClasses.iterator().next()); + } + + @Test + public void testGetUsedConversionClassesForNullableLogicalTypesInArray() throws Exception { + SpecificCompiler compiler = createCompiler(); + + final Schema schema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"NullableLogicalTypesArray\",\"namespace\":\"org.apache.avro.codegentest.testdata\",\"doc\":\"Test nested types with logical types in generated Java classes\",\"fields\":[{\"name\":\"arrayOfLogicalType\",\"type\":{\"type\":\"array\",\"items\":[\"null\",{\"type\":\"int\",\"logicalType\":\"date\"}]}}]}"); + + final Collection usedConversionClasses = compiler.getUsedConversionClasses(schema); + Assert.assertEquals(1, usedConversionClasses.size()); + Assert.assertEquals("org.apache.avro.data.TimeConversions.DateConversion", usedConversionClasses.iterator().next()); + } + + @Test + public void testGetUsedConversionClassesForNullableLogicalTypesInArrayOfRecords() throws Exception { + SpecificCompiler compiler = createCompiler(); + + final Schema schema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"NestedLogicalTypesArray\",\"namespace\":\"org.apache.avro.codegentest.testdata\",\"doc\":\"Test nested types with logical types in generated Java classes\",\"fields\":[{\"name\":\"arrayOfRecords\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"RecordInArray\",\"fields\":[{\"name\":\"nullableDateField\",\"type\":[\"null\",{\"type\":\"int\",\"logicalType\":\"date\"}]}]}}}]}"); + + final Collection usedConversionClasses = compiler.getUsedConversionClasses(schema); + Assert.assertEquals(1, usedConversionClasses.size()); + Assert.assertEquals("org.apache.avro.data.TimeConversions.DateConversion", usedConversionClasses.iterator().next()); + } + + @Test + public void testGetUsedConversionClassesForNullableLogicalTypesInUnionOfRecords() throws Exception { + SpecificCompiler compiler = createCompiler(); + + final Schema schema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"NestedLogicalTypesUnion\",\"namespace\":\"org.apache.avro.codegentest.testdata\",\"doc\":\"Test nested types with logical types in generated Java classes\",\"fields\":[{\"name\":\"unionOfRecords\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"RecordInUnion\",\"fields\":[{\"name\":\"nullableDateField\",\"type\":[\"null\",{\"type\":\"int\",\"logicalType\":\"date\"}]}]}]}]}"); + + final Collection usedConversionClasses = compiler.getUsedConversionClasses(schema); + Assert.assertEquals(1, usedConversionClasses.size()); + Assert.assertEquals("org.apache.avro.data.TimeConversions.DateConversion", usedConversionClasses.iterator().next()); + } + + @Test + public void testGetUsedConversionClassesForNullableLogicalTypesInMapOfRecords() throws Exception { + SpecificCompiler compiler = createCompiler(); + + final Schema schema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"NestedLogicalTypesMap\",\"namespace\":\"org.apache.avro.codegentest.testdata\",\"doc\":\"Test nested types with logical types in generated Java classes\",\"fields\":[{\"name\":\"mapOfRecords\",\"type\":{\"type\":\"map\",\"values\":{\"type\":\"record\",\"name\":\"RecordInMap\",\"fields\":[{\"name\":\"nullableDateField\",\"type\":[\"null\",{\"type\":\"int\",\"logicalType\":\"date\"}]}]},\"avro.java.string\":\"String\"}}]}"); + + final Collection usedConversionClasses = compiler.getUsedConversionClasses(schema); + Assert.assertEquals(1, usedConversionClasses.size()); + Assert.assertEquals("org.apache.avro.data.TimeConversions.DateConversion", usedConversionClasses.iterator().next()); + } + @Test public void testLogicalTypesWithMultipleFields() throws Exception { Schema logicalTypesWithMultipleFields = new Schema.Parser().parse( diff --git a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java index 8a0b2bb796d..aed64733c65 100644 --- a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java +++ b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java @@ -37,7 +37,8 @@ void verifySerDeAndStandardMethods(T original) { final T copy = deserialize(serialized, original.getSchema()); Assert.assertEquals(original, copy); // In addition to equals() tested above, make sure the other methods that use SpecificData work as intended - Assert.assertEquals(0, original.compareTo(copy)); + // compareTo() throws an exception for maps, otherwise we would have tested it here + // Assert.assertEquals(0, original.compareTo(copy)); Assert.assertEquals(original.hashCode(), copy.hashCode()); Assert.assertEquals(original.toString(), copy.toString()); } diff --git a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestNestedLogicalTypes.java b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestNestedLogicalTypes.java new file mode 100644 index 00000000000..a33d038f0aa --- /dev/null +++ b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/TestNestedLogicalTypes.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro.codegentest; + +import org.apache.avro.codegentest.testdata.*; +import org.joda.time.LocalDate; +import org.junit.Test; + +import java.util.Collections; + +public class TestNestedLogicalTypes extends AbstractSpecificRecordTest { + + @Test + public void testNullableLogicalTypeInNestedRecord() { + final NestedLogicalTypesRecord nestedLogicalTypesRecord = + NestedLogicalTypesRecord.newBuilder() + .setNestedRecord(NestedRecord.newBuilder() + .setNullableDateField(LocalDate.now()).build()).build(); + verifySerDeAndStandardMethods(nestedLogicalTypesRecord); + } + + @Test + public void testNullableLogicalTypeInArray() { + final NullableLogicalTypesArray logicalTypesArray = + NullableLogicalTypesArray.newBuilder().setArrayOfLogicalType(Collections.singletonList(LocalDate.now())).build(); + verifySerDeAndStandardMethods(logicalTypesArray); + } + + @Test + public void testNullableLogicalTypeInRecordInArray() { + final NestedLogicalTypesArray nestedLogicalTypesArray = + NestedLogicalTypesArray.newBuilder().setArrayOfRecords(Collections.singletonList( + RecordInArray.newBuilder().setNullableDateField(LocalDate.now()).build())).build(); + verifySerDeAndStandardMethods(nestedLogicalTypesArray); + } + + @Test + public void testNullableLogicalTypeInRecordInUnion() { + final NestedLogicalTypesUnion nestedLogicalTypesUnion = + NestedLogicalTypesUnion.newBuilder().setUnionOfRecords( + RecordInUnion.newBuilder().setNullableDateField(LocalDate.now()).build()).build(); + verifySerDeAndStandardMethods(nestedLogicalTypesUnion); + } + + @Test + public void testNullableLogicalTypeInRecordInMap() { + final NestedLogicalTypesMap nestedLogicalTypesMap = + NestedLogicalTypesMap.newBuilder().setMapOfRecords(Collections.singletonMap("key", + RecordInMap.newBuilder().setNullableDateField(LocalDate.now()).build())).build(); + verifySerDeAndStandardMethods(nestedLogicalTypesMap); + } +} diff --git a/lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_array.avsc b/lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_array.avsc new file mode 100644 index 00000000000..c5eba142323 --- /dev/null +++ b/lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_array.avsc @@ -0,0 +1,26 @@ +{"namespace": "org.apache.avro.codegentest.testdata", + "type": "record", + "name": "NestedLogicalTypesArray", + "doc" : "Test nested types with logical types in generated Java classes", + "fields": [ + { + "name": "arrayOfRecords", + "type": { + "type": "array", + "items": { + "namespace": "org.apache.avro.codegentest.testdata", + "name": "RecordInArray", + "type": "record", + "fields": [ + { + "name": "nullableDateField", + "type": ["null", {"type": "int", "logicalType": "date"}] + } + ] + } + } + }] +} + + + diff --git a/lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_map.avsc b/lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_map.avsc new file mode 100644 index 00000000000..f99e457db4a --- /dev/null +++ b/lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_map.avsc @@ -0,0 +1,26 @@ +{"namespace": "org.apache.avro.codegentest.testdata", + "type": "record", + "name": "NestedLogicalTypesMap", + "doc" : "Test nested types with logical types in generated Java classes", + "fields": [ + { + "name": "mapOfRecords", + "type": { + "type": "map", + "values": { + "namespace": "org.apache.avro.codegentest.testdata", + "name": "RecordInMap", + "type": "record", + "fields": [ + { + "name": "nullableDateField", + "type": ["null", {"type": "int", "logicalType": "date"}] + } + ] + } + } + }] +} + + + diff --git a/lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_record.avsc b/lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_record.avsc new file mode 100644 index 00000000000..d51ac86860d --- /dev/null +++ b/lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_record.avsc @@ -0,0 +1,23 @@ +{"namespace": "org.apache.avro.codegentest.testdata", + "type": "record", + "name": "NestedLogicalTypesRecord", + "doc" : "Test nested types with logical types in generated Java classes", + "fields": [ + { + "name": "nestedRecord", + "type": { + "namespace": "org.apache.avro.codegentest.testdata", + "type": "record", + "name": "NestedRecord", + "fields": [ + { + "name": "nullableDateField", + "type": ["null", {"type": "int", "logicalType": "date"}] + } + ] + } + }] +} + + + diff --git a/lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_union.avsc b/lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_union.avsc new file mode 100644 index 00000000000..44a495c4a97 --- /dev/null +++ b/lang/java/integration-test/codegen-test/src/test/resources/avro/nested_logical_types_union.avsc @@ -0,0 +1,23 @@ +{"namespace": "org.apache.avro.codegentest.testdata", + "type": "record", + "name": "NestedLogicalTypesUnion", + "doc" : "Test nested types with logical types in generated Java classes", + "fields": [ + { + "name": "unionOfRecords", + "type": ["null", { + "namespace": "org.apache.avro.codegentest.testdata", + "name": "RecordInUnion", + "type": "record", + "fields": [ + { + "name": "nullableDateField", + "type": ["null", {"type": "int", "logicalType": "date"}] + } + ] + }] + }] +} + + + diff --git a/lang/java/integration-test/codegen-test/src/test/resources/avro/nullable_logical_types_array.avsc b/lang/java/integration-test/codegen-test/src/test/resources/avro/nullable_logical_types_array.avsc new file mode 100644 index 00000000000..8e5caded5f7 --- /dev/null +++ b/lang/java/integration-test/codegen-test/src/test/resources/avro/nullable_logical_types_array.avsc @@ -0,0 +1,16 @@ +{"namespace": "org.apache.avro.codegentest.testdata", + "type": "record", + "name": "NullableLogicalTypesArray", + "doc" : "Test nested types with logical types in generated Java classes", + "fields": [ + { + "name": "arrayOfLogicalType", + "type": { + "type": "array", + "items": ["null", {"type": "int", "logicalType": "date"}] + } + }] +} + + + From 29fc837ca80fe2a7d94bb376d9eae2d8a8fe0c2c Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Mon, 12 Nov 2018 10:49:38 +0100 Subject: [PATCH 17/18] Fixed failing test. --- lang/java/integration-test/codegen-test/pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lang/java/integration-test/codegen-test/pom.xml b/lang/java/integration-test/codegen-test/pom.xml index 6fb9c54eafe..2be8ad77416 100644 --- a/lang/java/integration-test/codegen-test/pom.xml +++ b/lang/java/integration-test/codegen-test/pom.xml @@ -52,7 +52,9 @@ ${project.basedir}/src/test/resources/avro ${project.build.directory}/generated-test-sources/java true - org.apache.avro.codegentest.CustomDecimalConversion + + org.apache.avro.codegentest.CustomDecimalConversion + From 1c9441ac6c1b5ba6bd671b2f2ad21e0def884109 Mon Sep 17 00:00:00 2001 From: Katrin Skoglund Date: Wed, 21 Nov 2018 19:11:53 +0100 Subject: [PATCH 18/18] Fixed serialization problem when creating SpecificDatumReader from a class --- .../avro/specific/SpecificDatumReader.java | 2 +- .../AbstractSpecificRecordTest.java | 63 ++++++++++--------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java index c1226838580..7fc91df3094 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java @@ -33,7 +33,7 @@ public SpecificDatumReader() { /** Construct for reading instances of a class. */ public SpecificDatumReader(Class c) { - this(new SpecificData(c.getClassLoader())); + this(SpecificData.getForClass(c)); setSchema(getSpecificData().getSchema(c)); } diff --git a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java index aed64733c65..9d8a273cc1b 100644 --- a/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java +++ b/lang/java/integration-test/codegen-test/src/test/java/org/apache/avro/codegentest/AbstractSpecificRecordTest.java @@ -18,7 +18,6 @@ package org.apache.avro.codegentest; -import org.apache.avro.Schema; import org.apache.avro.io.DecoderFactory; import org.apache.avro.io.EncoderFactory; import org.apache.avro.specific.SpecificDatumReader; @@ -32,35 +31,43 @@ abstract class AbstractSpecificRecordTest { - void verifySerDeAndStandardMethods(T original) { - final byte[] serialized = serialize(original); - final T copy = deserialize(serialized, original.getSchema()); - Assert.assertEquals(original, copy); - // In addition to equals() tested above, make sure the other methods that use SpecificData work as intended - // compareTo() throws an exception for maps, otherwise we would have tested it here - // Assert.assertEquals(0, original.compareTo(copy)); - Assert.assertEquals(original.hashCode(), copy.hashCode()); - Assert.assertEquals(original.toString(), copy.toString()); - } + @SuppressWarnings("unchecked") + void verifySerDeAndStandardMethods(T original) { + final SpecificDatumWriter datumWriterFromSchema = new SpecificDatumWriter<>(original.getSchema()); + final SpecificDatumReader datumReaderFromSchema = new SpecificDatumReader<>(original.getSchema(), original.getSchema()); + verifySerDeAndStandardMethods(original, datumWriterFromSchema, datumReaderFromSchema); + final SpecificDatumWriter datumWriterFromClass = new SpecificDatumWriter(original.getClass()); + final SpecificDatumReader datumReaderFromClass = new SpecificDatumReader(original.getClass()); + verifySerDeAndStandardMethods(original, datumWriterFromClass, datumReaderFromClass); + } + + private void verifySerDeAndStandardMethods(T original, SpecificDatumWriter datumWriter, SpecificDatumReader datumReader) { + final byte[] serialized = serialize(original, datumWriter); + final T copy = deserialize(serialized, datumReader); + Assert.assertEquals(original, copy); + // In addition to equals() tested above, make sure the other methods that use SpecificData work as intended + // compareTo() throws an exception for maps, otherwise we would have tested it here + // Assert.assertEquals(0, original.compareTo(copy)); + Assert.assertEquals(original.hashCode(), copy.hashCode()); + Assert.assertEquals(original.toString(), copy.toString()); + } - private byte[] serialize(T object) { - SpecificDatumWriter datumWriter = new SpecificDatumWriter<>(object.getSchema()); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try { - datumWriter.write(object, EncoderFactory.get().directBinaryEncoder(outputStream, null)); - return outputStream.toByteArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } + private byte[] serialize(T object, SpecificDatumWriter datumWriter) { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try { + datumWriter.write(object, EncoderFactory.get().directBinaryEncoder(outputStream, null)); + return outputStream.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); } + } - private T deserialize(byte[] bytes, Schema schema) { - SpecificDatumReader datumReader = new SpecificDatumReader<>(schema, schema); - try { - final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); - return datumReader.read(null, DecoderFactory.get().directBinaryDecoder(byteArrayInputStream, null)); - } catch (IOException e) { - throw new RuntimeException(e); - } + private T deserialize(byte[] bytes, SpecificDatumReader datumReader) { + try { + final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + return datumReader.read(null, DecoderFactory.get().directBinaryDecoder(byteArrayInputStream, null)); + } catch (IOException e) { + throw new RuntimeException(e); } + } }