diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 27fde7608eef..3786a0b9ff28 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -43,7 +43,7 @@ dependencies { runtime "net.ltgt.gradle:gradle-apt-plugin:0.20" // Enable a Java annotation processor runtime "com.google.protobuf:protobuf-gradle-plugin:0.8.5" // Enable proto code generation runtime "io.spring.gradle:propdeps-plugin:0.0.9.RELEASE" // Enable provided and optional configurations - runtime "com.commercehub.gradle.plugin:gradle-avro-plugin:0.11.0" // Enable Avro code generation + runtime "com.commercehub.gradle.plugin:gradle-avro-plugin:0.17.0" // Enable Avro code generation runtime "com.diffplug.spotless:spotless-plugin-gradle:3.24.0" // Enable a code formatting plugin runtime "gradle.plugin.com.github.blindpirate:gogradle:0.11.4" // Enable Go code compilation runtime "gradle.plugin.com.palantir.gradle.docker:gradle-docker:0.20.1" // Enable building Docker containers diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index b7e9d2a1b398..abf4009d10b7 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -408,8 +408,8 @@ class BeamModulePlugin implements Plugin { apex_common : "org.apache.apex:apex-common:$apex_core_version", apex_engine : "org.apache.apex:apex-engine:$apex_core_version", args4j : "args4j:args4j:2.33", - avro : "org.apache.avro:avro:1.8.2", - avro_tests : "org.apache.avro:avro:1.8.2:tests", + avro : "org.apache.avro:avro:1.9.1", + avro_tests : "org.apache.avro:avro:1.9.1:tests", aws_java_sdk_cloudwatch : "com.amazonaws:aws-java-sdk-cloudwatch:$aws_java_sdk_version", aws_java_sdk_core : "com.amazonaws:aws-java-sdk-core:$aws_java_sdk_version", aws_java_sdk_dynamodb : "com.amazonaws:aws-java-sdk-dynamodb:$aws_java_sdk_version", diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java index b044165a703e..2c40389cb901 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/AvroCoder.java @@ -34,7 +34,7 @@ import javax.annotation.Nullable; import org.apache.avro.AvroRuntimeException; import org.apache.avro.Schema; -import org.apache.avro.data.TimeConversions.TimestampConversion; +import org.apache.avro.data.JodaTimeConversions; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.generic.GenericRecord; @@ -238,7 +238,7 @@ private SerializableReflectDataSupplier(Class clazz) { @Override public ReflectData get() { ReflectData reflectData = new ReflectData(clazz.getClassLoader()); - reflectData.addLogicalTypeConversion(new TimestampConversion()); + reflectData.addLogicalTypeConversion(new JodaTimeConversions.TimestampConversion()); return reflectData; } } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroUtils.java index 7b43961437c4..fcef9ef3b95d 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroUtils.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/utils/AvroUtils.java @@ -34,7 +34,7 @@ import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema.Type; -import org.apache.avro.data.TimeConversions; +import org.apache.avro.data.JodaTimeConversions; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericFixed; import org.apache.avro.generic.GenericRecord; @@ -75,8 +75,8 @@ public class AvroUtils { static { // This works around a bug in the Avro library (AVRO-1891) around SpecificRecord's handling // of DateTime types. - SpecificData.get().addLogicalTypeConversion(new TimeConversions.TimestampConversion()); - GenericData.get().addLogicalTypeConversion(new TimeConversions.TimestampConversion()); + SpecificData.get().addLogicalTypeConversion(new JodaTimeConversions.TimestampConversion()); + GenericData.get().addLogicalTypeConversion(new JodaTimeConversions.TimestampConversion()); } // Unwrap an AVRO schema into the base type an whether it is nullable. diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/AvroSchemaTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/AvroSchemaTest.java index f10733237fa1..82d5cfaa1d9e 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/AvroSchemaTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/AvroSchemaTest.java @@ -20,6 +20,8 @@ import static org.junit.Assert.assertEquals; import java.nio.ByteBuffer; +import java.time.Month; +import java.time.ZoneId; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -259,8 +261,9 @@ public AvroPojo() {} private static final byte[] BYTE_ARRAY = new byte[] {1, 2, 3, 4}; private static final DateTime DATE_TIME = new DateTime().withDate(1979, 3, 14).withTime(1, 2, 3, 4); - private static final LocalDate DATE = new LocalDate(1979, 3, 14); + private static final java.time.LocalDate DATE = java.time.LocalDate.of(1979, Month.March, 14); private static final TestAvroNested AVRO_NESTED_SPECIFIC_RECORD = new TestAvroNested(true, 42); + private static final TestAvro AVRO_SPECIFIC_RECORD = new TestAvro( true, @@ -296,7 +299,8 @@ public AvroPojo() {} GenericData.get() .createFixed( null, BYTE_ARRAY, org.apache.avro.Schema.createFixed("fixed4", "", "", 4))) - .set("date", (int) Days.daysBetween(new LocalDate(1970, 1, 1), DATE).getDays()) + .set("date", (int) java.time.temporal.ChronoUnit.DAYS.daysBetween( + java.time.LocalDate.of(1970, Month.January, 1), DATE)) .set("timestampMillis", DATE_TIME.getMillis()) .set("testEnum", TestEnum.abc) .set("row", AVRO_NESTED_GENERIC_RECORD) @@ -395,7 +399,7 @@ public void testRowToGenericRecord() { "mystring", ByteBuffer.wrap(BYTE_ARRAY), BYTE_ARRAY, - DATE.toDateTimeAtStartOfDay(DateTimeZone.UTC), + DATE.atStartOfDay().atZone(ZoneId.of("UTC")), DATE_TIME, "abc", NESTED_ROW, diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/AvroUtilsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/AvroUtilsTest.java index cedeb77ee938..32b0e0de251b 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/AvroUtilsTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/utils/AvroUtilsTest.java @@ -34,11 +34,11 @@ import org.apache.avro.Conversions; import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; -import org.apache.avro.RandomData; import org.apache.avro.Schema.Type; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecordBuilder; import org.apache.avro.reflect.ReflectData; +import org.apache.avro.util.RandomData; import org.apache.avro.util.Utf8; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.coders.AvroCoder;