Skip to content

AVRO-1891: Fix specific nested logical types #118

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
rdblue wants to merge 3 commits into from
Closed

AVRO-1891: Fix specific nested logical types #118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6c04cb3
AVRO-1908: Fix TestSpecificCompiler reference to private method.
rdblue Sep 4, 2016
31be7fa
AVRO-1891: Add conversion getters to datum readers.
rdblue Sep 5, 2016
cf5bff3
AVRO-1891: Java: Add converions to compiled specific records.
rdblue Sep 5, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,14 +153,23 @@ protected Object read(Object old, Schema expected,
Object datum = readWithoutConversion(old, expected, in);
LogicalType logicalType = expected.getLogicalType();
if (logicalType != null) {
Conversion<?> conversion = getData().getConversionFor(logicalType);
Conversion<?> conversion = getConversionFor(logicalType);
if (conversion != null) {
return convert(datum, expected, logicalType, conversion);
}
}
return datum;
}

protected Conversion<?> getConversionFor(LogicalType logicalType) {
return getData().getConversionFor(logicalType);
}

protected Conversion<?> getConversionByClass(Class<?> type,
LogicalType logicalType) {
return getData().getConversionByClass(type, logicalType);
}

protected Object readWithConversion(Object old, Schema expected,
LogicalType logicalType,
Conversion<?> conversion,
Expand Down Expand Up @@ -253,7 +262,7 @@ protected Object readArray(Object old, Schema expected,
long base = 0;
if (l > 0) {
LogicalType logicalType = expectedType.getLogicalType();
Conversion<?> conversion = getData().getConversionFor(logicalType);
Conversion<?> conversion = getConversionFor(logicalType);
Object array = newArray(old, (int) l, expected);
do {
if (logicalType != null && conversion != null) {
Expand Down Expand Up @@ -299,7 +308,7 @@ protected Object readMap(Object old, Schema expected,
Schema eValue = expected.getValueType();
long l = in.readMapStart();
LogicalType logicalType = eValue.getLogicalType();
Conversion<?> conversion = getData().getConversionFor(logicalType);
Conversion<?> conversion = getConversionFor(logicalType);
Object map = newMap(old, (int) l);
if (l > 0) {
do {
Expand Down
11 changes: 6 additions & 5 deletions lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,8 @@ protected Object newArray(Object old, int size, Schema schema) {
if (elementClass == null) {
// see if the element class will be converted and use that class
// logical types cannot conflict with java-element-class
Conversion<?> elementConversion = getData()
.getConversionFor(schema.getElementType().getLogicalType());
Conversion<?> elementConversion = getConversionFor(
schema.getElementType().getLogicalType());
if (elementConversion != null) {
elementClass = elementConversion.getConvertedType();
}
Expand Down Expand Up @@ -176,7 +176,8 @@ private Object readPrimitiveArray(Object array, Class<?> c, long l, ResolvingDec
private Object readObjectArray(Object[] array, Schema expectedType, long l,
ResolvingDecoder in) throws IOException {
LogicalType logicalType = expectedType.getLogicalType();
Conversion<?> conversion = getData().getConversionFor(logicalType);
Conversion<?> conversion = getConversionByClass(
array.getClass().getComponentType(), logicalType);
int index = 0;
if (logicalType != null && conversion != null) {
do {
Expand Down Expand Up @@ -204,7 +205,7 @@ private Object readObjectArray(Object[] array, Schema expectedType, long l,
private Object readCollection(Collection<Object> c, Schema expectedType,
long l, ResolvingDecoder in) throws IOException {
LogicalType logicalType = expectedType.getLogicalType();
Conversion<?> conversion = getData().getConversionFor(logicalType);
Conversion<?> conversion = getConversionFor(logicalType);
if (logicalType != null && conversion != null) {
do {
for (int i = 0; i < l; i++) {
Expand Down Expand Up @@ -285,7 +286,7 @@ protected void readField(Object record, Field f, Object oldDatum,
}
LogicalType logicalType = f.schema().getLogicalType();
if (logicalType != null) {
Conversion<?> conversion = getData().getConversionByClass(
Conversion<?> conversion = getConversionByClass(
accessor.getField().getType(), logicalType);
if (conversion != null) {
try {
Expand Down
77 changes: 77 additions & 0 deletions lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,27 @@
package org.apache.avro.specific;

import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.LinkedHashMap;
import java.nio.ByteBuffer;
import java.lang.reflect.Constructor;
import java.lang.reflect.ParameterizedType;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.concurrent.ExecutionException;

import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import org.apache.avro.Conversion;
import org.apache.avro.Schema;
import org.apache.avro.Protocol;
import org.apache.avro.AvroRuntimeException;
Expand Down Expand Up @@ -386,4 +391,76 @@ public static BinaryEncoder getEncoder(ObjectOutput out) {
.directBinaryEncoder(new ExternalizableOutput(out), null);
}

/**
* Returns the conversions used to compile a specific record.
*
* @return a Map from logical type name to Conversion
*/
Map<String, Conversion<?>> getConversionMap(final Schema schema) {
try {
// This uses get(Schema, Callable) so that the buildConversionMap method
// can be an instance method of this SpecificData instance. Otherwise, it
// would not use the correct ClassLoader to load the specific class for
// the schema.
return conversionMapCache.get(schema,
new Callable<Map<String, Conversion<?>>>() {
@Override
public Map<String, Conversion<?>> call() {
return buildConversionMap(schema);
}
});
} catch (ExecutionException e) {
throw new AvroRuntimeException(
"Could not get conversions needed for schema: " + schema);
}
}

private static Cache<Schema, Map<String, Conversion<?>>> conversionMapCache =
CacheBuilder.<Schema, Map<String, Conversion<?>>>newBuilder()
.weakKeys().build();

@SuppressWarnings("unchecked")
private Map<String, Conversion<?>> buildConversionMap(Schema schema) {
Class<?> specificClass = getClass(schema);

Map<String, Conversion<?>> conversionMap = null;
if (specificClass != null &&
SpecificRecordBase.class.isAssignableFrom(specificClass)) {
try {
Collection<Conversion<?>> conversions = (Collection<Conversion<?>>)
specificClass.getDeclaredField("CONVERSIONS").get(null);
if (conversions != null) {
conversionMap = new HashMap<String, Conversion<?>>();
for (Conversion<?> conversion : conversions) {
conversionMap.put(conversion.getLogicalTypeName(), conversion);
}
}
} catch (IllegalAccessException e) {
throw new AvroRuntimeException(
"Cannot access conversion map for specific class: " +
specificClass.getName());
} catch (NoSuchFieldException e) {
// the conversion map will be built from getConversion results below
}

if (conversionMap == null) {
SpecificRecordBase emptyRecord =
(SpecificRecordBase) newRecord(null, schema);
// attempt to construct the conversions map from the record itself
conversionMap = new HashMap<String, Conversion<?>>();
for (Schema.Field field : emptyRecord.getSchema().getFields()) {
Conversion<?> conversion = emptyRecord.getConversion(field.pos());

@yibing-shi yibing-shi Sep 6, 2016

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume this section is to add support for the legacy records that have a conversion for each field. Problem is that the getConversion method in generated classes doesn't work properly for complex data structures such as array, map and union. It doesn't work for nested records either. Do we want to support them even if it only partially works?

@rdblue rdblue Sep 6, 2016

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually, there's a better way we can handle this. I didn't realize it at the time, but when I added the CONVERSIONS field I found that the DATE_CONVERSION, TIME_CONVERSION, etc. are set on classes that were compiled with date/time types. So we can check for those and add them to the set if they are present. That should cover all existing classes.

@yibing-shi yibing-shi Sep 7, 2016

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a good idea. But we may need to SpecificRecordCompiler to make sure the proper conversion is generated for complex types. The trouble here is union type. In theory, an union field can use multiple conversions, and the current field conversion array may not be able to hold them.

if (conversion != null) {
conversionMap.put(conversion.getLogicalTypeName(), conversion);
}
}
}
}

if (conversionMap == null) {
conversionMap = new HashMap<String, Conversion<?>>();
}

return conversionMap;
}
}
53 changes: 53 additions & 0 deletions lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,15 @@
package org.apache.avro.specific;

import org.apache.avro.Conversion;
import org.apache.avro.LogicalType;
import org.apache.avro.Schema;
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.io.ResolvingDecoder;
import org.apache.avro.util.ClassUtils;
import java.io.IOException;
import java.util.LinkedList;
import java.util.Map;

/** {@link org.apache.avro.io.DatumReader DatumReader} for generated Java classes. */
public class SpecificDatumReader<T> extends GenericDatumReader<T> {
Expand Down Expand Up @@ -62,6 +65,42 @@ public SpecificDatumReader(SpecificData data) {
/** Return the contained {@link SpecificData}. */
public SpecificData getSpecificData() { return (SpecificData)getData(); }

private final ThreadLocal<LinkedList<Map<String, Conversion<?>>>> conversions
= new ThreadLocal<LinkedList<Map<String, Conversion<?>>>>() {
@Override
protected LinkedList<Map<String, Conversion<?>>> initialValue() {
return new LinkedList<Map<String, Conversion<?>>>();
}
};

protected Conversion<?> getConversionFor(LogicalType logicalType) {
if (logicalType == null) {
return null;
}
Conversion<?> conversion = conversions.get().peekLast()

@yibing-shi yibing-shi Sep 6, 2016

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

peekLast() could return null. We need to check it to avoid NPE.

.get(logicalType.getName());
if (conversion == null) {
conversion = super.getConversionFor(logicalType);
}
return conversion;
}

protected Conversion<?> getConversionByClass(Class<?> type,
LogicalType logicalType) {
if (logicalType == null) {
return null;
}
// Check for logical types conversions used by the specific compiler. No
// need to use the class because the compiler always produces code with the
// same class if it used a conversion.
Conversion<?> conversion = conversions.get().peekLast()

@yibing-shi yibing-shi Sep 6, 2016

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Need check whether peekLast returns null

@rdblue rdblue Sep 6, 2016

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch. Thanks!

.get(logicalType.getName());
if (conversion == null) {
conversion = super.getConversionByClass(type, logicalType);
}
return conversion;
}

@Override
public void setSchema(Schema actual) {
// if expected is unset and actual is a specific record,
Expand Down Expand Up @@ -122,5 +161,19 @@ protected void readField(Object r, Schema.Field f, Object oldDatum,
super.readField(r, f, oldDatum, in, state);
}
}

@Override
protected Object readRecord(Object old, final Schema expected,
ResolvingDecoder in)
throws IOException {
// Update the current set of conversions for the record class
Map<String, Conversion<?>> conversionMap =
getSpecificData().getConversionMap(expected);
conversions.get().addLast(conversionMap);
Object record = super.readRecord(old, expected, in);
conversions.get().removeLast();
return record;
}

}

12 changes: 12 additions & 0 deletions lang/java/avro/src/test/java/org/apache/avro/specific/TestRecordWithLogicalTypes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@
*/
package org.apache.avro.specific;

import org.apache.avro.Conversion;
import org.apache.avro.message.BinaryMessageDecoder;
import org.apache.avro.message.BinaryMessageEncoder;

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collection;

@SuppressWarnings("all")
@org.apache.avro.specific.AvroGenerated
Expand Down Expand Up @@ -268,6 +271,15 @@ public void setTs(org.joda.time.DateTime value) {
protected static final org.apache.avro.data.TimeConversions.TimeConversion TIME_CONVERSION = new org.apache.avro.data.TimeConversions.TimeConversion();
protected static final org.apache.avro.data.TimeConversions.TimestampConversion TIMESTAMP_CONVERSION = new org.apache.avro.data.TimeConversions.TimestampConversion();
protected static final org.apache.avro.Conversions.DecimalConversion DECIMAL_CONVERSION = new org.apache.avro.Conversions.DecimalConversion();

public static final Collection<Conversion<?>> CONVERSIONS = new ArrayList<Conversion<?>>();
static {
CONVERSIONS.add(DATE_CONVERSION);
CONVERSIONS.add(TIME_CONVERSION);
CONVERSIONS.add(TIMESTAMP_CONVERSION);
CONVERSIONS.add(DECIMAL_CONVERSION);
}

private final org.apache.avro.Conversion<?>[] conversions =
new org.apache.avro.Conversion<?>[] {
null,
Expand Down
9 changes: 8 additions & 1 deletion lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,13 @@ public void setEnableDecimalLogicalType(boolean enableDecimalLogicalType) {
this.enableDecimalLogicalType = enableDecimalLogicalType;
}

/**
* @return true if decimal logical type conversion is enabled
*/
public boolean isDecimalEnabled() {
return enableDecimalLogicalType;
}

private static String logChuteName = null;

private void initializeVelocity() {
Expand Down Expand Up @@ -406,7 +413,7 @@ OutputFile compileInterface(Protocol protocol) {
return outputFile;
}

private String makePath(String name, String space) {
String makePath(String name, String space) {
if (space == null || space.isEmpty()) {
return name + suffix;
} else {
Expand Down
12 changes: 12 additions & 0 deletions ...iler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,19 @@ public class ${this.mangle($schema.getName())}#if ($schema.isError()) extends or
protected static final org.apache.avro.data.TimeConversions.DateConversion DATE_CONVERSION = new org.apache.avro.data.TimeConversions.DateConversion();
protected static final org.apache.avro.data.TimeConversions.TimeConversion TIME_CONVERSION = new org.apache.avro.data.TimeConversions.TimeConversion();
protected static final org.apache.avro.data.TimeConversions.TimestampConversion TIMESTAMP_CONVERSION = new org.apache.avro.data.TimeConversions.TimestampConversion();
#if ($this.isDecimalEnabled())
protected static final org.apache.avro.Conversions.DecimalConversion DECIMAL_CONVERSION = new org.apache.avro.Conversions.DecimalConversion();
#end

public static final Collection<Conversion<?>> CONVERSIONS = new ArrayList<Conversion<?>>();

@nandorKollar nandorKollar Aug 15, 2017

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are missing imports from the template:

import java.util.ArrayList;
import java.util.Collection;
import org.apache.avro.Conversion;

What is CONVERSIONS used for? Is this needed?

static {
CONVERSIONS.add(DATE_CONVERSION);
CONVERSIONS.add(TIME_CONVERSION);
CONVERSIONS.add(TIMESTAMP_CONVERSION);
#if ($this.isDecimalEnabled())
CONVERSIONS.add(DECIMAL_CONVERSION);
#end
}

private static final org.apache.avro.Conversion<?>[] conversions =
new org.apache.avro.Conversion<?>[] {
Expand Down
5 changes: 3 additions & 2 deletions lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,9 @@ public void testEsc() {

@Test
public void testMakePath() {
assertEquals("foo/bar/Baz.java".replace("/", File.separator), SpecificCompiler.makePath("Baz", "foo.bar"));
assertEquals("baz.java", SpecificCompiler.makePath("baz", ""));
SpecificCompiler compiler = new SpecificCompiler();
assertEquals("foo/bar/Baz.java".replace("/", File.separator), compiler.makePath("Baz", "foo.bar"));
assertEquals("baz.java", compiler.makePath("baz", ""));
}

@Test
Expand Down