From 3749644e991db78331a975ad757aba8eaf2c0e89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Sat, 2 Mar 2019 09:08:11 +0800 Subject: [PATCH 01/35] define default schema unable null --- .../pulsar/client/impl/schema/AvroSchema.java | 2 +- .../client/impl/schema/AvroSchemaTest.java | 24 ++++++++++----- .../impl/schema/KeyValueSchemaTest.java | 1 + .../client/impl/schema/SchemaBuilderTest.java | 7 +++++ .../client/impl/schema/SchemaTestUtils.java | 29 ++++++++++++++----- 5 files changed, 46 insertions(+), 17 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index c7e4a93aa1399..1381ad840e2d6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -100,7 +100,7 @@ public SchemaInfo getSchemaInfo() { } private static org.apache.avro.Schema createAvroSchema(Class pojo) { - return ReflectData.AllowNull.get().getSchema(pojo); + return ReflectData.get().getSchema(pojo); } public static AvroSchema of(Class pojo) { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index e00e1a653cbaa..d239f60dad397 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -19,11 +19,12 @@ package org.apache.pulsar.client.impl.schema; import static org.apache.pulsar.client.impl.schema.SchemaTestUtils.FOO_FIELDS; -import static org.apache.pulsar.client.impl.schema.SchemaTestUtils.SCHEMA_JSON; +import static org.apache.pulsar.client.impl.schema.SchemaTestUtils.SCHEMA_AVRO; import static org.testng.Assert.assertEquals; import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; +import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Bar; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Foo; import org.apache.pulsar.common.schema.SchemaType; @@ -39,7 +40,7 @@ public void testSchema() { assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); - assertEquals(schemaJson, SCHEMA_JSON); + assertEquals(schemaJson, SCHEMA_AVRO); Schema schema = parser.parse(schemaJson); for (String fieldName : FOO_FIELDS) { @@ -49,6 +50,9 @@ public void testSchema() { if (field.name().equals("field4")) { Assert.assertNotNull(field.schema().getTypes().get(1).getField("field1")); } + if (field.name().equals("fieldUnableNull")) { + Assert.assertNotNull(field.schema().getType()); + } } } @@ -60,22 +64,26 @@ public void testEncodeAndDecode() { foo1.setField1("foo1"); foo1.setField2("bar1"); foo1.setField4(new Bar()); + foo1.setFieldUnableNull("notNull"); Foo foo2 = new Foo(); foo2.setField1("foo2"); foo2.setField2("bar2"); byte[] bytes1 = avroSchema.encode(foo1); + Foo object1 = avroSchema.decode(bytes1); Assert.assertTrue(bytes1.length > 0); + assertEquals(object1, foo1); - byte[] bytes2 = avroSchema.encode(foo2); - Assert.assertTrue(bytes2.length > 0); + try { - Foo object1 = avroSchema.decode(bytes1); - Foo object2 = avroSchema.decode(bytes2); + avroSchema.encode(foo2); + + } catch (Exception e) { + Assert.assertTrue(e instanceof SchemaSerializationException); + } - assertEquals(object1, foo1); - assertEquals(object2, foo2); } + } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java index d355fc7cf5c15..6b3cc5b84319c 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java @@ -133,6 +133,7 @@ public void testBytesSchemaEncodeAndDecode() { foo.setField3(3); foo.setField4(bar); foo.setColor(Color.RED); + foo.setFieldUnableNull("notNull"); byte[] fooBytes = fooAvroSchema.encode(foo); byte[] barBytes = barAvroSchema.encode(bar); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaBuilderTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaBuilderTest.java index eb11bc8b63ab3..20d3edca981a8 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaBuilderTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaBuilderTest.java @@ -21,6 +21,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.testng.Assert.assertEquals; +import org.apache.avro.reflect.Nullable; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; import org.apache.pulsar.client.api.schema.SchemaBuilder; @@ -34,11 +35,17 @@ public class SchemaBuilderTest { private static class AllOptionalFields { + @Nullable private Integer intField; + @Nullable private Long longField; + @Nullable private String stringField; + @Nullable private Boolean boolField; + @Nullable private Float floatField; + @Nullable private Double doubleField; } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java index 0081f0e457ec1..363e1bedb46d5 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java @@ -23,6 +23,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; +import org.apache.avro.reflect.AvroDefault; +import org.apache.avro.reflect.Nullable; /** * Utils for testing avro. @@ -33,11 +35,16 @@ public class SchemaTestUtils { @ToString @EqualsAndHashCode public static class Foo { + @Nullable private String field1; + @Nullable private String field2; private int field3; + @Nullable private Bar field4; + @Nullable private Color color; + private String fieldUnableNull; } @Data @@ -87,21 +94,27 @@ public static class DerivedDerivedFoo extends DerivedFoo { private Foo foo2; } + public static final String SCHEMA_AVRO + = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\"," + + "\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field2\",\"type\":[\"null\"," + + "\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\"," + + "\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\"," + + "{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\",\"type\":\"string\"}]}"; + public static final String SCHEMA_JSON - = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema" + - ".SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"],\"default\":null}," + - "{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\"," + - "\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"Bar\"," + - "\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\"," + - "\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\",\"BLUE\"]}]," + - "\"default\":null}]}"; + = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\"" + + ",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\"" + + ":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default" + + "\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\"," + + "\"type\":[\"null\",\"string\"],\"default\":null}]}"; public static String[] FOO_FIELDS = { "field1", "field2", "field3", "field4", - "color" + "color", + "fieldUnableNull" }; } From 8c06348eece7aaf02adaeb968a51eaff59f95ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Tue, 5 Mar 2019 10:36:54 +0800 Subject: [PATCH 02/35] Modify the Schema default unableNull test --- .../client/impl/schema/SchemaTestUtils.java | 19 +++++++++---------- .../schema/generic/GenericSchemaImplTest.java | 4 +++- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java index 363e1bedb46d5..dc676127ea7dd 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java @@ -44,6 +44,7 @@ public static class Foo { private Bar field4; @Nullable private Color color; + @AvroDefault("\"defaultValue\"") private String fieldUnableNull; } @@ -95,18 +96,16 @@ public static class DerivedDerivedFoo extends DerivedFoo { } public static final String SCHEMA_AVRO - = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\"," + - "\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field2\",\"type\":[\"null\"," + - "\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\"," + - "\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\"," + - "{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\",\"type\":\"string\"}]}"; + = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"]," + + "\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":" + + "\"record\",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\"," + + "\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\",\"type\":\"string\",\"default\":\"defaultValue\"}]}"; public static final String SCHEMA_JSON - = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\"" + - ",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\"" + - ":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default" + - "\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\"," + - "\"type\":[\"null\",\"string\"],\"default\":null}]}"; + = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"]," + + "\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\"" + + ",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\"," + + "\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\",\"type\":[\"null\",\"string\"],\"default\":\"defaultValue\"}]}"; public static String[] FOO_FIELDS = { "field1", diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericSchemaImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericSchemaImplTest.java index fc65d70859624..c7b1e1efed66f 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericSchemaImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericSchemaImplTest.java @@ -77,7 +77,7 @@ public void testEncodeAndDecodeGenericRecord(Schema encodeSchema, Bar bar = new Bar(); bar.setField1(i % 2 == 0); foo.setField4(bar); - + foo.setFieldUnableNull("fieldUnableNull-1-"+i); byte[] data = encodeSchema.encode(foo); log.info("Decoding : {}", new String(data, UTF_8)); @@ -93,6 +93,8 @@ public void testEncodeAndDecodeGenericRecord(Schema encodeSchema, assertTrue(field4 instanceof GenericRecord); GenericRecord field4Record = (GenericRecord) field4; assertEquals(i % 2 == 0, field4Record.getField("field1")); + Object fieldUnableNull = record.getField("fieldUnableNull"); + assertEquals("fieldUnableNull-1-" + i, fieldUnableNull, "fieldUnableNull 1 is " + fieldUnableNull.getClass()); } } From 0c68e9fce0851dc2e3d953d199d78956de13fe8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 6 Mar 2019 11:00:13 +0800 Subject: [PATCH 03/35] Modify the Schema type not null --- .../org/apache/pulsar/client/api/Schema.java | 84 ++++++--- .../internal/DefaultImplementation.java | 21 +-- .../pulsar/client/impl/schema/AvroSchema.java | 28 ++- .../pulsar/client/impl/schema/JSONSchema.java | 14 +- .../client/impl/schema/KeyValueSchema.java | 11 +- .../client/impl/schema/AvroSchemaTest.java | 64 ++++++- .../client/impl/schema/JSONSchemaTest.java | 164 ++++++++++++++++-- .../impl/schema/KeyValueSchemaTest.java | 132 ++++++++++++-- .../client/impl/schema/SchemaTestUtils.java | 22 ++- .../SampleAsyncProducerWithSchema.java | 2 +- .../tutorial/SampleConsumerWithSchema.java | 2 +- 11 files changed, 454 insertions(+), 90 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java index 58b707e051eee..a2c9ca72abff6 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,6 +19,7 @@ package org.apache.pulsar.client.api; import java.nio.ByteBuffer; + import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.GenericSchema; import org.apache.pulsar.client.internal.DefaultImplementation; @@ -50,11 +51,9 @@ default void validate(byte[] message) { /** * Encode an object representing the message content into a byte array. * - * @param message - * the message object + * @param message the message object * @return a byte array with the serialized content - * @throws SchemaSerializationException - * if the serialization fails + * @throws SchemaSerializationException if the serialization fails */ byte[] encode(T message); @@ -78,8 +77,7 @@ default boolean supportSchemaVersioning() { /** * Decode a byte array into an object using the schema definition and deserializer implementation * - * @param bytes - * the byte array to decode + * @param bytes the byte array to decode * @return the deserialized object */ default T decode(byte[] bytes) { @@ -90,10 +88,8 @@ default T decode(byte[] bytes) { /** * Decode a byte array into an object using a given version. * - * @param bytes - * the byte array to decode - * @param schemaVersion - * the schema version to decode the object. null indicates using latest version. + * @param bytes the byte array to decode + * @param schemaVersion the schema version to decode the object. null indicates using latest version. * @return the deserialized object */ default T decode(byte[] bytes, byte[] schemaVersion) { @@ -167,49 +163,95 @@ static Schema PROTOBUF(Cla } /** - * Create a Avro schema type by extracting the fields of the specified class. + * Create a allow null Avro schema type by extracting the fields of the specified class. * * @param clazz the POJO class to be used to extract the Avro schema * @return a Schema instance */ static Schema AVRO(Class clazz) { - return DefaultImplementation.newAvroSchema(clazz); + return DefaultImplementation.newAvroSchema(clazz, true); + } + + /** + * Create a Avro schema type by extracting the fields of the specified class. + * + * @param clazz the POJO class to be used to extract the Avro schema + * allowNull the create a allow null or not null Avro schema + * @return a Schema instance + */ + static Schema AVRO(Class clazz, Boolean allowNull) { + return DefaultImplementation.newAvroSchema(clazz, allowNull); } + /** - * Create a JSON schema type by extracting the fields of the specified class. + * Create a allow null JSON schema type by extracting the fields of the specified class. * * @param clazz the POJO class to be used to extract the JSON schema * @return a Schema instance */ static Schema JSON(Class clazz) { - return DefaultImplementation.newJSONSchema(clazz); + return DefaultImplementation.newJSONSchema(clazz, true); } + /** + * Create a allow null JSON schema type by extracting the fields of the specified class. + * + * @param clazz the POJO class to be used to extract the JSON schema + * allowNull the create a allow null or not null JSON schema + * @return a Schema instance + */ + static Schema JSON(Class clazz, Boolean allowNull) { + return DefaultImplementation.newJSONSchema(clazz, allowNull); + } + + /** * Key Value Schema using passed in schema type, support JSON and AVRO currently. */ static Schema> KeyValue(Class key, Class value, SchemaType type) { - return DefaultImplementation.newKeyValueSchema(key, value, type); + return DefaultImplementation.newKeyValueSchema(key, value, type,true); } + /** + * Key Value Schema using passed in schema type, support JSON and AVRO currently. + */ + static Schema> KeyValue(Class key, Class value, SchemaType type, Boolean allowNull) { + return DefaultImplementation.newKeyValueSchema(key, value, type, allowNull); + } + + /** * Schema that can be used to encode/decode KeyValue. */ - Schema> KV_BYTES = DefaultImplementation.newKeyValueSchema(BYTES, BYTES); + Schema> KV_BYTES = DefaultImplementation.newKeyValueSchema(BYTES, BYTES, true); /** * Key Value Schema whose underneath key and value schemas are JSONSchema. */ static Schema> KeyValue(Class key, Class value) { - return DefaultImplementation.newKeyValueSchema(key, value, SchemaType.JSON); + return DefaultImplementation.newKeyValueSchema(key, value, SchemaType.JSON,true); } /** * Key Value Schema using passed in key and value schemas. */ static Schema> KeyValue(Schema key, Schema value) { - return DefaultImplementation.newKeyValueSchema(key, value); + return DefaultImplementation.newKeyValueSchema(key, value, true); + } + + /** + * Key Value Schema whose underneath key and value schemas are JSONSchema. + */ + static Schema> KeyValue(Class key, Class value, Boolean allowNull) { + return DefaultImplementation.newKeyValueSchema(key, value, SchemaType.JSON ,allowNull); + } + + /** + * Key Value Schema using passed in key and value schemas. + */ + static Schema> KeyValue(Schema key, Schema value, Boolean allowNull) { + return DefaultImplementation.newKeyValueSchema(key, value, allowNull); } @Deprecated diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java index a4f4cfdbf0dbd..a581583dcc541 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java @@ -182,10 +182,10 @@ public static Schema newDoubleSchema() { .newInstance()); } - public static Schema newAvroSchema(Class clazz) { + public static Schema newAvroSchema(Class clazz,Boolean allowNull) { return catchExceptions( - () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.AvroSchema", "of", Class.class) - .invoke(null, clazz)); + () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.AvroSchema", "of", Class.class,Boolean.class) + .invoke(null, clazz,allowNull)); } public static Schema newProtobufSchema(Class clazz) { @@ -194,10 +194,10 @@ public static Schema newPr .invoke(null, clazz)); } - public static Schema newJSONSchema(Class clazz) { + public static Schema newJSONSchema(Class clazz,Boolean allowNull) { return catchExceptions( - () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.JSONSchema", "of", Class.class) - .invoke(null, clazz)); + () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.JSONSchema", "of", Class.class,Boolean.class) + .invoke(null, clazz,allowNull)); } public static Schema newAutoConsumeSchema() { @@ -212,18 +212,19 @@ public static Schema newAutoProduceSchema() { .newInstance()); } - public static Schema> newKeyValueSchema(Schema keySchema, Schema valueSchema) { + public static Schema> newKeyValueSchema(Schema keySchema, Schema valueSchema, Boolean allowNull) { return catchExceptions( () -> (Schema>) getConstructor("org.apache.pulsar.client.impl.schema.KeyValueSchema", - Schema.class, Schema.class).newInstance(keySchema, valueSchema)); + Schema.class, Schema.class ,Boolean.class).newInstance(keySchema, valueSchema, allowNull)); } - public static Schema> newKeyValueSchema(Class key, Class value, SchemaType type) { + public static Schema> newKeyValueSchema(Class key, Class value, SchemaType type, Boolean allowNull) { return catchExceptions( () -> (Schema>) getStaticMethod("org.apache.pulsar.client.impl.schema.KeyValueSchema", - "of", Class.class, Class.class, SchemaType.class).invoke(null, key, value, type)); + "of", Class.class, Class.class, SchemaType.class, Boolean.class).invoke(null, key, value, type,allowNull)); } + public static Schema getSchema(SchemaInfo schemaInfo) { return catchExceptions( () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.AutoConsumeSchema", diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index 1381ad840e2d6..0b6f53cb10917 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -36,6 +36,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.Map; @Slf4j @@ -52,7 +53,11 @@ public class AvroSchema implements Schema { new ThreadLocal<>(); private AvroSchema(org.apache.avro.Schema schema, + Boolean allowNull, Map properties) { + + properties.put("allowNull", allowNull?"true":"false"); + this.schema = schema; this.schemaInfo = new SchemaInfo(); @@ -99,16 +104,21 @@ public SchemaInfo getSchemaInfo() { return this.schemaInfo; } - private static org.apache.avro.Schema createAvroSchema(Class pojo) { - return ReflectData.get().getSchema(pojo); - } + private static org.apache.avro.Schema createAvroSchema(Class pojo,Boolean allowNull) { - public static AvroSchema of(Class pojo) { - return new AvroSchema<>(createAvroSchema(pojo), Collections.emptyMap()); + return allowNull?ReflectData.AllowNull.get().getSchema(pojo):ReflectData.get().getSchema(pojo); } public static AvroSchema of(Class pojo, Map properties) { - return new AvroSchema<>(createAvroSchema(pojo), properties); + return new AvroSchema<>(createAvroSchema(pojo,true), true, new HashMap<>()); + } + + public static AvroSchema of(Class pojo, Boolean allowNull) { + return new AvroSchema<>(createAvroSchema(pojo,allowNull), allowNull, new HashMap<>()); + } + + public static AvroSchema of(Class pojo, Boolean allowNull, Map properties) { + return new AvroSchema<>(createAvroSchema(pojo,allowNull), true, properties); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java index 3289584829bfb..89abc6fbb8228 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.Map; @Slf4j @@ -56,14 +57,15 @@ public class JSONSchema implements Schema{ private final ObjectMapper objectMapper; - private JSONSchema(Class pojo, Map properties) { + private JSONSchema(Class pojo,Boolean allowNull, Map properties) { this.pojo = pojo; this.properties = properties; - this.schema = ReflectData.AllowNull.get().getSchema(pojo); + this.schema = allowNull?ReflectData.AllowNull.get().getSchema(pojo):ReflectData.get().getSchema(pojo); this.schemaInfo = new SchemaInfo(); this.schemaInfo.setName(""); this.schemaInfo.setProperties(properties); + this.schemaInfo.getProperties().put("allowNull",allowNull?"true":"false"); this.schemaInfo.setType(SchemaType.JSON); this.schemaInfo.setSchema(this.schema.toString().getBytes(UTF_8)); this.objectMapper = JSON_MAPPER.get(); @@ -115,11 +117,11 @@ public SchemaInfo getBackwardsCompatibleJsonSchemaInfo() { return backwardsCompatibleSchemaInfo; } - public static JSONSchema of(Class pojo) { - return new JSONSchema<>(pojo, Collections.emptyMap()); + public static JSONSchema of(Class pojo,Boolean allowNull) { + return new JSONSchema<>(pojo,allowNull ,new HashMap<>()); } - public static JSONSchema of(Class pojo, Map properties) { - return new JSONSchema<>(pojo, properties); + public static JSONSchema of(Class pojo,Boolean allowNull ,Map properties) { + return new JSONSchema<>(pojo,allowNull,properties); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java index 0689b7e78e827..1aaca15308af6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkArgument; import java.nio.ByteBuffer; +import java.util.HashMap; import lombok.Getter; @@ -45,18 +46,18 @@ public class KeyValueSchema implements Schema> { /** * Key Value Schema using passed in schema type, support JSON and AVRO currently. */ - public static Schema> of(Class key, Class value, SchemaType type) { + public static Schema> of(Class key, Class value, SchemaType type ,Boolean allowNull) { checkArgument(SchemaType.JSON == type || SchemaType.AVRO == type); if (SchemaType.JSON == type) { - return new KeyValueSchema<>(JSONSchema.of(key), JSONSchema.of(value)); + return new KeyValueSchema<>(JSONSchema.of(key,allowNull), JSONSchema.of(value,allowNull),allowNull); } else { // AVRO - return new KeyValueSchema<>(AvroSchema.of(key), AvroSchema.of(value)); + return new KeyValueSchema<>(AvroSchema.of(key,allowNull), AvroSchema.of(value,allowNull),allowNull); } } public KeyValueSchema(Schema keySchema, - Schema valueSchema) { + Schema valueSchema ,Boolean allowNull) { this.keySchema = keySchema; this.valueSchema = valueSchema; @@ -64,6 +65,8 @@ public KeyValueSchema(Schema keySchema, this.schemaInfo = new SchemaInfo() .setName("KeyValue") .setType(SchemaType.KEY_VALUE); + this.schemaInfo.setProperties(new HashMap<>()); + this.schemaInfo.getProperties().put("allowNull",allowNull?"true":"false"); byte[] keySchemaInfo = keySchema.getSchemaInfo().getSchema(); byte[] valueSchemaInfo = valueSchema.getSchemaInfo().getSchema(); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index d239f60dad397..72cd9ed92a443 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -19,7 +19,8 @@ package org.apache.pulsar.client.impl.schema; import static org.apache.pulsar.client.impl.schema.SchemaTestUtils.FOO_FIELDS; -import static org.apache.pulsar.client.impl.schema.SchemaTestUtils.SCHEMA_AVRO; +import static org.apache.pulsar.client.impl.schema.SchemaTestUtils.SCHEMA_AVRO_NOT_ALLOW_NULL; +import static org.apache.pulsar.client.impl.schema.SchemaTestUtils.SCHEMA_AVRO_ALLOW_NULL; import static org.testng.Assert.assertEquals; import lombok.extern.slf4j.Slf4j; @@ -31,16 +32,18 @@ import org.testng.Assert; import org.testng.annotations.Test; +import java.util.HashMap; + @Slf4j public class AvroSchemaTest { @Test - public void testSchema() { - AvroSchema avroSchema = AvroSchema.of(Foo.class); + public void testNotAllowNullSchema() { + AvroSchema avroSchema = AvroSchema.of(Foo.class,false); assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); - assertEquals(schemaJson, SCHEMA_AVRO); + assertEquals(schemaJson, SCHEMA_AVRO_NOT_ALLOW_NULL); Schema schema = parser.parse(schemaJson); for (String fieldName : FOO_FIELDS) { @@ -57,8 +60,30 @@ public void testSchema() { } @Test - public void testEncodeAndDecode() { - AvroSchema avroSchema = AvroSchema.of(Foo.class, null); + public void testAllowNullSchema() { + AvroSchema avroSchema = AvroSchema.of(Foo.class,true); + assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); + Schema.Parser parser = new Schema.Parser(); + String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); + assertEquals(schemaJson, SCHEMA_AVRO_ALLOW_NULL); + Schema schema = parser.parse(schemaJson); + + for (String fieldName : FOO_FIELDS) { + Schema.Field field = schema.getField(fieldName); + Assert.assertNotNull(field); + + if (field.name().equals("field4")) { + Assert.assertNotNull(field.schema().getTypes().get(1).getField("field1")); + } + if (field.name().equals("fieldUnableNull")) { + Assert.assertNotNull(field.schema().getType()); + } + } + } + + @Test + public void testNotAllowNullEncodeAndDecode() { + AvroSchema avroSchema = AvroSchema.of(Foo.class,false); Foo foo1 = new Foo(); foo1.setField1("foo1"); @@ -84,6 +109,33 @@ public void testEncodeAndDecode() { } } + @Test + public void testAllowNullEncodeAndDecode() { + + AvroSchema avroSchema = AvroSchema.of(Foo.class, new HashMap<>()); + + Foo foo1 = new Foo(); + foo1.setField1("foo1"); + foo1.setField2("bar1"); + foo1.setField4(new Bar()); + + Foo foo2 = new Foo(); + foo2.setField1("foo2"); + foo2.setField2("bar2"); + + byte[] bytes1 = avroSchema.encode(foo1); + Assert.assertTrue(bytes1.length > 0); + + byte[] bytes2 = avroSchema.encode(foo2); + Assert.assertTrue(bytes2.length > 0); + + Foo object1 = avroSchema.decode(bytes1); + Foo object2 = avroSchema.decode(bytes2); + + assertEquals(object1, foo1); + assertEquals(object2, foo2); + + } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java index 9184faa45910f..55c08a5077683 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java @@ -24,7 +24,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.pulsar.client.api.SchemaSerializationException; -import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Bar; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.DerivedFoo; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Foo; @@ -35,18 +34,20 @@ import org.testng.annotations.Test; import static org.apache.pulsar.client.impl.schema.SchemaTestUtils.FOO_FIELDS; -import static org.apache.pulsar.client.impl.schema.SchemaTestUtils.SCHEMA_JSON; +import static org.apache.pulsar.client.impl.schema.SchemaTestUtils.SCHEMA_JSON_NOT_ALLOW_NULL; +import static org.apache.pulsar.client.impl.schema.SchemaTestUtils.SCHEMA_JSON_ALLOW_NULL; +import static org.testng.Assert.assertEquals; @Slf4j public class JSONSchemaTest { @Test - public void testSchema() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class); + public void testNotAllowNullSchema() { + JSONSchema jsonSchema = JSONSchema.of(Foo.class,false); Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema()); - Assert.assertEquals(schemaJson, SCHEMA_JSON); + Assert.assertEquals(schemaJson, SCHEMA_JSON_NOT_ALLOW_NULL); Schema schema = parser.parse(schemaJson); for (String fieldName : FOO_FIELDS) { @@ -56,12 +57,36 @@ public void testSchema() { if (field.name().equals("field4")) { Assert.assertNotNull(field.schema().getTypes().get(1).getField("field1")); } + if (field.name().equals("fieldUnableNull")) { + Assert.assertNotNull(field.schema().getType()); + } + } + } + @Test + public void testAllowNullSchema() { + JSONSchema jsonSchema = JSONSchema.of(Foo.class,true); + Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON); + Schema.Parser parser = new Schema.Parser(); + String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema()); + Assert.assertEquals(schemaJson, SCHEMA_JSON_ALLOW_NULL); + Schema schema = parser.parse(schemaJson); + + for (String fieldName : FOO_FIELDS) { + Schema.Field field = schema.getField(fieldName); + Assert.assertNotNull(field); + + if (field.name().equals("field4")) { + Assert.assertNotNull(field.schema().getTypes().get(1).getField("field1")); + } + if (field.name().equals("fieldUnableNull")) { + Assert.assertNotNull(field.schema().getType()); + } } } @Test - public void testEncodeAndDecode() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class, null); + public void testAllowNullEncodeAndDecode() { + JSONSchema jsonSchema = JSONSchema.of(Foo.class, true); Bar bar = new Bar(); bar.setField1(true); @@ -90,9 +115,65 @@ public void testEncodeAndDecode() { } @Test - public void testNestedClasses() { - JSONSchema jsonSchema = JSONSchema.of(NestedBar.class, null); - JSONSchema listJsonSchema = JSONSchema.of(NestedBarList.class, null); + public void testNotAllowNullEncodeAndDecode() { + JSONSchema jsonSchema = JSONSchema.of(Foo.class,false); + + Foo foo1 = new Foo(); + foo1.setField1("foo1"); + foo1.setField2("bar1"); + foo1.setField4(new Bar()); + foo1.setFieldUnableNull("notNull"); + + Foo foo2 = new Foo(); + foo2.setField1("foo2"); + foo2.setField2("bar2"); + + byte[] bytes1 = jsonSchema.encode(foo1); + Foo object1 = jsonSchema.decode(bytes1); + Assert.assertTrue(bytes1.length > 0); + assertEquals(object1, foo1); + + try { + + jsonSchema.encode(foo2); + + } catch (Exception e) { + Assert.assertTrue(e instanceof SchemaSerializationException); + } + + } + + @Test + public void testAllowNullNestedClasses() { + JSONSchema jsonSchema = JSONSchema.of(NestedBar.class, true); + JSONSchema listJsonSchema = JSONSchema.of(NestedBarList.class, true); + + Bar bar = new Bar(); + bar.setField1(true); + + NestedBar nested = new NestedBar(); + nested.setField1(true); + nested.setNested(bar); + + byte[] bytes = jsonSchema.encode(nested); + Assert.assertTrue(bytes.length > 0); + Assert.assertEquals(jsonSchema.decode(bytes), nested); + + List list = Collections.singletonList(bar); + NestedBarList nestedList = new NestedBarList(); + nestedList.setField1(true); + nestedList.setList(list); + + bytes = listJsonSchema.encode(nestedList); + Assert.assertTrue(bytes.length > 0); + + Assert.assertEquals(listJsonSchema.decode(bytes), nestedList); + } + + @Test + public void testNotAllowNullNestedClasses() { + JSONSchema jsonSchema = JSONSchema.of(NestedBar.class, false); + JSONSchema listJsonSchema = JSONSchema.of(NestedBarList.class, false); Bar bar = new Bar(); bar.setField1(true); @@ -117,7 +198,58 @@ public void testNestedClasses() { } @Test - public void testCorrectPolymorphism() { + public void testNotAllowNullCorrectPolymorphism() { + Bar bar = new Bar(); + bar.setField1(true); + + DerivedFoo derivedFoo = new DerivedFoo(); + derivedFoo.setField1("foo1"); + derivedFoo.setField2("bar2"); + derivedFoo.setField3(4); + derivedFoo.setField4(bar); + derivedFoo.setField5("derived1"); + derivedFoo.setField6(2); + + Foo foo = new Foo(); + foo.setField1("foo1"); + foo.setField2("bar2"); + foo.setField3(4); + foo.setField4(bar); + + SchemaTestUtils.DerivedDerivedFoo derivedDerivedFoo = new SchemaTestUtils.DerivedDerivedFoo(); + derivedDerivedFoo.setField1("foo1"); + derivedDerivedFoo.setField2("bar2"); + derivedDerivedFoo.setField3(4); + derivedDerivedFoo.setField4(bar); + derivedDerivedFoo.setField5("derived1"); + derivedDerivedFoo.setField6(2); + derivedDerivedFoo.setFoo2(foo); + derivedDerivedFoo.setDerivedFoo(derivedFoo); + + // schema for base class + JSONSchema baseJsonSchema = JSONSchema.of(Foo.class,true); + Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(foo)), foo); + Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedFoo)), foo); + Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedDerivedFoo)), foo); + + // schema for derived class + JSONSchema derivedJsonSchema = JSONSchema.of(DerivedFoo.class,true); + Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedFoo)), derivedFoo); + Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedDerivedFoo)), derivedFoo); + + //schema for derived derived class + JSONSchema derivedDerivedJsonSchema + = JSONSchema.of(SchemaTestUtils.DerivedDerivedFoo.class,true); + Assert.assertEquals(derivedDerivedJsonSchema.decode(derivedDerivedJsonSchema.encode(derivedDerivedFoo)), derivedDerivedFoo); + } + + @Test(expectedExceptions = SchemaSerializationException.class) + public void testAllowNullDecodeWithInvalidContent() { + JSONSchema jsonSchema = JSONSchema.of(Foo.class,true); + jsonSchema.decode(new byte[0]); + } + @Test + public void testAllowNullCorrectPolymorphism() { Bar bar = new Bar(); bar.setField1(true); @@ -146,25 +278,25 @@ public void testCorrectPolymorphism() { derivedDerivedFoo.setDerivedFoo(derivedFoo); // schema for base class - JSONSchema baseJsonSchema = JSONSchema.of(Foo.class); + JSONSchema baseJsonSchema = JSONSchema.of(Foo.class,false); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(foo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedFoo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedDerivedFoo)), foo); // schema for derived class - JSONSchema derivedJsonSchema = JSONSchema.of(DerivedFoo.class); + JSONSchema derivedJsonSchema = JSONSchema.of(DerivedFoo.class,false); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedFoo)), derivedFoo); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedDerivedFoo)), derivedFoo); //schema for derived derived class JSONSchema derivedDerivedJsonSchema - = JSONSchema.of(SchemaTestUtils.DerivedDerivedFoo.class); + = JSONSchema.of(SchemaTestUtils.DerivedDerivedFoo.class,false); Assert.assertEquals(derivedDerivedJsonSchema.decode(derivedDerivedJsonSchema.encode(derivedDerivedFoo)), derivedDerivedFoo); } @Test(expectedExceptions = SchemaSerializationException.class) - public void testDecodeWithInvalidContent() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class); + public void testNotAllowNullDecodeWithInvalidContent() { + JSONSchema jsonSchema = JSONSchema.of(Foo.class,false); jsonSchema.decode(new byte[0]); } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java index 6b3cc5b84319c..42753b729cfa8 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java @@ -37,9 +37,9 @@ public class KeyValueSchemaTest { @Test - public void testAvroSchemaCreate() { - AvroSchema fooSchema = AvroSchema.of(Foo.class); - AvroSchema barSchema = AvroSchema.of(Bar.class); + public void testAllowNullAvroSchemaCreate() { + AvroSchema fooSchema = AvroSchema.of(Foo.class,true); + AvroSchema barSchema = AvroSchema.of(Bar.class,true); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.AVRO); @@ -62,9 +62,34 @@ public void testAvroSchemaCreate() { } @Test - public void testJsonSchemaCreate() { - JSONSchema fooSchema = JSONSchema.of(Foo.class); - JSONSchema barSchema = JSONSchema.of(Bar.class); + public void testNotAllowNullAvroSchemaCreate() { + AvroSchema fooSchema = AvroSchema.of(Foo.class,false); + AvroSchema barSchema = AvroSchema.of(Bar.class,false); + + Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema ,false); + Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.AVRO, false); + + assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE); + assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); + + assertEquals(((KeyValueSchema)keyValueSchema1).getKeySchema().getSchemaInfo().getType(), + SchemaType.AVRO); + assertEquals(((KeyValueSchema)keyValueSchema1).getValueSchema().getSchemaInfo().getType(), + SchemaType.AVRO); + assertEquals(((KeyValueSchema)keyValueSchema2).getKeySchema().getSchemaInfo().getType(), + SchemaType.AVRO); + assertEquals(((KeyValueSchema)keyValueSchema2).getValueSchema().getSchemaInfo().getType(), + SchemaType.AVRO); + + String schemaInfo1 = new String(keyValueSchema1.getSchemaInfo().getSchema()); + String schemaInfo2 = new String(keyValueSchema2.getSchemaInfo().getSchema()); + assertEquals(schemaInfo1, schemaInfo2); + } + + @Test + public void testAllowNullJsonSchemaCreate() { + JSONSchema fooSchema = JSONSchema.of(Foo.class,true); + JSONSchema barSchema = JSONSchema.of(Bar.class,true); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.JSON); @@ -95,7 +120,40 @@ public void testJsonSchemaCreate() { } @Test - public void testSchemaEncodeAndDecode() { + public void testNotAllowNullJsonSchemaCreate() { + JSONSchema fooSchema = JSONSchema.of(Foo.class,false); + JSONSchema barSchema = JSONSchema.of(Bar.class,false); + + Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema,false); + Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.JSON,false); + Schema> keyValueSchema3 = Schema.KeyValue(Foo.class, Bar.class,false); + + assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE); + assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); + assertEquals(keyValueSchema3.getSchemaInfo().getType(), SchemaType.KEY_VALUE); + + assertEquals(((KeyValueSchema)keyValueSchema1).getKeySchema().getSchemaInfo().getType(), + SchemaType.JSON); + assertEquals(((KeyValueSchema)keyValueSchema1).getValueSchema().getSchemaInfo().getType(), + SchemaType.JSON); + assertEquals(((KeyValueSchema)keyValueSchema2).getKeySchema().getSchemaInfo().getType(), + SchemaType.JSON); + assertEquals(((KeyValueSchema)keyValueSchema2).getValueSchema().getSchemaInfo().getType(), + SchemaType.JSON); + assertEquals(((KeyValueSchema)keyValueSchema3).getKeySchema().getSchemaInfo().getType(), + SchemaType.JSON); + assertEquals(((KeyValueSchema)keyValueSchema3).getValueSchema().getSchemaInfo().getType(), + SchemaType.JSON); + + String schemaInfo1 = new String(keyValueSchema1.getSchemaInfo().getSchema()); + String schemaInfo2 = new String(keyValueSchema2.getSchemaInfo().getSchema()); + String schemaInfo3 = new String(keyValueSchema3.getSchemaInfo().getSchema()); + assertEquals(schemaInfo1, schemaInfo2); + assertEquals(schemaInfo1, schemaInfo3); + } + + @Test + public void testAllowNullSchemaEncodeAndDecode() { Schema keyValueSchema = Schema.KeyValue(Foo.class, Bar.class); Bar bar = new Bar(); @@ -120,9 +178,63 @@ public void testSchemaEncodeAndDecode() { } @Test - public void testBytesSchemaEncodeAndDecode() { - AvroSchema fooAvroSchema = AvroSchema.of(Foo.class); - AvroSchema barAvroSchema = AvroSchema.of(Bar.class); + public void testNotAllowNullSchemaEncodeAndDecode() { + Schema keyValueSchema = Schema.KeyValue(Foo.class, Bar.class,false); + + Bar bar = new Bar(); + bar.setField1(true); + + Foo foo = new Foo(); + foo.setField1("field1"); + foo.setField2("field2"); + foo.setField3(3); + foo.setField4(bar); + foo.setColor(Color.RED); + + byte[] encodeBytes = keyValueSchema.encode(new KeyValue(foo, bar)); + Assert.assertTrue(encodeBytes.length > 0); + + KeyValue keyValue = (KeyValue)keyValueSchema.decode(encodeBytes); + Foo fooBack = keyValue.getKey(); + Bar barBack = keyValue.getValue(); + + assertEquals(foo, fooBack); + assertEquals(bar, barBack); + } + + @Test + public void testAllowNullBytesSchemaEncodeAndDecode() { + AvroSchema fooAvroSchema = AvroSchema.of(Foo.class,true); + AvroSchema barAvroSchema = AvroSchema.of(Bar.class,true); + + Bar bar = new Bar(); + bar.setField1(true); + + Foo foo = new Foo(); + foo.setField1("field1"); + foo.setField2("field2"); + foo.setField3(3); + foo.setField4(bar); + foo.setColor(Color.RED); + foo.setFieldUnableNull("notNull"); + + byte[] fooBytes = fooAvroSchema.encode(foo); + byte[] barBytes = barAvroSchema.encode(bar); + + byte[] encodeBytes = Schema.KV_BYTES.encode(new KeyValue<>(fooBytes, barBytes)); + KeyValue decodeKV = Schema.KV_BYTES.decode(encodeBytes); + + Foo fooBack = fooAvroSchema.decode(decodeKV.getKey()); + Bar barBack = barAvroSchema.decode(decodeKV.getValue()); + + assertEquals(foo, fooBack); + assertEquals(bar, barBack); + } + + @Test + public void testNotAllowNullBytesSchemaEncodeAndDecode() { + AvroSchema fooAvroSchema = AvroSchema.of(Foo.class,false); + AvroSchema barAvroSchema = AvroSchema.of(Bar.class,false); Bar bar = new Bar(); bar.setField1(true); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java index dc676127ea7dd..a4446bed4b742 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java @@ -95,17 +95,27 @@ public static class DerivedDerivedFoo extends DerivedFoo { private Foo foo2; } - public static final String SCHEMA_AVRO + public static final String SCHEMA_AVRO_NOT_ALLOW_NULL = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"]," + "\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":" + "\"record\",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\"," + "\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\",\"type\":\"string\",\"default\":\"defaultValue\"}]}"; - public static final String SCHEMA_JSON - = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"]," + - "\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\"" + - ",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\"," + - "\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\",\"type\":[\"null\",\"string\"],\"default\":\"defaultValue\"}]}"; + public static final String SCHEMA_AVRO_ALLOW_NULL="{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\"," + + "\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"" + + "null\",{\"type\":\"record\",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\"" + + ",\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\",\"type\":[\"null\",\"string\"],\"default\":\"defaultValue\"}]}"; + + public static final String SCHEMA_JSON_NOT_ALLOW_NULL + = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\"" + + ":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"Bar\",\"fields\":[{\"name\":\"" + + "field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\"," + + "\"type\":\"string\",\"default\":\"defaultValue\"}]}"; + public static final String SCHEMA_JSON_ALLOW_NULL + = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"],\"default\":null}," + + "{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"Bar\",\"fields\":" + + "[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":" + + "\"fieldUnableNull\",\"type\":[\"null\",\"string\"],\"default\":\"defaultValue\"}]}"; public static String[] FOO_FIELDS = { "field1", diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java index 90758b6edaef4..c1ae007e08389 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java @@ -35,7 +35,7 @@ public class SampleAsyncProducerWithSchema { public static void main(String[] args) throws IOException { PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("http://localhost:8080").build(); - Producer producer = pulsarClient.newProducer(JSONSchema.of(JsonPojo.class)).topic("persistent://my-property/use/my-ns/my-topic") + Producer producer = pulsarClient.newProducer(JSONSchema.of(JsonPojo.class,true)).topic("persistent://my-property/use/my-ns/my-topic") .sendTimeout(3, TimeUnit.SECONDS).create(); List> futures = Lists.newArrayList(); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java index 378033247e78c..465df1b5270d6 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java @@ -30,7 +30,7 @@ public static void main(String[] args) throws PulsarClientException, JsonProcess PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("http://localhost:8080").build(); - Consumer consumer = pulsarClient.newConsumer(JSONSchema.of(JsonPojo.class)) // + Consumer consumer = pulsarClient.newConsumer(JSONSchema.of(JsonPojo.class,true)) // .topic("persistent://my-property/use/my-ns/my-topic") // .subscriptionName("my-subscription-name").subscribe(); From 8241d6970dd77f5795b639d42e14cd99645237d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 6 Mar 2019 11:14:13 +0800 Subject: [PATCH 04/35] recover the format --- .../org/apache/pulsar/client/api/Schema.java | 20 +++++++++++-------- .../pulsar/client/impl/schema/AvroSchema.java | 4 ++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java index a2c9ca72abff6..7e5f927c39dc0 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -19,7 +19,6 @@ package org.apache.pulsar.client.api; import java.nio.ByteBuffer; - import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.GenericSchema; import org.apache.pulsar.client.internal.DefaultImplementation; @@ -51,9 +50,11 @@ default void validate(byte[] message) { /** * Encode an object representing the message content into a byte array. * - * @param message the message object + * @param message + * the message object * @return a byte array with the serialized content - * @throws SchemaSerializationException if the serialization fails + * @throws SchemaSerializationException + * if the serialization fails */ byte[] encode(T message); @@ -77,7 +78,8 @@ default boolean supportSchemaVersioning() { /** * Decode a byte array into an object using the schema definition and deserializer implementation * - * @param bytes the byte array to decode + * @param bytes + * the byte array to decode * @return the deserialized object */ default T decode(byte[] bytes) { @@ -88,8 +90,10 @@ default T decode(byte[] bytes) { /** * Decode a byte array into an object using a given version. * - * @param bytes the byte array to decode - * @param schemaVersion the schema version to decode the object. null indicates using latest version. + * @param bytes + * the byte array to decode + * @param schemaVersion + * the schema version to decode the object. null indicates using latest version. * @return the deserialized object */ default T decode(byte[] bytes, byte[] schemaVersion) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index 0b6f53cb10917..2b672cf9ae424 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY From bbda7b4f27a3800e8f37832f7c3205b844759a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 6 Mar 2019 11:38:00 +0800 Subject: [PATCH 05/35] Modify the code format --- .../org/apache/pulsar/client/api/Schema.java | 8 +- .../internal/DefaultImplementation.java | 10 +- .../pulsar/client/impl/schema/AvroSchema.java | 17 ++-- .../pulsar/client/impl/schema/JSONSchema.java | 16 +-- .../client/impl/schema/KeyValueSchema.java | 16 +-- .../client/impl/schema/AvroSchemaTest.java | 9 +- .../client/impl/schema/JSONSchemaTest.java | 24 ++--- .../impl/schema/KeyValueSchemaTest.java | 98 +++++++++---------- .../client/impl/schema/SchemaTestUtils.java | 4 +- 9 files changed, 103 insertions(+), 99 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java index 7e5f927c39dc0..ebcaa3ad038d0 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -214,7 +214,7 @@ static Schema JSON(Class clazz, Boolean allowNull) { * Key Value Schema using passed in schema type, support JSON and AVRO currently. */ static Schema> KeyValue(Class key, Class value, SchemaType type) { - return DefaultImplementation.newKeyValueSchema(key, value, type,true); + return DefaultImplementation.newKeyValueSchema(key, value, type, true); } /** @@ -234,7 +234,7 @@ static Schema> KeyValue(Class key, Class value, Sche * Key Value Schema whose underneath key and value schemas are JSONSchema. */ static Schema> KeyValue(Class key, Class value) { - return DefaultImplementation.newKeyValueSchema(key, value, SchemaType.JSON,true); + return DefaultImplementation.newKeyValueSchema(key, value, SchemaType.JSON, true); } /** @@ -248,7 +248,7 @@ static Schema> KeyValue(Schema key, Schema value) { * Key Value Schema whose underneath key and value schemas are JSONSchema. */ static Schema> KeyValue(Class key, Class value, Boolean allowNull) { - return DefaultImplementation.newKeyValueSchema(key, value, SchemaType.JSON ,allowNull); + return DefaultImplementation.newKeyValueSchema(key, value, SchemaType.JSON, allowNull); } /** diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java index a581583dcc541..c9b6bb1e224a9 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java @@ -182,9 +182,9 @@ public static Schema newDoubleSchema() { .newInstance()); } - public static Schema newAvroSchema(Class clazz,Boolean allowNull) { + public static Schema newAvroSchema(Class clazz, Boolean allowNull) { return catchExceptions( - () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.AvroSchema", "of", Class.class,Boolean.class) + () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.AvroSchema", "of", Class.class, Boolean.class) .invoke(null, clazz,allowNull)); } @@ -194,9 +194,9 @@ public static Schema newPr .invoke(null, clazz)); } - public static Schema newJSONSchema(Class clazz,Boolean allowNull) { + public static Schema newJSONSchema(Class clazz, Boolean allowNull) { return catchExceptions( - () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.JSONSchema", "of", Class.class,Boolean.class) + () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.JSONSchema", "of", Class.class, Boolean.class) .invoke(null, clazz,allowNull)); } @@ -215,7 +215,7 @@ public static Schema newAutoProduceSchema() { public static Schema> newKeyValueSchema(Schema keySchema, Schema valueSchema, Boolean allowNull) { return catchExceptions( () -> (Schema>) getConstructor("org.apache.pulsar.client.impl.schema.KeyValueSchema", - Schema.class, Schema.class ,Boolean.class).newInstance(keySchema, valueSchema, allowNull)); + Schema.class, Schema.class , Boolean.class).newInstance(keySchema, valueSchema, allowNull)); } public static Schema> newKeyValueSchema(Class key, Class value, SchemaType type, Boolean allowNull) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index 2b672cf9ae424..5666e97c517ac 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -7,8 +7,8 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -56,7 +56,7 @@ private AvroSchema(org.apache.avro.Schema schema, Boolean allowNull, Map properties) { - properties.put("allowNull", allowNull?"true":"false"); + properties.put("allowNull", allowNull ? "true" : "false"); this.schema = schema; @@ -99,26 +99,27 @@ public T decode(byte[] bytes) { } } + @Override public SchemaInfo getSchemaInfo() { return this.schemaInfo; } - private static org.apache.avro.Schema createAvroSchema(Class pojo,Boolean allowNull) { + private static org.apache.avro.Schema createAvroSchema(Class pojo, Boolean allowNull) { - return allowNull?ReflectData.AllowNull.get().getSchema(pojo):ReflectData.get().getSchema(pojo); + return allowNull ? ReflectData.AllowNull.get().getSchema(pojo) : ReflectData.get().getSchema(pojo); } public static AvroSchema of(Class pojo, Map properties) { - return new AvroSchema<>(createAvroSchema(pojo,true), true, new HashMap<>()); + return new AvroSchema<>(createAvroSchema(pojo, true), true, new HashMap<>()); } public static AvroSchema of(Class pojo, Boolean allowNull) { - return new AvroSchema<>(createAvroSchema(pojo,allowNull), allowNull, new HashMap<>()); + return new AvroSchema<>(createAvroSchema(pojo, allowNull), allowNull, new HashMap<>()); } public static AvroSchema of(Class pojo, Boolean allowNull, Map properties) { - return new AvroSchema<>(createAvroSchema(pojo,allowNull), true, properties); + return new AvroSchema<>(createAvroSchema(pojo, allowNull), true, properties); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java index 89abc6fbb8228..b0aa1f2a862f7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java @@ -39,7 +39,7 @@ import java.util.Map; @Slf4j -public class JSONSchema implements Schema{ +public class JSONSchema implements Schema { private final org.apache.avro.Schema schema; private final SchemaInfo schemaInfo; @@ -57,15 +57,15 @@ public class JSONSchema implements Schema{ private final ObjectMapper objectMapper; - private JSONSchema(Class pojo,Boolean allowNull, Map properties) { + private JSONSchema(Class pojo, Boolean allowNull, Map properties) { this.pojo = pojo; this.properties = properties; - this.schema = allowNull?ReflectData.AllowNull.get().getSchema(pojo):ReflectData.get().getSchema(pojo); + this.schema = allowNull ? ReflectData.AllowNull.get().getSchema(pojo) : ReflectData.get().getSchema(pojo); this.schemaInfo = new SchemaInfo(); this.schemaInfo.setName(""); this.schemaInfo.setProperties(properties); - this.schemaInfo.getProperties().put("allowNull",allowNull?"true":"false"); + this.schemaInfo.getProperties().put("allowNull", allowNull ? "true" : "false"); this.schemaInfo.setType(SchemaType.JSON); this.schemaInfo.setSchema(this.schema.toString().getBytes(UTF_8)); this.objectMapper = JSON_MAPPER.get(); @@ -117,11 +117,11 @@ public SchemaInfo getBackwardsCompatibleJsonSchemaInfo() { return backwardsCompatibleSchemaInfo; } - public static JSONSchema of(Class pojo,Boolean allowNull) { - return new JSONSchema<>(pojo,allowNull ,new HashMap<>()); + public static JSONSchema of(Class pojo, Boolean allowNull) { + return new JSONSchema<>(pojo, allowNull, new HashMap<>()); } - public static JSONSchema of(Class pojo,Boolean allowNull ,Map properties) { - return new JSONSchema<>(pojo,allowNull,properties); + public static JSONSchema of(Class pojo, Boolean allowNull, Map properties) { + return new JSONSchema<>(pojo, allowNull, properties); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java index 1aaca15308af6..605bc3b4650b5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java @@ -46,34 +46,34 @@ public class KeyValueSchema implements Schema> { /** * Key Value Schema using passed in schema type, support JSON and AVRO currently. */ - public static Schema> of(Class key, Class value, SchemaType type ,Boolean allowNull) { + public static Schema> of(Class key, Class value, SchemaType type, Boolean allowNull) { checkArgument(SchemaType.JSON == type || SchemaType.AVRO == type); if (SchemaType.JSON == type) { - return new KeyValueSchema<>(JSONSchema.of(key,allowNull), JSONSchema.of(value,allowNull),allowNull); + return new KeyValueSchema<>(JSONSchema.of(key, allowNull), JSONSchema.of(value, allowNull), allowNull); } else { // AVRO - return new KeyValueSchema<>(AvroSchema.of(key,allowNull), AvroSchema.of(value,allowNull),allowNull); + return new KeyValueSchema<>(AvroSchema.of(key, allowNull), AvroSchema.of(value, allowNull), allowNull); } } public KeyValueSchema(Schema keySchema, - Schema valueSchema ,Boolean allowNull) { + Schema valueSchema, Boolean allowNull) { this.keySchema = keySchema; this.valueSchema = valueSchema; // set schemaInfo this.schemaInfo = new SchemaInfo() - .setName("KeyValue") - .setType(SchemaType.KEY_VALUE); + .setName("KeyValue") + .setType(SchemaType.KEY_VALUE); this.schemaInfo.setProperties(new HashMap<>()); - this.schemaInfo.getProperties().put("allowNull",allowNull?"true":"false"); + this.schemaInfo.getProperties().put("allowNull", allowNull ? "true" : "false"); byte[] keySchemaInfo = keySchema.getSchemaInfo().getSchema(); byte[] valueSchemaInfo = valueSchema.getSchemaInfo().getSchema(); ByteBuffer byteBuffer = ByteBuffer.allocate(4 + keySchemaInfo.length + 4 + valueSchemaInfo.length); byteBuffer.putInt(keySchemaInfo.length).put(keySchemaInfo) - .putInt(valueSchemaInfo.length).put(valueSchemaInfo); + .putInt(valueSchemaInfo.length).put(valueSchemaInfo); this.schemaInfo.setSchema(byteBuffer.array()); } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index 72cd9ed92a443..43a6c4c99a24d 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -39,7 +39,7 @@ public class AvroSchemaTest { @Test public void testNotAllowNullSchema() { - AvroSchema avroSchema = AvroSchema.of(Foo.class,false); + AvroSchema avroSchema = AvroSchema.of(Foo.class, false); assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); @@ -61,7 +61,7 @@ public void testNotAllowNullSchema() { @Test public void testAllowNullSchema() { - AvroSchema avroSchema = AvroSchema.of(Foo.class,true); + AvroSchema avroSchema = AvroSchema.of(Foo.class, true); assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); @@ -83,7 +83,7 @@ public void testAllowNullSchema() { @Test public void testNotAllowNullEncodeAndDecode() { - AvroSchema avroSchema = AvroSchema.of(Foo.class,false); + AvroSchema avroSchema = AvroSchema.of(Foo.class, false); Foo foo1 = new Foo(); foo1.setField1("foo1"); @@ -102,13 +102,14 @@ public void testNotAllowNullEncodeAndDecode() { try { - avroSchema.encode(foo2); + avroSchema.encode(foo2); } catch (Exception e) { Assert.assertTrue(e instanceof SchemaSerializationException); } } + @Test public void testAllowNullEncodeAndDecode() { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java index 55c08a5077683..fc62427c34ded 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java @@ -43,7 +43,7 @@ public class JSONSchemaTest { @Test public void testNotAllowNullSchema() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class,false); + JSONSchema jsonSchema = JSONSchema.of(Foo.class, false); Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema()); @@ -62,9 +62,10 @@ public void testNotAllowNullSchema() { } } } + @Test public void testAllowNullSchema() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class,true); + JSONSchema jsonSchema = JSONSchema.of(Foo.class, true); Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema()); @@ -116,7 +117,7 @@ public void testAllowNullEncodeAndDecode() { @Test public void testNotAllowNullEncodeAndDecode() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class,false); + JSONSchema jsonSchema = JSONSchema.of(Foo.class, false); Foo foo1 = new Foo(); foo1.setField1("foo1"); @@ -227,27 +228,28 @@ public void testNotAllowNullCorrectPolymorphism() { derivedDerivedFoo.setDerivedFoo(derivedFoo); // schema for base class - JSONSchema baseJsonSchema = JSONSchema.of(Foo.class,true); + JSONSchema baseJsonSchema = JSONSchema.of(Foo.class, true); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(foo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedFoo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedDerivedFoo)), foo); // schema for derived class - JSONSchema derivedJsonSchema = JSONSchema.of(DerivedFoo.class,true); + JSONSchema derivedJsonSchema = JSONSchema.of(DerivedFoo.class, true); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedFoo)), derivedFoo); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedDerivedFoo)), derivedFoo); //schema for derived derived class JSONSchema derivedDerivedJsonSchema - = JSONSchema.of(SchemaTestUtils.DerivedDerivedFoo.class,true); + = JSONSchema.of(SchemaTestUtils.DerivedDerivedFoo.class, true); Assert.assertEquals(derivedDerivedJsonSchema.decode(derivedDerivedJsonSchema.encode(derivedDerivedFoo)), derivedDerivedFoo); } @Test(expectedExceptions = SchemaSerializationException.class) public void testAllowNullDecodeWithInvalidContent() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class,true); + JSONSchema jsonSchema = JSONSchema.of(Foo.class, true); jsonSchema.decode(new byte[0]); } + @Test public void testAllowNullCorrectPolymorphism() { Bar bar = new Bar(); @@ -278,25 +280,25 @@ public void testAllowNullCorrectPolymorphism() { derivedDerivedFoo.setDerivedFoo(derivedFoo); // schema for base class - JSONSchema baseJsonSchema = JSONSchema.of(Foo.class,false); + JSONSchema baseJsonSchema = JSONSchema.of(Foo.class, false); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(foo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedFoo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedDerivedFoo)), foo); // schema for derived class - JSONSchema derivedJsonSchema = JSONSchema.of(DerivedFoo.class,false); + JSONSchema derivedJsonSchema = JSONSchema.of(DerivedFoo.class, false); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedFoo)), derivedFoo); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedDerivedFoo)), derivedFoo); //schema for derived derived class JSONSchema derivedDerivedJsonSchema - = JSONSchema.of(SchemaTestUtils.DerivedDerivedFoo.class,false); + = JSONSchema.of(SchemaTestUtils.DerivedDerivedFoo.class, false); Assert.assertEquals(derivedDerivedJsonSchema.decode(derivedDerivedJsonSchema.encode(derivedDerivedFoo)), derivedDerivedFoo); } @Test(expectedExceptions = SchemaSerializationException.class) public void testNotAllowNullDecodeWithInvalidContent() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class,false); + JSONSchema jsonSchema = JSONSchema.of(Foo.class, false); jsonSchema.decode(new byte[0]); } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java index 42753b729cfa8..9a64950dc4856 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java @@ -38,8 +38,8 @@ public class KeyValueSchemaTest { @Test public void testAllowNullAvroSchemaCreate() { - AvroSchema fooSchema = AvroSchema.of(Foo.class,true); - AvroSchema barSchema = AvroSchema.of(Bar.class,true); + AvroSchema fooSchema = AvroSchema.of(Foo.class, true); + AvroSchema barSchema = AvroSchema.of(Bar.class, true); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.AVRO); @@ -47,14 +47,14 @@ public void testAllowNullAvroSchemaCreate() { assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); - assertEquals(((KeyValueSchema)keyValueSchema1).getKeySchema().getSchemaInfo().getType(), - SchemaType.AVRO); - assertEquals(((KeyValueSchema)keyValueSchema1).getValueSchema().getSchemaInfo().getType(), - SchemaType.AVRO); - assertEquals(((KeyValueSchema)keyValueSchema2).getKeySchema().getSchemaInfo().getType(), - SchemaType.AVRO); - assertEquals(((KeyValueSchema)keyValueSchema2).getValueSchema().getSchemaInfo().getType(), - SchemaType.AVRO); + assertEquals(((KeyValueSchema) keyValueSchema1).getKeySchema().getSchemaInfo().getType(), + SchemaType.AVRO); + assertEquals(((KeyValueSchema) keyValueSchema1).getValueSchema().getSchemaInfo().getType(), + SchemaType.AVRO); + assertEquals(((KeyValueSchema) keyValueSchema2).getKeySchema().getSchemaInfo().getType(), + SchemaType.AVRO); + assertEquals(((KeyValueSchema) keyValueSchema2).getValueSchema().getSchemaInfo().getType(), + SchemaType.AVRO); String schemaInfo1 = new String(keyValueSchema1.getSchemaInfo().getSchema()); String schemaInfo2 = new String(keyValueSchema2.getSchemaInfo().getSchema()); @@ -63,22 +63,22 @@ public void testAllowNullAvroSchemaCreate() { @Test public void testNotAllowNullAvroSchemaCreate() { - AvroSchema fooSchema = AvroSchema.of(Foo.class,false); - AvroSchema barSchema = AvroSchema.of(Bar.class,false); + AvroSchema fooSchema = AvroSchema.of(Foo.class, false); + AvroSchema barSchema = AvroSchema.of(Bar.class, false); - Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema ,false); + Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema, false); Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.AVRO, false); assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); - assertEquals(((KeyValueSchema)keyValueSchema1).getKeySchema().getSchemaInfo().getType(), + assertEquals(((KeyValueSchema) keyValueSchema1).getKeySchema().getSchemaInfo().getType(), SchemaType.AVRO); - assertEquals(((KeyValueSchema)keyValueSchema1).getValueSchema().getSchemaInfo().getType(), + assertEquals(((KeyValueSchema) keyValueSchema1).getValueSchema().getSchemaInfo().getType(), SchemaType.AVRO); - assertEquals(((KeyValueSchema)keyValueSchema2).getKeySchema().getSchemaInfo().getType(), + assertEquals(((KeyValueSchema) keyValueSchema2).getKeySchema().getSchemaInfo().getType(), SchemaType.AVRO); - assertEquals(((KeyValueSchema)keyValueSchema2).getValueSchema().getSchemaInfo().getType(), + assertEquals(((KeyValueSchema) keyValueSchema2).getValueSchema().getSchemaInfo().getType(), SchemaType.AVRO); String schemaInfo1 = new String(keyValueSchema1.getSchemaInfo().getSchema()); @@ -88,8 +88,8 @@ public void testNotAllowNullAvroSchemaCreate() { @Test public void testAllowNullJsonSchemaCreate() { - JSONSchema fooSchema = JSONSchema.of(Foo.class,true); - JSONSchema barSchema = JSONSchema.of(Bar.class,true); + JSONSchema fooSchema = JSONSchema.of(Foo.class, true); + JSONSchema barSchema = JSONSchema.of(Bar.class, true); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.JSON); @@ -99,18 +99,18 @@ public void testAllowNullJsonSchemaCreate() { assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(keyValueSchema3.getSchemaInfo().getType(), SchemaType.KEY_VALUE); - assertEquals(((KeyValueSchema)keyValueSchema1).getKeySchema().getSchemaInfo().getType(), - SchemaType.JSON); - assertEquals(((KeyValueSchema)keyValueSchema1).getValueSchema().getSchemaInfo().getType(), - SchemaType.JSON); - assertEquals(((KeyValueSchema)keyValueSchema2).getKeySchema().getSchemaInfo().getType(), - SchemaType.JSON); - assertEquals(((KeyValueSchema)keyValueSchema2).getValueSchema().getSchemaInfo().getType(), - SchemaType.JSON); - assertEquals(((KeyValueSchema)keyValueSchema3).getKeySchema().getSchemaInfo().getType(), - SchemaType.JSON); - assertEquals(((KeyValueSchema)keyValueSchema3).getValueSchema().getSchemaInfo().getType(), - SchemaType.JSON); + assertEquals(((KeyValueSchema) keyValueSchema1).getKeySchema().getSchemaInfo().getType(), + SchemaType.JSON); + assertEquals(((KeyValueSchema) keyValueSchema1).getValueSchema().getSchemaInfo().getType(), + SchemaType.JSON); + assertEquals(((KeyValueSchema) keyValueSchema2).getKeySchema().getSchemaInfo().getType(), + SchemaType.JSON); + assertEquals(((KeyValueSchema) keyValueSchema2).getValueSchema().getSchemaInfo().getType(), + SchemaType.JSON); + assertEquals(((KeyValueSchema) keyValueSchema3).getKeySchema().getSchemaInfo().getType(), + SchemaType.JSON); + assertEquals(((KeyValueSchema) keyValueSchema3).getValueSchema().getSchemaInfo().getType(), + SchemaType.JSON); String schemaInfo1 = new String(keyValueSchema1.getSchemaInfo().getSchema()); String schemaInfo2 = new String(keyValueSchema2.getSchemaInfo().getSchema()); @@ -121,28 +121,28 @@ public void testAllowNullJsonSchemaCreate() { @Test public void testNotAllowNullJsonSchemaCreate() { - JSONSchema fooSchema = JSONSchema.of(Foo.class,false); - JSONSchema barSchema = JSONSchema.of(Bar.class,false); + JSONSchema fooSchema = JSONSchema.of(Foo.class, false); + JSONSchema barSchema = JSONSchema.of(Bar.class, false); - Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema,false); - Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.JSON,false); - Schema> keyValueSchema3 = Schema.KeyValue(Foo.class, Bar.class,false); + Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema, false); + Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.JSON, false); + Schema> keyValueSchema3 = Schema.KeyValue(Foo.class, Bar.class, false); assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(keyValueSchema3.getSchemaInfo().getType(), SchemaType.KEY_VALUE); - assertEquals(((KeyValueSchema)keyValueSchema1).getKeySchema().getSchemaInfo().getType(), + assertEquals(((KeyValueSchema) keyValueSchema1).getKeySchema().getSchemaInfo().getType(), SchemaType.JSON); - assertEquals(((KeyValueSchema)keyValueSchema1).getValueSchema().getSchemaInfo().getType(), + assertEquals(((KeyValueSchema) keyValueSchema1).getValueSchema().getSchemaInfo().getType(), SchemaType.JSON); - assertEquals(((KeyValueSchema)keyValueSchema2).getKeySchema().getSchemaInfo().getType(), + assertEquals(((KeyValueSchema) keyValueSchema2).getKeySchema().getSchemaInfo().getType(), SchemaType.JSON); - assertEquals(((KeyValueSchema)keyValueSchema2).getValueSchema().getSchemaInfo().getType(), + assertEquals(((KeyValueSchema) keyValueSchema2).getValueSchema().getSchemaInfo().getType(), SchemaType.JSON); - assertEquals(((KeyValueSchema)keyValueSchema3).getKeySchema().getSchemaInfo().getType(), + assertEquals(((KeyValueSchema) keyValueSchema3).getKeySchema().getSchemaInfo().getType(), SchemaType.JSON); - assertEquals(((KeyValueSchema)keyValueSchema3).getValueSchema().getSchemaInfo().getType(), + assertEquals(((KeyValueSchema) keyValueSchema3).getValueSchema().getSchemaInfo().getType(), SchemaType.JSON); String schemaInfo1 = new String(keyValueSchema1.getSchemaInfo().getSchema()); @@ -169,7 +169,7 @@ public void testAllowNullSchemaEncodeAndDecode() { byte[] encodeBytes = keyValueSchema.encode(new KeyValue(foo, bar)); Assert.assertTrue(encodeBytes.length > 0); - KeyValue keyValue = (KeyValue)keyValueSchema.decode(encodeBytes); + KeyValue keyValue = (KeyValue) keyValueSchema.decode(encodeBytes); Foo fooBack = keyValue.getKey(); Bar barBack = keyValue.getValue(); @@ -179,7 +179,7 @@ public void testAllowNullSchemaEncodeAndDecode() { @Test public void testNotAllowNullSchemaEncodeAndDecode() { - Schema keyValueSchema = Schema.KeyValue(Foo.class, Bar.class,false); + Schema keyValueSchema = Schema.KeyValue(Foo.class, Bar.class, false); Bar bar = new Bar(); bar.setField1(true); @@ -194,7 +194,7 @@ public void testNotAllowNullSchemaEncodeAndDecode() { byte[] encodeBytes = keyValueSchema.encode(new KeyValue(foo, bar)); Assert.assertTrue(encodeBytes.length > 0); - KeyValue keyValue = (KeyValue)keyValueSchema.decode(encodeBytes); + KeyValue keyValue = (KeyValue) keyValueSchema.decode(encodeBytes); Foo fooBack = keyValue.getKey(); Bar barBack = keyValue.getValue(); @@ -204,8 +204,8 @@ public void testNotAllowNullSchemaEncodeAndDecode() { @Test public void testAllowNullBytesSchemaEncodeAndDecode() { - AvroSchema fooAvroSchema = AvroSchema.of(Foo.class,true); - AvroSchema barAvroSchema = AvroSchema.of(Bar.class,true); + AvroSchema fooAvroSchema = AvroSchema.of(Foo.class, true); + AvroSchema barAvroSchema = AvroSchema.of(Bar.class, true); Bar bar = new Bar(); bar.setField1(true); @@ -233,8 +233,8 @@ public void testAllowNullBytesSchemaEncodeAndDecode() { @Test public void testNotAllowNullBytesSchemaEncodeAndDecode() { - AvroSchema fooAvroSchema = AvroSchema.of(Foo.class,false); - AvroSchema barAvroSchema = AvroSchema.of(Bar.class,false); + AvroSchema fooAvroSchema = AvroSchema.of(Foo.class, false); + AvroSchema barAvroSchema = AvroSchema.of(Bar.class, false); Bar bar = new Bar(); bar.setField1(true); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java index a4446bed4b742..d25f8417d724f 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java @@ -80,7 +80,7 @@ public static class DerivedFoo extends Foo { private Foo foo; } - public enum Color { + public enum Color { RED, BLUE } @@ -101,7 +101,7 @@ public static class DerivedDerivedFoo extends DerivedFoo { "\"record\",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\"," + "\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\",\"type\":\"string\",\"default\":\"defaultValue\"}]}"; - public static final String SCHEMA_AVRO_ALLOW_NULL="{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\"," + + public static final String SCHEMA_AVRO_ALLOW_NULL = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\"," + "\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"" + "null\",{\"type\":\"record\",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\"" + ",\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\",\"type\":[\"null\",\"string\"],\"default\":\"defaultValue\"}]}"; From 5bdc4dad1b26832f77b6cbfe1a5dce64f488aef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 6 Mar 2019 14:04:16 +0800 Subject: [PATCH 06/35] recover the code format --- .../internal/DefaultImplementation.java | 6 +++--- .../pulsar/client/impl/schema/AvroSchema.java | 2 +- .../client/impl/schema/KeyValueSchema.java | 6 +++--- .../client/impl/schema/AvroSchemaTest.java | 1 - .../impl/schema/KeyValueSchemaTest.java | 20 +++++++++---------- .../client/impl/schema/SchemaTestUtils.java | 2 +- 6 files changed, 18 insertions(+), 19 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java index c9b6bb1e224a9..51eca6c717bea 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java @@ -185,7 +185,7 @@ public static Schema newDoubleSchema() { public static Schema newAvroSchema(Class clazz, Boolean allowNull) { return catchExceptions( () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.AvroSchema", "of", Class.class, Boolean.class) - .invoke(null, clazz,allowNull)); + .invoke(null, clazz, allowNull)); } public static Schema newProtobufSchema(Class clazz) { @@ -197,7 +197,7 @@ public static Schema newPr public static Schema newJSONSchema(Class clazz, Boolean allowNull) { return catchExceptions( () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.JSONSchema", "of", Class.class, Boolean.class) - .invoke(null, clazz,allowNull)); + .invoke(null, clazz, allowNull)); } public static Schema newAutoConsumeSchema() { @@ -221,7 +221,7 @@ public static Schema> newKeyValueSchema(Schema keySchem public static Schema> newKeyValueSchema(Class key, Class value, SchemaType type, Boolean allowNull) { return catchExceptions( () -> (Schema>) getStaticMethod("org.apache.pulsar.client.impl.schema.KeyValueSchema", - "of", Class.class, Class.class, SchemaType.class, Boolean.class).invoke(null, key, value, type,allowNull)); + "of", Class.class, Class.class, SchemaType.class, Boolean.class).invoke(null, key, value, type, allowNull)); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index 5666e97c517ac..963325a7ccfcc 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -8,7 +8,7 @@ * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java index 605bc3b4650b5..4c6133186de04 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java @@ -63,8 +63,8 @@ public KeyValueSchema(Schema keySchema, // set schemaInfo this.schemaInfo = new SchemaInfo() - .setName("KeyValue") - .setType(SchemaType.KEY_VALUE); + .setName("KeyValue") + .setType(SchemaType.KEY_VALUE); this.schemaInfo.setProperties(new HashMap<>()); this.schemaInfo.getProperties().put("allowNull", allowNull ? "true" : "false"); @@ -73,7 +73,7 @@ public KeyValueSchema(Schema keySchema, ByteBuffer byteBuffer = ByteBuffer.allocate(4 + keySchemaInfo.length + 4 + valueSchemaInfo.length); byteBuffer.putInt(keySchemaInfo.length).put(keySchemaInfo) - .putInt(valueSchemaInfo.length).put(valueSchemaInfo); + .putInt(valueSchemaInfo.length).put(valueSchemaInfo); this.schemaInfo.setSchema(byteBuffer.array()); } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index 43a6c4c99a24d..fc02e28adbe25 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -112,7 +112,6 @@ public void testNotAllowNullEncodeAndDecode() { @Test public void testAllowNullEncodeAndDecode() { - AvroSchema avroSchema = AvroSchema.of(Foo.class, new HashMap<>()); Foo foo1 = new Foo(); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java index 9a64950dc4856..ad2c8ab09ce8e 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java @@ -73,13 +73,13 @@ public void testNotAllowNullAvroSchemaCreate() { assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(((KeyValueSchema) keyValueSchema1).getKeySchema().getSchemaInfo().getType(), - SchemaType.AVRO); + SchemaType.AVRO); assertEquals(((KeyValueSchema) keyValueSchema1).getValueSchema().getSchemaInfo().getType(), - SchemaType.AVRO); + SchemaType.AVRO); assertEquals(((KeyValueSchema) keyValueSchema2).getKeySchema().getSchemaInfo().getType(), - SchemaType.AVRO); + SchemaType.AVRO); assertEquals(((KeyValueSchema) keyValueSchema2).getValueSchema().getSchemaInfo().getType(), - SchemaType.AVRO); + SchemaType.AVRO); String schemaInfo1 = new String(keyValueSchema1.getSchemaInfo().getSchema()); String schemaInfo2 = new String(keyValueSchema2.getSchemaInfo().getSchema()); @@ -133,17 +133,17 @@ public void testNotAllowNullJsonSchemaCreate() { assertEquals(keyValueSchema3.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(((KeyValueSchema) keyValueSchema1).getKeySchema().getSchemaInfo().getType(), - SchemaType.JSON); + SchemaType.JSON); assertEquals(((KeyValueSchema) keyValueSchema1).getValueSchema().getSchemaInfo().getType(), - SchemaType.JSON); + SchemaType.JSON); assertEquals(((KeyValueSchema) keyValueSchema2).getKeySchema().getSchemaInfo().getType(), - SchemaType.JSON); + SchemaType.JSON); assertEquals(((KeyValueSchema) keyValueSchema2).getValueSchema().getSchemaInfo().getType(), - SchemaType.JSON); + SchemaType.JSON); assertEquals(((KeyValueSchema) keyValueSchema3).getKeySchema().getSchemaInfo().getType(), - SchemaType.JSON); + SchemaType.JSON); assertEquals(((KeyValueSchema) keyValueSchema3).getValueSchema().getSchemaInfo().getType(), - SchemaType.JSON); + SchemaType.JSON); String schemaInfo1 = new String(keyValueSchema1.getSchemaInfo().getSchema()); String schemaInfo2 = new String(keyValueSchema2.getSchemaInfo().getSchema()); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java index d25f8417d724f..98d53675b5301 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java @@ -80,7 +80,7 @@ public static class DerivedFoo extends Foo { private Foo foo; } - public enum Color { + public enum Color { RED, BLUE } From f24932e6b4555777831181ecc70bf4895f3cb9d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 6 Mar 2019 15:33:32 +0800 Subject: [PATCH 07/35] function topic schema by default allow null need to fix --- .../java/org/apache/pulsar/functions/source/TopicSchema.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java index 01ab9586f8433..5d065fd54279f 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java @@ -124,10 +124,10 @@ private static Schema newSchemaInstance(Class clazz, SchemaType type) return (Schema) Schema.STRING; case AVRO: - return AvroSchema.of(clazz); + return AvroSchema.of(clazz,true); case JSON: - return JSONSchema.of(clazz); + return JSONSchema.of(clazz,true); case KEY_VALUE: return (Schema)Schema.KV_BYTES; From 9040c82594058d859a07065913fed1926e00a53c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 6 Mar 2019 15:53:43 +0800 Subject: [PATCH 08/35] Modify the code format --- .../client/impl/schema/generic/GenericSchemaImplTest.java | 2 +- .../pulsar/client/tutorial/SampleAsyncProducerWithSchema.java | 2 +- .../pulsar/client/tutorial/SampleConsumerWithSchema.java | 2 +- .../java/org/apache/pulsar/functions/source/TopicSchema.java | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericSchemaImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericSchemaImplTest.java index c7b1e1efed66f..ed554cce8e9a5 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericSchemaImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericSchemaImplTest.java @@ -77,7 +77,7 @@ public void testEncodeAndDecodeGenericRecord(Schema encodeSchema, Bar bar = new Bar(); bar.setField1(i % 2 == 0); foo.setField4(bar); - foo.setFieldUnableNull("fieldUnableNull-1-"+i); + foo.setFieldUnableNull("fieldUnableNull-1-" + i); byte[] data = encodeSchema.encode(foo); log.info("Decoding : {}", new String(data, UTF_8)); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java index c1ae007e08389..29225163ee913 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java @@ -35,7 +35,7 @@ public class SampleAsyncProducerWithSchema { public static void main(String[] args) throws IOException { PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("http://localhost:8080").build(); - Producer producer = pulsarClient.newProducer(JSONSchema.of(JsonPojo.class,true)).topic("persistent://my-property/use/my-ns/my-topic") + Producer producer = pulsarClient.newProducer(JSONSchema.of(JsonPojo.class, true)).topic("persistent://my-property/use/my-ns/my-topic") .sendTimeout(3, TimeUnit.SECONDS).create(); List> futures = Lists.newArrayList(); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java index 465df1b5270d6..b0818f4084619 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java @@ -30,7 +30,7 @@ public static void main(String[] args) throws PulsarClientException, JsonProcess PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("http://localhost:8080").build(); - Consumer consumer = pulsarClient.newConsumer(JSONSchema.of(JsonPojo.class,true)) // + Consumer consumer = pulsarClient.newConsumer(JSONSchema.of(JsonPojo.class, true)) // .topic("persistent://my-property/use/my-ns/my-topic") // .subscriptionName("my-subscription-name").subscribe(); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java index 5d065fd54279f..c98580604ddbd 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java @@ -124,10 +124,10 @@ private static Schema newSchemaInstance(Class clazz, SchemaType type) return (Schema) Schema.STRING; case AVRO: - return AvroSchema.of(clazz,true); + return AvroSchema.of(clazz, true); case JSON: - return JSONSchema.of(clazz,true); + return JSONSchema.of(clazz, true); case KEY_VALUE: return (Schema)Schema.KV_BYTES; From ed118b3aa419f4d490051ce45a3f5f0af1d4fe39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 6 Mar 2019 16:09:36 +0800 Subject: [PATCH 09/35] Fix the test define schema type AllowNull --- .../api/SimpleTypedProducerConsumerTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java index c9e919e06794e..b7e9b48b9e50f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java @@ -63,7 +63,7 @@ public void testJsonProducerAndConsumer() throws Exception { log.info("-- Starting {} test --", methodName); JSONSchema jsonSchema = - JSONSchema.of(JsonEncodedPojo.class); + JSONSchema.of(JsonEncodedPojo.class, true); Consumer consumer = pulsarClient .newConsumer(jsonSchema) @@ -108,7 +108,7 @@ public void testJsonProducerAndConsumerWithPrestoredSchema() throws Exception { log.info("-- Starting {} test --", methodName); JSONSchema jsonSchema = - JSONSchema.of(JsonEncodedPojo.class); + JSONSchema.of(JsonEncodedPojo.class, true); pulsar.getSchemaRegistryService() .putSchemaIfAbsent("my-property/my-ns/my-topic1", @@ -166,7 +166,7 @@ public void testJsonConsumerWithWrongCorruptedSchema() throws Exception { ).get(); Consumer consumer = pulsarClient - .newConsumer(JSONSchema.of(JsonEncodedPojo.class)) + .newConsumer(JSONSchema.of(JsonEncodedPojo.class, true)) .topic("persistent://my-property/use/my-ns/my-topic1") .subscriptionName("my-subscriber-name") .subscribe(); @@ -194,7 +194,7 @@ public void testJsonProducerWithWrongCorruptedSchema() throws Exception { ).get(); Producer producer = pulsarClient - .newProducer(JSONSchema.of(JsonEncodedPojo.class)) + .newProducer(JSONSchema.of(JsonEncodedPojo.class, true)) .topic("persistent://my-property/use/my-ns/my-topic1") .create(); @@ -273,7 +273,7 @@ public void testProtobufConsumerWithWrongPrestoredSchema() throws Exception { ).get(); Consumer consumer = pulsarClient - .newConsumer(AvroSchema.of(org.apache.pulsar.client.api.schema.proto.Test.TestMessageWrong.class)) + .newConsumer(AvroSchema.of(org.apache.pulsar.client.api.schema.proto.Test.TestMessageWrong.class, true)) .topic("persistent://my-property/use/my-ns/my-topic1") .subscriptionName("my-subscriber-name") .subscribe(); @@ -286,7 +286,7 @@ public void testAvroProducerAndConsumer() throws Exception { log.info("-- Starting {} test --", methodName); AvroSchema avroSchema = - AvroSchema.of(AvroEncodedPojo.class); + AvroSchema.of(AvroEncodedPojo.class, true); Consumer consumer = pulsarClient .newConsumer(avroSchema) @@ -355,7 +355,7 @@ public void testAvroConsumerWithWrongPrestoredSchema() throws Exception { ).get(); Consumer consumer = pulsarClient - .newConsumer(AvroSchema.of(AvroEncodedPojo.class)) + .newConsumer(AvroSchema.of(AvroEncodedPojo.class, true)) .topic("persistent://my-property/use/my-ns/my-topic1") .subscriptionName("my-subscriber-name") .subscribe(); @@ -454,7 +454,7 @@ public void testAvroProducerAndAutoSchemaConsumer() throws Exception { log.info("-- Starting {} test --", methodName); AvroSchema avroSchema = - AvroSchema.of(AvroEncodedPojo.class); + AvroSchema.of(AvroEncodedPojo.class, true); Producer producer = pulsarClient .newProducer(avroSchema) @@ -502,7 +502,7 @@ public void testAvroProducerAndAutoSchemaReader() throws Exception { log.info("-- Starting {} test --", methodName); AvroSchema avroSchema = - AvroSchema.of(AvroEncodedPojo.class); + AvroSchema.of(AvroEncodedPojo.class, true); Producer producer = pulsarClient .newProducer(avroSchema) @@ -548,7 +548,7 @@ public void testAutoBytesProducer() throws Exception { log.info("-- Starting {} test --", methodName); AvroSchema avroSchema = - AvroSchema.of(AvroEncodedPojo.class); + AvroSchema.of(AvroEncodedPojo.class, true); try (Producer producer = pulsarClient .newProducer(avroSchema) From 307023450ccfeeae7ce89a1f81ef0fa9735563d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 6 Mar 2019 16:20:54 +0800 Subject: [PATCH 10/35] fix the test schema type allow null --- .../service/schema/JsonSchemaCompatibilityCheckTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java index 17b39b79677cc..89506d6ce61cc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java @@ -51,11 +51,11 @@ public SchemaCompatibilityCheck getSchemaCheck() { public void testJsonSchemaBackwardsCompatibility() throws JsonProcessingException { SchemaData from = SchemaData.builder().data(OldJSONSchema.of(Foo.class).getSchemaInfo().getSchema()).build(); - SchemaData to = SchemaData.builder().data(JSONSchema.of(Foo.class).getSchemaInfo().getSchema()).build(); + SchemaData to = SchemaData.builder().data(JSONSchema.of(Foo.class,true).getSchemaInfo().getSchema()).build(); JsonSchemaCompatibilityCheck jsonSchemaCompatibilityCheck = new JsonSchemaCompatibilityCheck(); Assert.assertTrue(jsonSchemaCompatibilityCheck.isCompatible(from, to, SchemaCompatibilityStrategy.FULL)); - from = SchemaData.builder().data(JSONSchema.of(Foo.class).getSchemaInfo().getSchema()).build(); + from = SchemaData.builder().data(JSONSchema.of(Foo.class,true).getSchemaInfo().getSchema()).build(); to = SchemaData.builder().data(OldJSONSchema.of(Foo.class).getSchemaInfo().getSchema()).build(); Assert.assertTrue(jsonSchemaCompatibilityCheck.isCompatible(from, to, SchemaCompatibilityStrategy.FULL)); } From 4300123ccc236ff40bd0d3d510bb96d4c6c59985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 6 Mar 2019 16:50:08 +0800 Subject: [PATCH 11/35] fix the test schema define AllowNull --- .../sql/presto/TestPulsarConnector.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index b6eade22da821..0bf084ed34cc4 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -219,21 +219,21 @@ public static class Boo { partitionedTopicsToPartitions.put(PARTITIONED_TOPIC_6.toString(), 7); topicsToSchemas = new HashMap<>(); - topicsToSchemas.put(TOPIC_1.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(TOPIC_2.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(TOPIC_3.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(TOPIC_4.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(TOPIC_5.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(TOPIC_6.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class).getSchemaInfo()); + topicsToSchemas.put(TOPIC_1.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); + topicsToSchemas.put(TOPIC_2.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); + topicsToSchemas.put(TOPIC_3.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); + topicsToSchemas.put(TOPIC_4.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); + topicsToSchemas.put(TOPIC_5.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); + topicsToSchemas.put(TOPIC_6.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_1.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_1.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_2.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_3.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_4.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_5.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_6.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_2.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_3.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_4.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_5.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_6.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); fooTypes = new HashMap<>(); fooTypes.put("field1", IntegerType.INTEGER); @@ -622,7 +622,7 @@ private static List getTopicEntries(String topicSchemaName) { .setProducerName("test-producer").setSequenceId(i) .setPublishTime(currentTimeMs + i).build(); - Schema schema = topicsToSchemas.get(topicSchemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class) : JSONSchema.of(Foo.class); + Schema schema = topicsToSchemas.get(topicSchemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class, true) : JSONSchema.of(Foo.class, true); org.apache.pulsar.shade.io.netty.buffer.ByteBuf payload = org.apache.pulsar.shade.io.netty.buffer.Unpooled .copiedBuffer(schema.encode(foo)); @@ -872,7 +872,7 @@ public void run() { .setProducerName("test-producer").setSequenceId(positions.get(topic)) .setPublishTime(System.currentTimeMillis()).build(); - Schema schema = topicsToSchemas.get(schemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class) : JSONSchema.of(Foo.class); + Schema schema = topicsToSchemas.get(schemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class,true) : JSONSchema.of(Foo.class,true); org.apache.pulsar.shade.io.netty.buffer.ByteBuf payload = org.apache.pulsar.shade.io.netty.buffer.Unpooled .copiedBuffer(schema.encode(foo)); From 82c23dd6c86123061a656a458f5c33b2b1dc448b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 6 Mar 2019 17:12:48 +0800 Subject: [PATCH 12/35] Fix the test schema type allow null --- .../apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java | 2 +- .../src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java | 2 +- .../pulsar/tests/integration/functions/PulsarFunctionsTest.java | 2 +- .../org/apache/pulsar/tests/integration/io/JdbcSinkTester.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java b/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java index 0478775bd05dd..9ddf7b899c675 100644 --- a/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java +++ b/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java @@ -108,7 +108,7 @@ public void TestOpenAndWriteSink() throws Exception { obj.setAddress("address_value"); obj.setAge(30); obj.setFlag(true); - AvroSchema schema = AvroSchema.of(Foo.class); + AvroSchema schema = AvroSchema.of(Foo.class, true); byte[] bytes = schema.encode(obj); ByteBuf payload = Unpooled.copiedBuffer(bytes); diff --git a/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java b/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java index 7cbaa57714c18..6a9e7a6f9e601 100644 --- a/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java +++ b/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java @@ -95,7 +95,7 @@ public void TestOpenAndWriteSink() throws Exception { obj.setField1("ValueOfField1"); obj.setField2("ValueOfField1"); obj.setField3(3); - AvroSchema schema = AvroSchema.of(Foo.class); + AvroSchema schema = AvroSchema.of(Foo.class, true); byte[] bytes = schema.encode(obj); ByteBuf payload = Unpooled.copiedBuffer(bytes); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 18eaba945fb88..fb378d30f5601 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -161,7 +161,7 @@ private void runSinkTester(SinkTester tester, boolean builtin) throws Exception // produce messages Map kvs; if (tester instanceof JdbcSinkTester) { - kvs = produceSchemaMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcSinkTester.Foo.class)); + kvs = produceSchemaMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcSinkTester.Foo.class, true)); } else { kvs = produceMessagesToInputTopic(inputTopicName, numMessages); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java index 72d9b0152c01b..c7b0ec75bb1a7 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java @@ -60,7 +60,7 @@ public static class Foo { private static final String NAME = "jdbc"; private static final String MYSQL = "mysql"; - private AvroSchema schema = AvroSchema.of(Foo.class); + private AvroSchema schema = AvroSchema.of(Foo.class, true); private String tableName = "test"; private Connection connection; From aa6d36864096dc0c4d07f6440a50e694f2dce05d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Wed, 6 Mar 2019 17:39:40 +0800 Subject: [PATCH 13/35] fix test schema type allow null --- .../apache/pulsar/tests/integration/presto/TestBasicPresto.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java index 48cffb7022022..1de96c127fd75 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java @@ -86,7 +86,7 @@ public void testSimpleSQLQuery(boolean isBatched) throws Exception { final String stocksTopic = "stocks"; @Cleanup - Producer producer = pulsarClient.newProducer(JSONSchema.of(Stock.class)) + Producer producer = pulsarClient.newProducer(JSONSchema.of(Stock.class, true)) .topic(stocksTopic) .enableBatching(isBatched) .create(); From 79167ec9238e89ed7412a1ce5f9e13708b6d7acd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 7 Mar 2019 18:33:13 +0800 Subject: [PATCH 14/35] taking an object with the class extraction configuration --- .../service/schema/ClientGetSchemaTest.java | 8 + .../JsonSchemaCompatibilityCheckTest.java | 5 +- .../api/SimpleTypedProducerConsumerTest.java | 21 ++- .../org/apache/pulsar/client/api/Schema.java | 61 +++--- .../api/schema/KeyValueSchemaDefinition.java | 177 ++++++++++++++++++ .../client/api/schema/SchemaDefinition.java | 104 ++++++++++ .../internal/DefaultImplementation.java | 28 ++- .../pulsar/client/impl/schema/AvroSchema.java | 30 ++- .../pulsar/client/impl/schema/JSONSchema.java | 23 +-- .../client/impl/schema/KeyValueSchema.java | 39 ++-- .../client/impl/schema/AvroSchemaTest.java | 9 +- .../client/impl/schema/JSONSchemaTest.java | 33 ++-- .../impl/schema/KeyValueSchemaTest.java | 38 ++-- .../SampleAsyncProducerWithSchema.java | 3 +- .../tutorial/SampleConsumerWithSchema.java | 3 +- .../pulsar/functions/source/TopicSchema.java | 5 +- .../sink/HbaseGenericRecordSinkTest.java | 3 +- .../apache/pulsar/io/jdbc/JdbcSinkTest.java | 3 +- .../sql/presto/TestPulsarConnector.java | 29 +-- .../functions/PulsarFunctionsTest.java | 3 +- .../tests/integration/io/JdbcSinkTester.java | 3 +- .../integration/presto/TestBasicPresto.java | 3 +- 22 files changed, 460 insertions(+), 171 deletions(-) create mode 100644 pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/KeyValueSchemaDefinition.java create mode 100644 pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java index 70554926c9dc0..732a917a0acaf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java @@ -31,6 +31,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -43,6 +44,8 @@ public class ClientGetSchemaTest extends ProducerConsumerBase { private static final String topicString = "my-property/my-ns/topic-string"; private static final String topicJson = "my-property/my-ns/topic-json"; private static final String topicAvro = "my-property/my-ns/topic-avro"; + private static final String topicJsonNotNull = "my-property/my-ns/topic-json-not-null"; + private static final String topicAvroNotNull = "my-property/my-ns/topic-avro-not-null"; List> producers = new ArrayList<>(); @@ -62,6 +65,11 @@ protected void setup() throws Exception { producers.add(pulsarClient.newProducer(Schema.STRING).topic(topicString).create()); producers.add(pulsarClient.newProducer(Schema.AVRO(MyClass.class)).topic(topicAvro).create()); producers.add(pulsarClient.newProducer(Schema.JSON(MyClass.class)).topic(topicJson).create()); + producers.add(pulsarClient.newProducer(Schema.AVRO(new SchemaDefinition<>(MyClass.class))).topic(topicAvro).create()); + producers.add(pulsarClient.newProducer(Schema.JSON(new SchemaDefinition<>(MyClass.class))).topic(topicJson).create()); + producers.add(pulsarClient.newProducer(Schema.AVRO(new SchemaDefinition<>(MyClass.class).alwaysNull(false))).topic(topicAvroNotNull).create()); + producers.add(pulsarClient.newProducer(Schema.JSON(new SchemaDefinition<>(MyClass.class).alwaysNull(false))).topic(topicJsonNotNull).create()); + } @AfterClass diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java index 89506d6ce61cc..24786ca464ed5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java @@ -33,6 +33,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SchemaSerializationException; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.common.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaInfo; @@ -51,11 +52,11 @@ public SchemaCompatibilityCheck getSchemaCheck() { public void testJsonSchemaBackwardsCompatibility() throws JsonProcessingException { SchemaData from = SchemaData.builder().data(OldJSONSchema.of(Foo.class).getSchemaInfo().getSchema()).build(); - SchemaData to = SchemaData.builder().data(JSONSchema.of(Foo.class,true).getSchemaInfo().getSchema()).build(); + SchemaData to = SchemaData.builder().data(JSONSchema.of(new SchemaDefinition<>(Foo.class)).getSchemaInfo().getSchema()).build(); JsonSchemaCompatibilityCheck jsonSchemaCompatibilityCheck = new JsonSchemaCompatibilityCheck(); Assert.assertTrue(jsonSchemaCompatibilityCheck.isCompatible(from, to, SchemaCompatibilityStrategy.FULL)); - from = SchemaData.builder().data(JSONSchema.of(Foo.class,true).getSchemaInfo().getSchema()).build(); + from = SchemaData.builder().data(JSONSchema.of(new SchemaDefinition<>(Foo.class)).getSchemaInfo().getSchema()).build(); to = SchemaData.builder().data(OldJSONSchema.of(Foo.class).getSchemaInfo().getSchema()).build(); Assert.assertTrue(jsonSchemaCompatibilityCheck.isCompatible(from, to, SchemaCompatibilityStrategy.FULL)); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java index b7e9b48b9e50f..4b1742b77f305 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java @@ -30,6 +30,7 @@ import org.apache.pulsar.broker.service.schema.SchemaCompatibilityStrategy; import org.apache.pulsar.broker.service.schema.SchemaRegistry; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.client.impl.schema.ProtobufSchema; @@ -63,7 +64,7 @@ public void testJsonProducerAndConsumer() throws Exception { log.info("-- Starting {} test --", methodName); JSONSchema jsonSchema = - JSONSchema.of(JsonEncodedPojo.class, true); + JSONSchema.of(new SchemaDefinition<>(JsonEncodedPojo.class)); Consumer consumer = pulsarClient .newConsumer(jsonSchema) @@ -108,7 +109,7 @@ public void testJsonProducerAndConsumerWithPrestoredSchema() throws Exception { log.info("-- Starting {} test --", methodName); JSONSchema jsonSchema = - JSONSchema.of(JsonEncodedPojo.class, true); + JSONSchema.of(new SchemaDefinition<>(JsonEncodedPojo.class)); pulsar.getSchemaRegistryService() .putSchemaIfAbsent("my-property/my-ns/my-topic1", @@ -166,7 +167,7 @@ public void testJsonConsumerWithWrongCorruptedSchema() throws Exception { ).get(); Consumer consumer = pulsarClient - .newConsumer(JSONSchema.of(JsonEncodedPojo.class, true)) + .newConsumer(JSONSchema.of(new SchemaDefinition<>(JsonEncodedPojo.class))) .topic("persistent://my-property/use/my-ns/my-topic1") .subscriptionName("my-subscriber-name") .subscribe(); @@ -194,7 +195,7 @@ public void testJsonProducerWithWrongCorruptedSchema() throws Exception { ).get(); Producer producer = pulsarClient - .newProducer(JSONSchema.of(JsonEncodedPojo.class, true)) + .newProducer(JSONSchema.of(new SchemaDefinition<>(JsonEncodedPojo.class))) .topic("persistent://my-property/use/my-ns/my-topic1") .create(); @@ -273,7 +274,7 @@ public void testProtobufConsumerWithWrongPrestoredSchema() throws Exception { ).get(); Consumer consumer = pulsarClient - .newConsumer(AvroSchema.of(org.apache.pulsar.client.api.schema.proto.Test.TestMessageWrong.class, true)) + .newConsumer(AvroSchema.of(new SchemaDefinition<>(org.apache.pulsar.client.api.schema.proto.Test.TestMessageWrong.class))) .topic("persistent://my-property/use/my-ns/my-topic1") .subscriptionName("my-subscriber-name") .subscribe(); @@ -286,7 +287,7 @@ public void testAvroProducerAndConsumer() throws Exception { log.info("-- Starting {} test --", methodName); AvroSchema avroSchema = - AvroSchema.of(AvroEncodedPojo.class, true); + AvroSchema.of(new SchemaDefinition<>(AvroEncodedPojo.class)); Consumer consumer = pulsarClient .newConsumer(avroSchema) @@ -355,7 +356,7 @@ public void testAvroConsumerWithWrongPrestoredSchema() throws Exception { ).get(); Consumer consumer = pulsarClient - .newConsumer(AvroSchema.of(AvroEncodedPojo.class, true)) + .newConsumer(AvroSchema.of(new SchemaDefinition<>(AvroEncodedPojo.class))) .topic("persistent://my-property/use/my-ns/my-topic1") .subscriptionName("my-subscriber-name") .subscribe(); @@ -454,7 +455,7 @@ public void testAvroProducerAndAutoSchemaConsumer() throws Exception { log.info("-- Starting {} test --", methodName); AvroSchema avroSchema = - AvroSchema.of(AvroEncodedPojo.class, true); + AvroSchema.of(new SchemaDefinition<>(AvroEncodedPojo.class)); Producer producer = pulsarClient .newProducer(avroSchema) @@ -502,7 +503,7 @@ public void testAvroProducerAndAutoSchemaReader() throws Exception { log.info("-- Starting {} test --", methodName); AvroSchema avroSchema = - AvroSchema.of(AvroEncodedPojo.class, true); + AvroSchema.of(new SchemaDefinition<>(AvroEncodedPojo.class)); Producer producer = pulsarClient .newProducer(avroSchema) @@ -548,7 +549,7 @@ public void testAutoBytesProducer() throws Exception { log.info("-- Starting {} test --", methodName); AvroSchema avroSchema = - AvroSchema.of(AvroEncodedPojo.class, true); + AvroSchema.of(new SchemaDefinition<>(AvroEncodedPojo.class)); try (Producer producer = pulsarClient .newProducer(avroSchema) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java index ebcaa3ad038d0..e2a90607c3298 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java @@ -21,6 +21,8 @@ import java.nio.ByteBuffer; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.GenericSchema; +import org.apache.pulsar.client.api.schema.KeyValueSchemaDefinition; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.internal.DefaultImplementation; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaInfo; @@ -167,46 +169,44 @@ static Schema PROTOBUF(Cla } /** - * Create a allow null Avro schema type by extracting the fields of the specified class. + * Create a Avro schema type by default configuration of the class * * @param clazz the POJO class to be used to extract the Avro schema * @return a Schema instance */ static Schema AVRO(Class clazz) { - return DefaultImplementation.newAvroSchema(clazz, true); + return DefaultImplementation.newAvroSchema(new SchemaDefinition<>(clazz)); } /** - * Create a Avro schema type by extracting the fields of the specified class. + * Create a Avro schema type with schema definition * - * @param clazz the POJO class to be used to extract the Avro schema - * allowNull the create a allow null or not null Avro schema + * @param schemaDefinition the definition of the schema * @return a Schema instance */ - static Schema AVRO(Class clazz, Boolean allowNull) { - return DefaultImplementation.newAvroSchema(clazz, allowNull); + static Schema AVRO(SchemaDefinition schemaDefinition) { + return DefaultImplementation.newAvroSchema(schemaDefinition); } /** - * Create a allow null JSON schema type by extracting the fields of the specified class. + * Create a JSON schema type by default configuration of the class * * @param clazz the POJO class to be used to extract the JSON schema * @return a Schema instance */ static Schema JSON(Class clazz) { - return DefaultImplementation.newJSONSchema(clazz, true); + return DefaultImplementation.newJSONSchema(new SchemaDefinition<>(clazz)); } /** - * Create a allow null JSON schema type by extracting the fields of the specified class. + * Create a JSON schema type with schema definition * - * @param clazz the POJO class to be used to extract the JSON schema - * allowNull the create a allow null or not null JSON schema + * @param schemaDefinition the definition of the schema * @return a Schema instance */ - static Schema JSON(Class clazz, Boolean allowNull) { - return DefaultImplementation.newJSONSchema(clazz, allowNull); + static Schema JSON(SchemaDefinition schemaDefinition) { + return DefaultImplementation.newJSONSchema(schemaDefinition); } @@ -214,48 +214,33 @@ static Schema JSON(Class clazz, Boolean allowNull) { * Key Value Schema using passed in schema type, support JSON and AVRO currently. */ static Schema> KeyValue(Class key, Class value, SchemaType type) { - return DefaultImplementation.newKeyValueSchema(key, value, type, true); + return DefaultImplementation.newKeyValueSchema(new KeyValueSchemaDefinition<>(key, value, type)); } - /** - * Key Value Schema using passed in schema type, support JSON and AVRO currently. - */ - static Schema> KeyValue(Class key, Class value, SchemaType type, Boolean allowNull) { - return DefaultImplementation.newKeyValueSchema(key, value, type, allowNull); - } - - /** * Schema that can be used to encode/decode KeyValue. */ - Schema> KV_BYTES = DefaultImplementation.newKeyValueSchema(BYTES, BYTES, true); + Schema> KV_BYTES = DefaultImplementation.newKeyValueSchema(new KeyValueSchemaDefinition<>(BYTES,BYTES)); /** * Key Value Schema whose underneath key and value schemas are JSONSchema. */ static Schema> KeyValue(Class key, Class value) { - return DefaultImplementation.newKeyValueSchema(key, value, SchemaType.JSON, true); + return DefaultImplementation.newKeyValueSchema(new KeyValueSchemaDefinition<>(key, value, SchemaType.JSON)); } /** * Key Value Schema using passed in key and value schemas. */ static Schema> KeyValue(Schema key, Schema value) { - return DefaultImplementation.newKeyValueSchema(key, value, true); - } - - /** - * Key Value Schema whose underneath key and value schemas are JSONSchema. - */ - static Schema> KeyValue(Class key, Class value, Boolean allowNull) { - return DefaultImplementation.newKeyValueSchema(key, value, SchemaType.JSON, allowNull); + return DefaultImplementation.newKeyValueSchema(new KeyValueSchemaDefinition<>(key, value)); } /** - * Key Value Schema using passed in key and value schemas. + * Create key and value schema by keyValueSchemaDefinition. */ - static Schema> KeyValue(Schema key, Schema value, Boolean allowNull) { - return DefaultImplementation.newKeyValueSchema(key, value, allowNull); + static Schema> KeyValue(KeyValueSchemaDefinition keyValueSchemaDefinition) { + return DefaultImplementation.newKeyValueSchema(keyValueSchemaDefinition); } @Deprecated @@ -266,9 +251,9 @@ static Schema AUTO() { /** * Create a schema instance that automatically deserialize messages * based on the current topic schema. - *

+ * * The messages values are deserialized into a {@link GenericRecord} object. - *

+ * * Currently this is only supported with Avro and JSON schema types. * * @return the auto schema instance diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/KeyValueSchemaDefinition.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/KeyValueSchemaDefinition.java new file mode 100644 index 0000000000000..7c465f148f848 --- /dev/null +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/KeyValueSchemaDefinition.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api.schema; + + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.schema.SchemaType; + +import java.util.HashMap; +import java.util.Map; + +/** + * A key and value schema definition + * {@link org.apache.pulsar.client.api.Schema} for the key and value schema definition value. + */ +@Data +@EqualsAndHashCode +@ToString +public class KeyValueSchemaDefinition { + + private Class key; + + private Class value; + + private boolean alwaysNull = true; + + private SchemaType type; + + private Schema keySchema; + + private Schema valueSchema; + + private Map properties = new HashMap<>(); + + public KeyValueSchemaDefinition() { + + } + + public KeyValueSchemaDefinition(Schema keySchema, Schema valueSchema) { + this.keySchema = keySchema; + this.valueSchema = valueSchema; + } + + public KeyValueSchemaDefinition(Class key, Class value, SchemaType type) { + this.key = key; + this.value = value; + this.type = type; + } + + public KeyValueSchemaDefinition(Class key, Class value, boolean allowNull) { + this.key = key; + this.value = value; + this.alwaysNull = allowNull; + this.type = SchemaType.JSON; + } + + /** + * Get key schema class + * + * @return key schema class + */ + public Class getKey() { + + return key; + } + + /** + * Get value schema class + * + * @return value schema class + */ + public Class getValue() { + + return value; + } + + /** + * Set schema whether always null or not + * + * @param alwaysNull definition null or not + * @return Key and value schema definition + */ + public KeyValueSchemaDefinition alwaysNull(boolean alwaysNull) { + + this.alwaysNull = alwaysNull; + properties.put("alwaysNull", alwaysNull ? "true" : "false"); + return this; + } + + /** + * Set key schema + * + * @param keySchema definition key schema + * @return Key and value schema definition + */ + public KeyValueSchemaDefinition keySchema(Schema keySchema) { + + this.keySchema = keySchema; + return this; + } + + /** + * Set value schema + * + * @param valueSchema definition null or not + * @return Key and value schema definition + */ + public KeyValueSchemaDefinition valueSchema(Schema valueSchema) { + + this.valueSchema = valueSchema; + return this; + } + + /** + * get schema whether always null or not + * + * @return schema always null or not + */ + public boolean getAlwaysNull() { + + return alwaysNull; + } + + /** + * Set schema type + * + * @param type the schema type + * @return Key and value schema definition + */ + public KeyValueSchemaDefinition type(SchemaType type) { + + this.type = type; + return this; + } + + /** + * Get schema type + * + * @return Schema type + */ + public SchemaType getType() { + + return type; + } + + /** + * Set schema info properties + * + * @param properties schema info properties + * @return record schema definition + */ + public KeyValueSchemaDefinition properties(Map properties) { + + this.properties = properties; + properties.put("alwaysNull", alwaysNull ? "true" : "false"); + return this; + } +} diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java new file mode 100644 index 0000000000000..4adccb9aab6f0 --- /dev/null +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api.schema; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +import java.util.HashMap; +import java.util.Map; + +/** + * A schema definition + * {@link org.apache.pulsar.client.api.Schema} for the schema definition value. + */ +@Data +@EqualsAndHashCode +@ToString +public class SchemaDefinition { + + /** + * the schema definition class + */ + private final Class clazz; + /** + * The flag of schema type always null + */ + private boolean alwaysNull = true; + /** + * The schema info properties + */ + private Map properties = new HashMap<>(); + + + public SchemaDefinition(Class clazz) { + + properties.put("alwaysNull", "true"); + this.clazz = clazz; + + } + + /** + * Set schema whether always null or not + * + * @param alwaysNull definition null or not + * @return record schema definition + */ + public SchemaDefinition alwaysNull(boolean alwaysNull) { + + this.alwaysNull = alwaysNull; + properties.put("alwaysNull", alwaysNull ? "true" : "false"); + return this; + } + + /** + * get schema whether always null or not + * + * @return schema always null or not + */ + public boolean getAlwaysNull() { + + return alwaysNull; + } + + /** + * Get schema class + * + * @return schema class + */ + public Class getClazz() { + + return clazz; + } + + + /** + * Set schema info properties + * + * @param properties schema info properties + * @return record schema definition + */ + public SchemaDefinition properties(Map properties) { + + this.properties = properties; + properties.put("alwaysNull", alwaysNull ? "true" : "false"); + return this; + } +} diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java index 51eca6c717bea..5b35a43f58a4a 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java @@ -37,12 +37,13 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.PulsarClientException.UnsupportedAuthenticationException; -import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.GenericSchema; +import org.apache.pulsar.client.api.schema.KeyValueSchemaDefinition; import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; +import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; @SuppressWarnings("unchecked") @UtilityClass @@ -182,10 +183,10 @@ public static Schema newDoubleSchema() { .newInstance()); } - public static Schema newAvroSchema(Class clazz, Boolean allowNull) { + public static Schema newAvroSchema(SchemaDefinition schemaDefinition) { return catchExceptions( - () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.AvroSchema", "of", Class.class, Boolean.class) - .invoke(null, clazz, allowNull)); + () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.AvroSchema", "of", SchemaDefinition.class) + .invoke(null,schemaDefinition)); } public static Schema newProtobufSchema(Class clazz) { @@ -194,10 +195,10 @@ public static Schema newPr .invoke(null, clazz)); } - public static Schema newJSONSchema(Class clazz, Boolean allowNull) { + public static Schema newJSONSchema(SchemaDefinition schemaDefinition) { return catchExceptions( - () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.JSONSchema", "of", Class.class, Boolean.class) - .invoke(null, clazz, allowNull)); + () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.JSONSchema", "of", SchemaDefinition.class) + .invoke(null, schemaDefinition)); } public static Schema newAutoConsumeSchema() { @@ -212,19 +213,12 @@ public static Schema newAutoProduceSchema() { .newInstance()); } - public static Schema> newKeyValueSchema(Schema keySchema, Schema valueSchema, Boolean allowNull) { - return catchExceptions( - () -> (Schema>) getConstructor("org.apache.pulsar.client.impl.schema.KeyValueSchema", - Schema.class, Schema.class , Boolean.class).newInstance(keySchema, valueSchema, allowNull)); - } - - public static Schema> newKeyValueSchema(Class key, Class value, SchemaType type, Boolean allowNull) { + public static Schema> newKeyValueSchema(KeyValueSchemaDefinition keyValueSchemaDefinition) { return catchExceptions( () -> (Schema>) getStaticMethod("org.apache.pulsar.client.impl.schema.KeyValueSchema", - "of", Class.class, Class.class, SchemaType.class, Boolean.class).invoke(null, key, value, type, allowNull)); + "of", KeyValueSchemaDefinition.class).invoke(null, keyValueSchemaDefinition)); } - public static Schema getSchema(SchemaInfo schemaInfo) { return catchExceptions( () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.AutoConsumeSchema", diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index 963325a7ccfcc..1f7a5872a03c2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -30,13 +30,12 @@ import org.apache.avro.reflect.ReflectDatumWriter; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SchemaSerializationException; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; @Slf4j @@ -53,16 +52,11 @@ public class AvroSchema implements Schema { new ThreadLocal<>(); private AvroSchema(org.apache.avro.Schema schema, - Boolean allowNull, - Map properties) { - - properties.put("allowNull", allowNull ? "true" : "false"); - + SchemaDefinition schemaDefinition) { this.schema = schema; - this.schemaInfo = new SchemaInfo(); this.schemaInfo.setName(""); - this.schemaInfo.setProperties(properties); + this.schemaInfo.setProperties(schemaDefinition.getProperties()); this.schemaInfo.setType(SchemaType.AVRO); this.schemaInfo.setSchema(this.schema.toString().getBytes(UTF_8)); @@ -105,21 +99,21 @@ public SchemaInfo getSchemaInfo() { return this.schemaInfo; } - private static org.apache.avro.Schema createAvroSchema(Class pojo, Boolean allowNull) { + private static org.apache.avro.Schema createAvroSchema(SchemaDefinition schemaDefinition) { - return allowNull ? ReflectData.AllowNull.get().getSchema(pojo) : ReflectData.get().getSchema(pojo); - } + Class clazz = schemaDefinition.getClazz(); - public static AvroSchema of(Class pojo, Map properties) { - return new AvroSchema<>(createAvroSchema(pojo, true), true, new HashMap<>()); + return schemaDefinition.getAlwaysNull() ? ReflectData.AllowNull.get().getSchema(clazz) : ReflectData.get().getSchema(clazz); } - public static AvroSchema of(Class pojo, Boolean allowNull) { - return new AvroSchema<>(createAvroSchema(pojo, allowNull), allowNull, new HashMap<>()); + public static AvroSchema of(SchemaDefinition schemaDefinition) { + return new AvroSchema<>(createAvroSchema(schemaDefinition), schemaDefinition); } - public static AvroSchema of(Class pojo, Boolean allowNull, Map properties) { - return new AvroSchema<>(createAvroSchema(pojo, allowNull), true, properties); + public static AvroSchema of(Class pojo, Map properties) { + SchemaDefinition schemaDefinition = new SchemaDefinition<>(pojo).properties(properties); + return new AvroSchema<>(createAvroSchema(schemaDefinition), schemaDefinition); } + } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java index b0aa1f2a862f7..311cb559b8f65 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java @@ -30,12 +30,11 @@ import org.apache.avro.reflect.ReflectData; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SchemaSerializationException; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; @Slf4j @@ -57,15 +56,15 @@ public class JSONSchema implements Schema { private final ObjectMapper objectMapper; - private JSONSchema(Class pojo, Boolean allowNull, Map properties) { - this.pojo = pojo; - this.properties = properties; + private JSONSchema(SchemaDefinition schemaDefinition) { + this.pojo = schemaDefinition.getClazz(); + this.properties = schemaDefinition.getProperties(); + boolean alwaysNull = schemaDefinition.getAlwaysNull(); - this.schema = allowNull ? ReflectData.AllowNull.get().getSchema(pojo) : ReflectData.get().getSchema(pojo); + this.schema = alwaysNull ? ReflectData.AllowNull.get().getSchema(pojo) : ReflectData.get().getSchema(pojo); this.schemaInfo = new SchemaInfo(); this.schemaInfo.setName(""); this.schemaInfo.setProperties(properties); - this.schemaInfo.getProperties().put("allowNull", allowNull ? "true" : "false"); this.schemaInfo.setType(SchemaType.JSON); this.schemaInfo.setSchema(this.schema.toString().getBytes(UTF_8)); this.objectMapper = JSON_MAPPER.get(); @@ -98,6 +97,7 @@ public SchemaInfo getSchemaInfo() { * Implemented for backwards compatibility reasons * since the original schema generated by JSONSchema was based off the json schema standard * since then we have standardized on Avro + * * @return */ public SchemaInfo getBackwardsCompatibleJsonSchemaInfo() { @@ -117,11 +117,12 @@ public SchemaInfo getBackwardsCompatibleJsonSchemaInfo() { return backwardsCompatibleSchemaInfo; } - public static JSONSchema of(Class pojo, Boolean allowNull) { - return new JSONSchema<>(pojo, allowNull, new HashMap<>()); + public static JSONSchema of(SchemaDefinition schemaDefinition) { + return new JSONSchema<>(schemaDefinition); } - public static JSONSchema of(Class pojo, Boolean allowNull, Map properties) { - return new JSONSchema<>(pojo, allowNull, properties); + public static JSONSchema of(Class pojo, Map properties) { + SchemaDefinition schemaDefinition = new SchemaDefinition<>(pojo).properties(properties); + return new JSONSchema<>(schemaDefinition); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java index 4c6133186de04..184453f60181e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java @@ -26,6 +26,8 @@ import lombok.Getter; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.schema.KeyValueSchemaDefinition; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -46,37 +48,46 @@ public class KeyValueSchema implements Schema> { /** * Key Value Schema using passed in schema type, support JSON and AVRO currently. */ - public static Schema> of(Class key, Class value, SchemaType type, Boolean allowNull) { - checkArgument(SchemaType.JSON == type || SchemaType.AVRO == type); + public static Schema> of(KeyValueSchemaDefinition keyValueSchemaDefinition) { + SchemaType type = keyValueSchemaDefinition.getType(); + Class key = keyValueSchemaDefinition.getKey(); + Class value = keyValueSchemaDefinition.getValue(); + boolean alwaysNull = keyValueSchemaDefinition.getAlwaysNull(); + checkArgument((SchemaType.JSON == type || SchemaType.AVRO == type) || + (keyValueSchemaDefinition.getKeySchema() != null && keyValueSchemaDefinition.getValueSchema() != null)); if (SchemaType.JSON == type) { - return new KeyValueSchema<>(JSONSchema.of(key, allowNull), JSONSchema.of(value, allowNull), allowNull); - } else { + return new KeyValueSchema<>(keyValueSchemaDefinition.keySchema(JSONSchema.of(new SchemaDefinition<>(key).alwaysNull(alwaysNull))). + valueSchema(JSONSchema.of(new SchemaDefinition<>(value).alwaysNull(alwaysNull)))); + } else if (SchemaType.AVRO == type) { // AVRO - return new KeyValueSchema<>(AvroSchema.of(key, allowNull), AvroSchema.of(value, allowNull), allowNull); + return new KeyValueSchema<>(keyValueSchemaDefinition.keySchema(AvroSchema.of(new SchemaDefinition<>(key).alwaysNull(alwaysNull))). + valueSchema(AvroSchema.of(new SchemaDefinition<>(value).alwaysNull(alwaysNull)))); + } else { + return new KeyValueSchema<>(keyValueSchemaDefinition); } } - public KeyValueSchema(Schema keySchema, - Schema valueSchema, Boolean allowNull) { - this.keySchema = keySchema; - this.valueSchema = valueSchema; + public KeyValueSchema(KeyValueSchemaDefinition keyValueSchemaDefinition) { + this.keySchema = keyValueSchemaDefinition.getKeySchema(); + this.valueSchema = keyValueSchemaDefinition.getValueSchema(); // set schemaInfo this.schemaInfo = new SchemaInfo() - .setName("KeyValue") - .setType(SchemaType.KEY_VALUE); - this.schemaInfo.setProperties(new HashMap<>()); - this.schemaInfo.getProperties().put("allowNull", allowNull ? "true" : "false"); + .setName("KeyValue") + .setType(SchemaType.KEY_VALUE); + + this.schemaInfo.setProperties(keyValueSchemaDefinition.getProperties()); byte[] keySchemaInfo = keySchema.getSchemaInfo().getSchema(); byte[] valueSchemaInfo = valueSchema.getSchemaInfo().getSchema(); ByteBuffer byteBuffer = ByteBuffer.allocate(4 + keySchemaInfo.length + 4 + valueSchemaInfo.length); byteBuffer.putInt(keySchemaInfo.length).put(keySchemaInfo) - .putInt(valueSchemaInfo.length).put(valueSchemaInfo); + .putInt(valueSchemaInfo.length).put(valueSchemaInfo); this.schemaInfo.setSchema(byteBuffer.array()); } + // encode as bytes: [key.length][key.bytes][value.length][value.bytes] public byte[] encode(KeyValue message) { byte[] keyBytes = keySchema.encode(message.getKey()); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index fc02e28adbe25..8f2d52f08be35 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -26,6 +26,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.pulsar.client.api.SchemaSerializationException; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Bar; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Foo; import org.apache.pulsar.common.schema.SchemaType; @@ -39,7 +40,7 @@ public class AvroSchemaTest { @Test public void testNotAllowNullSchema() { - AvroSchema avroSchema = AvroSchema.of(Foo.class, false); + AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); @@ -61,7 +62,7 @@ public void testNotAllowNullSchema() { @Test public void testAllowNullSchema() { - AvroSchema avroSchema = AvroSchema.of(Foo.class, true); + AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(true)); assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); @@ -83,7 +84,7 @@ public void testAllowNullSchema() { @Test public void testNotAllowNullEncodeAndDecode() { - AvroSchema avroSchema = AvroSchema.of(Foo.class, false); + AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); Foo foo1 = new Foo(); foo1.setField1("foo1"); @@ -112,7 +113,7 @@ public void testNotAllowNullEncodeAndDecode() { @Test public void testAllowNullEncodeAndDecode() { - AvroSchema avroSchema = AvroSchema.of(Foo.class, new HashMap<>()); + AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class)); Foo foo1 = new Foo(); foo1.setField1("foo1"); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java index fc62427c34ded..e37e8b7bc1325 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.pulsar.client.api.SchemaSerializationException; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Bar; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.DerivedFoo; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Foo; @@ -43,7 +44,7 @@ public class JSONSchemaTest { @Test public void testNotAllowNullSchema() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class, false); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema()); @@ -65,7 +66,7 @@ public void testNotAllowNullSchema() { @Test public void testAllowNullSchema() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class, true); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class)); Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema()); @@ -87,7 +88,7 @@ public void testAllowNullSchema() { @Test public void testAllowNullEncodeAndDecode() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class, true); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class)); Bar bar = new Bar(); bar.setField1(true); @@ -117,7 +118,7 @@ public void testAllowNullEncodeAndDecode() { @Test public void testNotAllowNullEncodeAndDecode() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class, false); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); Foo foo1 = new Foo(); foo1.setField1("foo1"); @@ -146,8 +147,8 @@ public void testNotAllowNullEncodeAndDecode() { @Test public void testAllowNullNestedClasses() { - JSONSchema jsonSchema = JSONSchema.of(NestedBar.class, true); - JSONSchema listJsonSchema = JSONSchema.of(NestedBarList.class, true); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBar.class)); + JSONSchema listJsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBarList.class)); Bar bar = new Bar(); bar.setField1(true); @@ -173,8 +174,8 @@ public void testAllowNullNestedClasses() { @Test public void testNotAllowNullNestedClasses() { - JSONSchema jsonSchema = JSONSchema.of(NestedBar.class, false); - JSONSchema listJsonSchema = JSONSchema.of(NestedBarList.class, false); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBar.class).alwaysNull(false)); + JSONSchema listJsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBarList.class).alwaysNull(false)); Bar bar = new Bar(); bar.setField1(true); @@ -228,25 +229,25 @@ public void testNotAllowNullCorrectPolymorphism() { derivedDerivedFoo.setDerivedFoo(derivedFoo); // schema for base class - JSONSchema baseJsonSchema = JSONSchema.of(Foo.class, true); + JSONSchema baseJsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class)); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(foo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedFoo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedDerivedFoo)), foo); // schema for derived class - JSONSchema derivedJsonSchema = JSONSchema.of(DerivedFoo.class, true); + JSONSchema derivedJsonSchema = JSONSchema.of(new SchemaDefinition<>(DerivedFoo.class)); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedFoo)), derivedFoo); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedDerivedFoo)), derivedFoo); //schema for derived derived class JSONSchema derivedDerivedJsonSchema - = JSONSchema.of(SchemaTestUtils.DerivedDerivedFoo.class, true); + = JSONSchema.of(new SchemaDefinition<>(SchemaTestUtils.DerivedDerivedFoo.class)); Assert.assertEquals(derivedDerivedJsonSchema.decode(derivedDerivedJsonSchema.encode(derivedDerivedFoo)), derivedDerivedFoo); } @Test(expectedExceptions = SchemaSerializationException.class) public void testAllowNullDecodeWithInvalidContent() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class, true); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class)); jsonSchema.decode(new byte[0]); } @@ -280,25 +281,25 @@ public void testAllowNullCorrectPolymorphism() { derivedDerivedFoo.setDerivedFoo(derivedFoo); // schema for base class - JSONSchema baseJsonSchema = JSONSchema.of(Foo.class, false); + JSONSchema baseJsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(foo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedFoo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedDerivedFoo)), foo); // schema for derived class - JSONSchema derivedJsonSchema = JSONSchema.of(DerivedFoo.class, false); + JSONSchema derivedJsonSchema = JSONSchema.of(new SchemaDefinition<>(DerivedFoo.class).alwaysNull(false)); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedFoo)), derivedFoo); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedDerivedFoo)), derivedFoo); //schema for derived derived class JSONSchema derivedDerivedJsonSchema - = JSONSchema.of(SchemaTestUtils.DerivedDerivedFoo.class, false); + = JSONSchema.of(new SchemaDefinition<>(SchemaTestUtils.DerivedDerivedFoo.class).alwaysNull(false)); Assert.assertEquals(derivedDerivedJsonSchema.decode(derivedDerivedJsonSchema.encode(derivedDerivedFoo)), derivedDerivedFoo); } @Test(expectedExceptions = SchemaSerializationException.class) public void testNotAllowNullDecodeWithInvalidContent() { - JSONSchema jsonSchema = JSONSchema.of(Foo.class, false); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); jsonSchema.decode(new byte[0]); } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java index ad2c8ab09ce8e..6bdc21d03f9ef 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java @@ -22,6 +22,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.schema.KeyValueSchemaDefinition; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.client.impl.schema.KeyValueSchema; @@ -38,8 +40,8 @@ public class KeyValueSchemaTest { @Test public void testAllowNullAvroSchemaCreate() { - AvroSchema fooSchema = AvroSchema.of(Foo.class, true); - AvroSchema barSchema = AvroSchema.of(Bar.class, true); + AvroSchema fooSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class)); + AvroSchema barSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class)); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.AVRO); @@ -63,11 +65,11 @@ public void testAllowNullAvroSchemaCreate() { @Test public void testNotAllowNullAvroSchemaCreate() { - AvroSchema fooSchema = AvroSchema.of(Foo.class, false); - AvroSchema barSchema = AvroSchema.of(Bar.class, false); + AvroSchema fooSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); + AvroSchema barSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysNull(false)); - Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema, false); - Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.AVRO, false); + Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); + Schema> keyValueSchema2 = Schema.KeyValue(new KeyValueSchemaDefinition<>(Foo.class,Bar.class,SchemaType.AVRO).alwaysNull(false)); assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); @@ -88,8 +90,8 @@ public void testNotAllowNullAvroSchemaCreate() { @Test public void testAllowNullJsonSchemaCreate() { - JSONSchema fooSchema = JSONSchema.of(Foo.class, true); - JSONSchema barSchema = JSONSchema.of(Bar.class, true); + JSONSchema fooSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class)); + JSONSchema barSchema = JSONSchema.of(new SchemaDefinition<>(Bar.class)); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.JSON); @@ -121,12 +123,12 @@ public void testAllowNullJsonSchemaCreate() { @Test public void testNotAllowNullJsonSchemaCreate() { - JSONSchema fooSchema = JSONSchema.of(Foo.class, false); - JSONSchema barSchema = JSONSchema.of(Bar.class, false); + JSONSchema fooSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); + JSONSchema barSchema = JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysNull(false)); - Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema, false); - Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.JSON, false); - Schema> keyValueSchema3 = Schema.KeyValue(Foo.class, Bar.class, false); + Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); + Schema> keyValueSchema2 = Schema.KeyValue(new KeyValueSchemaDefinition<>(Foo.class,Bar.class,false).type(SchemaType.JSON)); + Schema> keyValueSchema3 = Schema.KeyValue(new KeyValueSchemaDefinition<>(Foo.class,Bar.class,false)); assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); @@ -179,7 +181,7 @@ public void testAllowNullSchemaEncodeAndDecode() { @Test public void testNotAllowNullSchemaEncodeAndDecode() { - Schema keyValueSchema = Schema.KeyValue(Foo.class, Bar.class, false); + Schema keyValueSchema = Schema.KeyValue(new KeyValueSchemaDefinition<>(Foo.class,Bar.class,false)); Bar bar = new Bar(); bar.setField1(true); @@ -204,8 +206,8 @@ public void testNotAllowNullSchemaEncodeAndDecode() { @Test public void testAllowNullBytesSchemaEncodeAndDecode() { - AvroSchema fooAvroSchema = AvroSchema.of(Foo.class, true); - AvroSchema barAvroSchema = AvroSchema.of(Bar.class, true); + AvroSchema fooAvroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class)); + AvroSchema barAvroSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class)); Bar bar = new Bar(); bar.setField1(true); @@ -233,8 +235,8 @@ public void testAllowNullBytesSchemaEncodeAndDecode() { @Test public void testNotAllowNullBytesSchemaEncodeAndDecode() { - AvroSchema fooAvroSchema = AvroSchema.of(Foo.class, false); - AvroSchema barAvroSchema = AvroSchema.of(Bar.class, false); + AvroSchema fooAvroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); + AvroSchema barAvroSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysNull(false)); Bar bar = new Bar(); bar.setField1(true); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java index 29225163ee913..d148d2c2bad38 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java @@ -27,6 +27,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.JSONSchema; @Slf4j @@ -35,7 +36,7 @@ public class SampleAsyncProducerWithSchema { public static void main(String[] args) throws IOException { PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("http://localhost:8080").build(); - Producer producer = pulsarClient.newProducer(JSONSchema.of(JsonPojo.class, true)).topic("persistent://my-property/use/my-ns/my-topic") + Producer producer = pulsarClient.newProducer(JSONSchema.of(new SchemaDefinition<>(JsonPojo.class))).topic("persistent://my-property/use/my-ns/my-topic") .sendTimeout(3, TimeUnit.SECONDS).create(); List> futures = Lists.newArrayList(); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java index b0818f4084619..d851e171d4863 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java @@ -23,6 +23,7 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.JSONSchema; public class SampleConsumerWithSchema { @@ -30,7 +31,7 @@ public static void main(String[] args) throws PulsarClientException, JsonProcess PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("http://localhost:8080").build(); - Consumer consumer = pulsarClient.newConsumer(JSONSchema.of(JsonPojo.class, true)) // + Consumer consumer = pulsarClient.newConsumer(JSONSchema.of(new SchemaDefinition<>(JsonPojo.class))) // .topic("persistent://my-property/use/my-ns/my-topic") // .subscriptionName("my-subscription-name").subscribe(); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java index c98580604ddbd..ee47a0215a383 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java @@ -27,6 +27,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.client.impl.schema.JSONSchema; @@ -124,10 +125,10 @@ private static Schema newSchemaInstance(Class clazz, SchemaType type) return (Schema) Schema.STRING; case AVRO: - return AvroSchema.of(clazz, true); + return AvroSchema.of(new SchemaDefinition<>(clazz)); case JSON: - return JSONSchema.of(clazz, true); + return JSONSchema.of(new SchemaDefinition<>(clazz)); case KEY_VALUE: return (Schema)Schema.KV_BYTES; diff --git a/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java b/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java index 9ddf7b899c675..e23f4317144fe 100644 --- a/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java +++ b/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java @@ -31,6 +31,7 @@ import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.schema.AvroSchema; @@ -108,7 +109,7 @@ public void TestOpenAndWriteSink() throws Exception { obj.setAddress("address_value"); obj.setAge(30); obj.setFlag(true); - AvroSchema schema = AvroSchema.of(Foo.class, true); + AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(true)); byte[] bytes = schema.encode(obj); ByteBuf payload = Unpooled.copiedBuffer(bytes); diff --git a/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java b/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java index 6a9e7a6f9e601..be7a566288a3e 100644 --- a/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java +++ b/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java @@ -29,6 +29,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.schema.AvroSchema; @@ -95,7 +96,7 @@ public void TestOpenAndWriteSink() throws Exception { obj.setField1("ValueOfField1"); obj.setField2("ValueOfField1"); obj.setField3(3); - AvroSchema schema = AvroSchema.of(Foo.class, true); + AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(true)); byte[] bytes = schema.encode(obj); ByteBuf payload = Unpooled.copiedBuffer(bytes); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index 0bf084ed34cc4..522233c07df21 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -41,6 +41,7 @@ import org.apache.pulsar.client.admin.Tenants; import org.apache.pulsar.client.admin.Topics; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.common.api.Commands; @@ -219,21 +220,21 @@ public static class Boo { partitionedTopicsToPartitions.put(PARTITIONED_TOPIC_6.toString(), 7); topicsToSchemas = new HashMap<>(); - topicsToSchemas.put(TOPIC_1.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); - topicsToSchemas.put(TOPIC_2.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); - topicsToSchemas.put(TOPIC_3.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); - topicsToSchemas.put(TOPIC_4.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); - topicsToSchemas.put(TOPIC_5.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); - topicsToSchemas.put(TOPIC_6.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); + topicsToSchemas.put(TOPIC_1.getSchemaName(), AvroSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); + topicsToSchemas.put(TOPIC_2.getSchemaName(), AvroSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); + topicsToSchemas.put(TOPIC_3.getSchemaName(), AvroSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); + topicsToSchemas.put(TOPIC_4.getSchemaName(), JSONSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); + topicsToSchemas.put(TOPIC_5.getSchemaName(), JSONSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); + topicsToSchemas.put(TOPIC_6.getSchemaName(), JSONSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_1.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_1.getSchemaName(), AvroSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_2.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_3.getSchemaName(), AvroSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_4.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_5.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_6.getSchemaName(), JSONSchema.of(TestPulsarMetadata.Foo.class, true).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_2.getSchemaName(), AvroSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_3.getSchemaName(), AvroSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_4.getSchemaName(), JSONSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_5.getSchemaName(), JSONSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_6.getSchemaName(), JSONSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); fooTypes = new HashMap<>(); fooTypes.put("field1", IntegerType.INTEGER); @@ -622,7 +623,7 @@ private static List getTopicEntries(String topicSchemaName) { .setProducerName("test-producer").setSequenceId(i) .setPublishTime(currentTimeMs + i).build(); - Schema schema = topicsToSchemas.get(topicSchemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class, true) : JSONSchema.of(Foo.class, true); + Schema schema = topicsToSchemas.get(topicSchemaName).getType() == SchemaType.AVRO ? AvroSchema.of(new SchemaDefinition<>(Foo.class)) : JSONSchema.of(new SchemaDefinition<>(Foo.class)); org.apache.pulsar.shade.io.netty.buffer.ByteBuf payload = org.apache.pulsar.shade.io.netty.buffer.Unpooled .copiedBuffer(schema.encode(foo)); @@ -872,7 +873,7 @@ public void run() { .setProducerName("test-producer").setSequenceId(positions.get(topic)) .setPublishTime(System.currentTimeMillis()).build(); - Schema schema = topicsToSchemas.get(schemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class,true) : JSONSchema.of(Foo.class,true); + Schema schema = topicsToSchemas.get(schemaName).getType() == SchemaType.AVRO ? AvroSchema.of(new SchemaDefinition<>(Foo.class)) : JSONSchema.of(new SchemaDefinition<>(Foo.class)); org.apache.pulsar.shade.io.netty.buffer.ByteBuf payload = org.apache.pulsar.shade.io.netty.buffer.Unpooled .copiedBuffer(schema.encode(foo)); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index fb378d30f5601..aae1f74244a75 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -40,6 +40,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.FunctionStats; @@ -161,7 +162,7 @@ private void runSinkTester(SinkTester tester, boolean builtin) throws Exception // produce messages Map kvs; if (tester instanceof JdbcSinkTester) { - kvs = produceSchemaMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcSinkTester.Foo.class, true)); + kvs = produceSchemaMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(new SchemaDefinition<>(JdbcSinkTester.Foo.class).alwaysNull(true))); } else { kvs = produceMessagesToInputTopic(inputTopicName, numMessages); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java index c7b0ec75bb1a7..0687f567a800d 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java @@ -33,6 +33,7 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.testcontainers.containers.MySQLContainer; @@ -60,7 +61,7 @@ public static class Foo { private static final String NAME = "jdbc"; private static final String MYSQL = "mysql"; - private AvroSchema schema = AvroSchema.of(Foo.class, true); + private AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(true)); private String tableName = "test"; private Connection connection; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java index 1de96c127fd75..754361d830a8a 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.tests.integration.docker.ContainerExecException; import org.apache.pulsar.tests.integration.docker.ContainerExecResult; @@ -86,7 +87,7 @@ public void testSimpleSQLQuery(boolean isBatched) throws Exception { final String stocksTopic = "stocks"; @Cleanup - Producer producer = pulsarClient.newProducer(JSONSchema.of(Stock.class, true)) + Producer producer = pulsarClient.newProducer(JSONSchema.of(new SchemaDefinition<>(Stock.class))) .topic(stocksTopic) .enableBatching(isBatched) .create(); From 1b2c494f6ec426ff4ba1f8509131764b0eb4e3a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Fri, 8 Mar 2019 08:50:14 +0800 Subject: [PATCH 15/35] fix the test --- .../schema/generic/MultiVersionGenericSchemaProviderTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionGenericSchemaProviderTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionGenericSchemaProviderTest.java index 884a674bcd9bd..2eb65ce3f0a8e 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionGenericSchemaProviderTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionGenericSchemaProviderTest.java @@ -24,6 +24,7 @@ import static org.testng.Assert.assertEquals; import org.apache.pulsar.client.api.schema.GenericSchema; +import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.LookupService; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.schema.AvroSchema; @@ -54,7 +55,7 @@ public void setup() { @Test public void testGetSchema() { CompletableFuture> completableFuture = new CompletableFuture<>(); - SchemaInfo schemaInfo = AvroSchema.of(SchemaTestUtils.Foo.class).getSchemaInfo(); + SchemaInfo schemaInfo = AvroSchema.of(new SchemaDefinition<>(SchemaTestUtils.Foo.class)).getSchemaInfo(); completableFuture.complete(Optional.of(schemaInfo)); when(schemaProvider.getPulsarClient().getLookup() .getSchema( From 5e175bb319fb530f760a7c7ce52c594b8d524432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Fri, 8 Mar 2019 13:52:20 +0800 Subject: [PATCH 16/35] Remove keyValueSchemaDefinition --- .../service/schema/ClientGetSchemaTest.java | 4 +- .../org/apache/pulsar/client/api/Schema.java | 15 +- .../api/schema/KeyValueSchemaDefinition.java | 177 ------------------ .../client/api/schema/SchemaDefinition.java | 18 +- .../internal/DefaultImplementation.java | 12 +- .../pulsar/client/impl/schema/AvroSchema.java | 6 +- .../pulsar/client/impl/schema/JSONSchema.java | 9 +- .../client/impl/schema/KeyValueSchema.java | 32 +--- .../client/impl/schema/AvroSchemaTest.java | 6 +- .../client/impl/schema/JSONSchemaTest.java | 16 +- .../impl/schema/KeyValueSchemaTest.java | 29 +-- .../sink/HbaseGenericRecordSinkTest.java | 2 +- .../apache/pulsar/io/jdbc/JdbcSinkTest.java | 2 +- .../functions/PulsarFunctionsTest.java | 2 +- .../tests/integration/io/JdbcSinkTester.java | 2 +- 15 files changed, 74 insertions(+), 258 deletions(-) delete mode 100644 pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/KeyValueSchemaDefinition.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java index 732a917a0acaf..296abfd6c552f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java @@ -67,8 +67,8 @@ protected void setup() throws Exception { producers.add(pulsarClient.newProducer(Schema.JSON(MyClass.class)).topic(topicJson).create()); producers.add(pulsarClient.newProducer(Schema.AVRO(new SchemaDefinition<>(MyClass.class))).topic(topicAvro).create()); producers.add(pulsarClient.newProducer(Schema.JSON(new SchemaDefinition<>(MyClass.class))).topic(topicJson).create()); - producers.add(pulsarClient.newProducer(Schema.AVRO(new SchemaDefinition<>(MyClass.class).alwaysNull(false))).topic(topicAvroNotNull).create()); - producers.add(pulsarClient.newProducer(Schema.JSON(new SchemaDefinition<>(MyClass.class).alwaysNull(false))).topic(topicJsonNotNull).create()); + producers.add(pulsarClient.newProducer(Schema.AVRO(new SchemaDefinition<>(MyClass.class).alwaysAllNull(false))).topic(topicAvroNotNull).create()); + producers.add(pulsarClient.newProducer(Schema.JSON(new SchemaDefinition<>(MyClass.class).alwaysAllNull(false))).topic(topicJsonNotNull).create()); } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java index e2a90607c3298..319612ff20616 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java @@ -214,33 +214,26 @@ static Schema JSON(SchemaDefinition schemaDefinition) { * Key Value Schema using passed in schema type, support JSON and AVRO currently. */ static Schema> KeyValue(Class key, Class value, SchemaType type) { - return DefaultImplementation.newKeyValueSchema(new KeyValueSchemaDefinition<>(key, value, type)); + return DefaultImplementation.newKeyValueSchema(key, value, type); } /** * Schema that can be used to encode/decode KeyValue. */ - Schema> KV_BYTES = DefaultImplementation.newKeyValueSchema(new KeyValueSchemaDefinition<>(BYTES,BYTES)); + Schema> KV_BYTES = DefaultImplementation.newKeyValueSchema(BYTES, BYTES); /** * Key Value Schema whose underneath key and value schemas are JSONSchema. */ static Schema> KeyValue(Class key, Class value) { - return DefaultImplementation.newKeyValueSchema(new KeyValueSchemaDefinition<>(key, value, SchemaType.JSON)); + return DefaultImplementation.newKeyValueSchema(key, value, SchemaType.JSON); } /** * Key Value Schema using passed in key and value schemas. */ static Schema> KeyValue(Schema key, Schema value) { - return DefaultImplementation.newKeyValueSchema(new KeyValueSchemaDefinition<>(key, value)); - } - - /** - * Create key and value schema by keyValueSchemaDefinition. - */ - static Schema> KeyValue(KeyValueSchemaDefinition keyValueSchemaDefinition) { - return DefaultImplementation.newKeyValueSchema(keyValueSchemaDefinition); + return DefaultImplementation.newKeyValueSchema(key, value); } @Deprecated diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/KeyValueSchemaDefinition.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/KeyValueSchemaDefinition.java deleted file mode 100644 index 7c465f148f848..0000000000000 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/KeyValueSchemaDefinition.java +++ /dev/null @@ -1,177 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.client.api.schema; - - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.common.schema.SchemaType; - -import java.util.HashMap; -import java.util.Map; - -/** - * A key and value schema definition - * {@link org.apache.pulsar.client.api.Schema} for the key and value schema definition value. - */ -@Data -@EqualsAndHashCode -@ToString -public class KeyValueSchemaDefinition { - - private Class key; - - private Class value; - - private boolean alwaysNull = true; - - private SchemaType type; - - private Schema keySchema; - - private Schema valueSchema; - - private Map properties = new HashMap<>(); - - public KeyValueSchemaDefinition() { - - } - - public KeyValueSchemaDefinition(Schema keySchema, Schema valueSchema) { - this.keySchema = keySchema; - this.valueSchema = valueSchema; - } - - public KeyValueSchemaDefinition(Class key, Class value, SchemaType type) { - this.key = key; - this.value = value; - this.type = type; - } - - public KeyValueSchemaDefinition(Class key, Class value, boolean allowNull) { - this.key = key; - this.value = value; - this.alwaysNull = allowNull; - this.type = SchemaType.JSON; - } - - /** - * Get key schema class - * - * @return key schema class - */ - public Class getKey() { - - return key; - } - - /** - * Get value schema class - * - * @return value schema class - */ - public Class getValue() { - - return value; - } - - /** - * Set schema whether always null or not - * - * @param alwaysNull definition null or not - * @return Key and value schema definition - */ - public KeyValueSchemaDefinition alwaysNull(boolean alwaysNull) { - - this.alwaysNull = alwaysNull; - properties.put("alwaysNull", alwaysNull ? "true" : "false"); - return this; - } - - /** - * Set key schema - * - * @param keySchema definition key schema - * @return Key and value schema definition - */ - public KeyValueSchemaDefinition keySchema(Schema keySchema) { - - this.keySchema = keySchema; - return this; - } - - /** - * Set value schema - * - * @param valueSchema definition null or not - * @return Key and value schema definition - */ - public KeyValueSchemaDefinition valueSchema(Schema valueSchema) { - - this.valueSchema = valueSchema; - return this; - } - - /** - * get schema whether always null or not - * - * @return schema always null or not - */ - public boolean getAlwaysNull() { - - return alwaysNull; - } - - /** - * Set schema type - * - * @param type the schema type - * @return Key and value schema definition - */ - public KeyValueSchemaDefinition type(SchemaType type) { - - this.type = type; - return this; - } - - /** - * Get schema type - * - * @return Schema type - */ - public SchemaType getType() { - - return type; - } - - /** - * Set schema info properties - * - * @param properties schema info properties - * @return record schema definition - */ - public KeyValueSchemaDefinition properties(Map properties) { - - this.properties = properties; - properties.put("alwaysNull", alwaysNull ? "true" : "false"); - return this; - } -} diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java index 4adccb9aab6f0..64cd2665e9c33 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java @@ -41,7 +41,7 @@ public class SchemaDefinition { /** * The flag of schema type always null */ - private boolean alwaysNull = true; + private boolean alwaysAllNull = true; /** * The schema info properties */ @@ -50,7 +50,7 @@ public class SchemaDefinition { public SchemaDefinition(Class clazz) { - properties.put("alwaysNull", "true"); + properties.put("alwaysAllNull", "true"); this.clazz = clazz; } @@ -58,13 +58,13 @@ public SchemaDefinition(Class clazz) { /** * Set schema whether always null or not * - * @param alwaysNull definition null or not + * @param alwaysAllNull definition null or not * @return record schema definition */ - public SchemaDefinition alwaysNull(boolean alwaysNull) { + public SchemaDefinition alwaysAllNull(boolean alwaysAllNull) { - this.alwaysNull = alwaysNull; - properties.put("alwaysNull", alwaysNull ? "true" : "false"); + this.alwaysAllNull = alwaysAllNull; + properties.put("alwaysAllNull", this.alwaysAllNull ? "true" : "false"); return this; } @@ -73,9 +73,9 @@ public SchemaDefinition alwaysNull(boolean alwaysNull) { * * @return schema always null or not */ - public boolean getAlwaysNull() { + public boolean getAlwaysAllNull() { - return alwaysNull; + return alwaysAllNull; } /** @@ -98,7 +98,7 @@ public Class getClazz() { public SchemaDefinition properties(Map properties) { this.properties = properties; - properties.put("alwaysNull", alwaysNull ? "true" : "false"); + properties.put("alwaysAllNull", alwaysAllNull ? "true" : "false"); return this; } } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java index 5b35a43f58a4a..b277129b522cd 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java @@ -38,12 +38,12 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.PulsarClientException.UnsupportedAuthenticationException; import org.apache.pulsar.client.api.schema.GenericSchema; -import org.apache.pulsar.client.api.schema.KeyValueSchemaDefinition; import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; @SuppressWarnings("unchecked") @UtilityClass @@ -213,10 +213,16 @@ public static Schema newAutoProduceSchema() { .newInstance()); } - public static Schema> newKeyValueSchema(KeyValueSchemaDefinition keyValueSchemaDefinition) { + public static Schema> newKeyValueSchema(Schema keySchema, Schema valueSchema) { + return catchExceptions( + () -> (Schema>) getConstructor("org.apache.pulsar.client.impl.schema.KeyValueSchema", + Schema.class, Schema.class).newInstance(keySchema, valueSchema)); + } + + public static Schema> newKeyValueSchema(Class key, Class value, SchemaType type) { return catchExceptions( () -> (Schema>) getStaticMethod("org.apache.pulsar.client.impl.schema.KeyValueSchema", - "of", KeyValueSchemaDefinition.class).invoke(null, keyValueSchemaDefinition)); + "of", Class.class, Class.class, SchemaType.class).invoke(null, key, value, type)); } public static Schema getSchema(SchemaInfo schemaInfo) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index 1f7a5872a03c2..9f4d6f0e15dda 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -103,13 +103,17 @@ private static org.apache.avro.Schema createAvroSchema(SchemaDefinition s Class clazz = schemaDefinition.getClazz(); - return schemaDefinition.getAlwaysNull() ? ReflectData.AllowNull.get().getSchema(clazz) : ReflectData.get().getSchema(clazz); + return schemaDefinition.getAlwaysAllNull() ? ReflectData.AllowNull.get().getSchema(clazz) : ReflectData.get().getSchema(clazz); } public static AvroSchema of(SchemaDefinition schemaDefinition) { return new AvroSchema<>(createAvroSchema(schemaDefinition), schemaDefinition); } + public static AvroSchema of(Class pojo) { + return AvroSchema.of(new SchemaDefinition<>(pojo)); + } + public static AvroSchema of(Class pojo, Map properties) { SchemaDefinition schemaDefinition = new SchemaDefinition<>(pojo).properties(properties); return new AvroSchema<>(createAvroSchema(schemaDefinition), schemaDefinition); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java index 311cb559b8f65..d731deb17fb13 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java @@ -59,7 +59,7 @@ public class JSONSchema implements Schema { private JSONSchema(SchemaDefinition schemaDefinition) { this.pojo = schemaDefinition.getClazz(); this.properties = schemaDefinition.getProperties(); - boolean alwaysNull = schemaDefinition.getAlwaysNull(); + boolean alwaysNull = schemaDefinition.getAlwaysAllNull(); this.schema = alwaysNull ? ReflectData.AllowNull.get().getSchema(pojo) : ReflectData.get().getSchema(pojo); this.schemaInfo = new SchemaInfo(); @@ -121,8 +121,11 @@ public static JSONSchema of(SchemaDefinition schemaDefinition) { return new JSONSchema<>(schemaDefinition); } + public static JSONSchema of(Class pojo) { + return new JSONSchema<>(new SchemaDefinition<>(pojo)); + } + public static JSONSchema of(Class pojo, Map properties) { - SchemaDefinition schemaDefinition = new SchemaDefinition<>(pojo).properties(properties); - return new JSONSchema<>(schemaDefinition); + return new JSONSchema<>(new SchemaDefinition<>(pojo).properties(properties)); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java index 184453f60181e..3c4205d119638 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java @@ -21,13 +21,10 @@ import static com.google.common.base.Preconditions.checkArgument; import java.nio.ByteBuffer; -import java.util.HashMap; import lombok.Getter; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.schema.KeyValueSchemaDefinition; -import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -48,36 +45,26 @@ public class KeyValueSchema implements Schema> { /** * Key Value Schema using passed in schema type, support JSON and AVRO currently. */ - public static Schema> of(KeyValueSchemaDefinition keyValueSchemaDefinition) { - SchemaType type = keyValueSchemaDefinition.getType(); - Class key = keyValueSchemaDefinition.getKey(); - Class value = keyValueSchemaDefinition.getValue(); - boolean alwaysNull = keyValueSchemaDefinition.getAlwaysNull(); - checkArgument((SchemaType.JSON == type || SchemaType.AVRO == type) || - (keyValueSchemaDefinition.getKeySchema() != null && keyValueSchemaDefinition.getValueSchema() != null)); + public static Schema> of(Class key, Class value, SchemaType type) { + checkArgument(SchemaType.JSON == type || SchemaType.AVRO == type); if (SchemaType.JSON == type) { - return new KeyValueSchema<>(keyValueSchemaDefinition.keySchema(JSONSchema.of(new SchemaDefinition<>(key).alwaysNull(alwaysNull))). - valueSchema(JSONSchema.of(new SchemaDefinition<>(value).alwaysNull(alwaysNull)))); - } else if (SchemaType.AVRO == type) { - // AVRO - return new KeyValueSchema<>(keyValueSchemaDefinition.keySchema(AvroSchema.of(new SchemaDefinition<>(key).alwaysNull(alwaysNull))). - valueSchema(AvroSchema.of(new SchemaDefinition<>(value).alwaysNull(alwaysNull)))); + return new KeyValueSchema<>(JSONSchema.of(key), JSONSchema.of(value)); } else { - return new KeyValueSchema<>(keyValueSchemaDefinition); + // AVRO + return new KeyValueSchema<>(AvroSchema.of(key), AvroSchema.of(value)); } } - public KeyValueSchema(KeyValueSchemaDefinition keyValueSchemaDefinition) { - this.keySchema = keyValueSchemaDefinition.getKeySchema(); - this.valueSchema = keyValueSchemaDefinition.getValueSchema(); + public KeyValueSchema(Schema keySchema, + Schema valueSchema) { + this.keySchema = keySchema; + this.valueSchema = valueSchema; // set schemaInfo this.schemaInfo = new SchemaInfo() .setName("KeyValue") .setType(SchemaType.KEY_VALUE); - this.schemaInfo.setProperties(keyValueSchemaDefinition.getProperties()); - byte[] keySchemaInfo = keySchema.getSchemaInfo().getSchema(); byte[] valueSchemaInfo = valueSchema.getSchemaInfo().getSchema(); @@ -87,7 +74,6 @@ public KeyValueSchema(KeyValueSchemaDefinition keyValueSchemaDefinition) { this.schemaInfo.setSchema(byteBuffer.array()); } - // encode as bytes: [key.length][key.bytes][value.length][value.bytes] public byte[] encode(KeyValue message) { byte[] keyBytes = keySchema.encode(message.getKey()); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index 8f2d52f08be35..1a1831d5f0c61 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -40,7 +40,7 @@ public class AvroSchemaTest { @Test public void testNotAllowNullSchema() { - AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); + AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); @@ -62,7 +62,7 @@ public void testNotAllowNullSchema() { @Test public void testAllowNullSchema() { - AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(true)); + AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(true)); assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); @@ -84,7 +84,7 @@ public void testAllowNullSchema() { @Test public void testNotAllowNullEncodeAndDecode() { - AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); + AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); Foo foo1 = new Foo(); foo1.setField1("foo1"); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java index e37e8b7bc1325..3782e2c781568 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java @@ -44,7 +44,7 @@ public class JSONSchemaTest { @Test public void testNotAllowNullSchema() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema()); @@ -118,7 +118,7 @@ public void testAllowNullEncodeAndDecode() { @Test public void testNotAllowNullEncodeAndDecode() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); Foo foo1 = new Foo(); foo1.setField1("foo1"); @@ -174,8 +174,8 @@ public void testAllowNullNestedClasses() { @Test public void testNotAllowNullNestedClasses() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBar.class).alwaysNull(false)); - JSONSchema listJsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBarList.class).alwaysNull(false)); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBar.class).alwaysAllNull(false)); + JSONSchema listJsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBarList.class).alwaysAllNull(false)); Bar bar = new Bar(); bar.setField1(true); @@ -281,25 +281,25 @@ public void testAllowNullCorrectPolymorphism() { derivedDerivedFoo.setDerivedFoo(derivedFoo); // schema for base class - JSONSchema baseJsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); + JSONSchema baseJsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(foo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedFoo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedDerivedFoo)), foo); // schema for derived class - JSONSchema derivedJsonSchema = JSONSchema.of(new SchemaDefinition<>(DerivedFoo.class).alwaysNull(false)); + JSONSchema derivedJsonSchema = JSONSchema.of(new SchemaDefinition<>(DerivedFoo.class).alwaysAllNull(false)); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedFoo)), derivedFoo); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedDerivedFoo)), derivedFoo); //schema for derived derived class JSONSchema derivedDerivedJsonSchema - = JSONSchema.of(new SchemaDefinition<>(SchemaTestUtils.DerivedDerivedFoo.class).alwaysNull(false)); + = JSONSchema.of(new SchemaDefinition<>(SchemaTestUtils.DerivedDerivedFoo.class).alwaysAllNull(false)); Assert.assertEquals(derivedDerivedJsonSchema.decode(derivedDerivedJsonSchema.encode(derivedDerivedFoo)), derivedDerivedFoo); } @Test(expectedExceptions = SchemaSerializationException.class) public void testNotAllowNullDecodeWithInvalidContent() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); jsonSchema.decode(new byte[0]); } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java index 6bdc21d03f9ef..a903b31c9550d 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java @@ -22,11 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.schema.KeyValueSchemaDefinition; import org.apache.pulsar.client.api.schema.SchemaDefinition; -import org.apache.pulsar.client.impl.schema.AvroSchema; -import org.apache.pulsar.client.impl.schema.JSONSchema; -import org.apache.pulsar.client.impl.schema.KeyValueSchema; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Bar; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Color; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Foo; @@ -65,11 +61,12 @@ public void testAllowNullAvroSchemaCreate() { @Test public void testNotAllowNullAvroSchemaCreate() { - AvroSchema fooSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); - AvroSchema barSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysNull(false)); + AvroSchema fooSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); + AvroSchema barSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false)); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); - Schema> keyValueSchema2 = Schema.KeyValue(new KeyValueSchemaDefinition<>(Foo.class,Bar.class,SchemaType.AVRO).alwaysNull(false)); + Schema> keyValueSchema2 = Schema.KeyValue(AvroSchema.of(new SchemaDefinition<> + (Foo.class).alwaysAllNull(false)),AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false))); assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); @@ -123,12 +120,15 @@ public void testAllowNullJsonSchemaCreate() { @Test public void testNotAllowNullJsonSchemaCreate() { - JSONSchema fooSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); - JSONSchema barSchema = JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysNull(false)); + JSONSchema fooSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); + JSONSchema barSchema = JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false)); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); - Schema> keyValueSchema2 = Schema.KeyValue(new KeyValueSchemaDefinition<>(Foo.class,Bar.class,false).type(SchemaType.JSON)); - Schema> keyValueSchema3 = Schema.KeyValue(new KeyValueSchemaDefinition<>(Foo.class,Bar.class,false)); + Schema> keyValueSchema2 = Schema.KeyValue(JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)), + JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false))); + + Schema> keyValueSchema3 = Schema.KeyValue(JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)), + JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false))); assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); @@ -181,7 +181,8 @@ public void testAllowNullSchemaEncodeAndDecode() { @Test public void testNotAllowNullSchemaEncodeAndDecode() { - Schema keyValueSchema = Schema.KeyValue(new KeyValueSchemaDefinition<>(Foo.class,Bar.class,false)); + Schema keyValueSchema = Schema.KeyValue(JSONSchema.of(new SchemaDefinition<>(Foo.class) + .alwaysAllNull(false)),JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false))); Bar bar = new Bar(); bar.setField1(true); @@ -235,8 +236,8 @@ public void testAllowNullBytesSchemaEncodeAndDecode() { @Test public void testNotAllowNullBytesSchemaEncodeAndDecode() { - AvroSchema fooAvroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(false)); - AvroSchema barAvroSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysNull(false)); + AvroSchema fooAvroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); + AvroSchema barAvroSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false)); Bar bar = new Bar(); bar.setField1(true); diff --git a/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java b/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java index e23f4317144fe..68b13241e358c 100644 --- a/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java +++ b/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java @@ -109,7 +109,7 @@ public void TestOpenAndWriteSink() throws Exception { obj.setAddress("address_value"); obj.setAge(30); obj.setFlag(true); - AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(true)); + AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(true)); byte[] bytes = schema.encode(obj); ByteBuf payload = Unpooled.copiedBuffer(bytes); diff --git a/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java b/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java index be7a566288a3e..9aed97ad79c09 100644 --- a/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java +++ b/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java @@ -96,7 +96,7 @@ public void TestOpenAndWriteSink() throws Exception { obj.setField1("ValueOfField1"); obj.setField2("ValueOfField1"); obj.setField3(3); - AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(true)); + AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(true)); byte[] bytes = schema.encode(obj); ByteBuf payload = Unpooled.copiedBuffer(bytes); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index aae1f74244a75..a2b6de19b9663 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -162,7 +162,7 @@ private void runSinkTester(SinkTester tester, boolean builtin) throws Exception // produce messages Map kvs; if (tester instanceof JdbcSinkTester) { - kvs = produceSchemaMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(new SchemaDefinition<>(JdbcSinkTester.Foo.class).alwaysNull(true))); + kvs = produceSchemaMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(new SchemaDefinition<>(JdbcSinkTester.Foo.class).alwaysAllNull(true))); } else { kvs = produceMessagesToInputTopic(inputTopicName, numMessages); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java index 0687f567a800d..d41ecaeea4917 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java @@ -61,7 +61,7 @@ public static class Foo { private static final String NAME = "jdbc"; private static final String MYSQL = "mysql"; - private AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysNull(true)); + private AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(true)); private String tableName = "test"; private Connection connection; From f41c91a7e2ae3073a2d70240161c312a1cbff43d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Fri, 8 Mar 2019 13:55:29 +0800 Subject: [PATCH 17/35] remove keyValueSchemaDefinition dependency --- .../src/main/java/org/apache/pulsar/client/api/Schema.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java index 319612ff20616..814df9f7be461 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java @@ -21,7 +21,6 @@ import java.nio.ByteBuffer; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.GenericSchema; -import org.apache.pulsar.client.api.schema.KeyValueSchemaDefinition; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.internal.DefaultImplementation; import org.apache.pulsar.common.schema.KeyValue; From dcb257ab7343a968077a38d0e455ce807eac393d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Fri, 8 Mar 2019 14:26:08 +0800 Subject: [PATCH 18/35] remove dependency --- .../org/apache/pulsar/client/impl/schema/AvroSchemaTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index 1a1831d5f0c61..049151a975a1e 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -33,7 +33,6 @@ import org.testng.Assert; import org.testng.annotations.Test; -import java.util.HashMap; @Slf4j public class AvroSchemaTest { From cee82902329dc7f3dd144a6fd9d62088ea64e71b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Sat, 9 Mar 2019 10:47:19 +0800 Subject: [PATCH 19/35] Modify the system schema config field alwaysAllNull to alwaysAllowNull --- .../service/schema/ClientGetSchemaTest.java | 4 +-- .../client/api/schema/SchemaDefinition.java | 25 +++++++++---------- .../pulsar/client/impl/schema/AvroSchema.java | 2 +- .../pulsar/client/impl/schema/JSONSchema.java | 4 +-- .../client/impl/schema/AvroSchemaTest.java | 6 ++--- .../client/impl/schema/JSONSchemaTest.java | 16 ++++++------ .../impl/schema/KeyValueSchemaTest.java | 24 +++++++++--------- .../sink/HbaseGenericRecordSinkTest.java | 2 +- .../apache/pulsar/io/jdbc/JdbcSinkTest.java | 2 +- .../functions/PulsarFunctionsTest.java | 2 +- .../tests/integration/io/JdbcSinkTester.java | 2 +- 11 files changed, 44 insertions(+), 45 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java index 296abfd6c552f..395eacf3322c9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java @@ -67,8 +67,8 @@ protected void setup() throws Exception { producers.add(pulsarClient.newProducer(Schema.JSON(MyClass.class)).topic(topicJson).create()); producers.add(pulsarClient.newProducer(Schema.AVRO(new SchemaDefinition<>(MyClass.class))).topic(topicAvro).create()); producers.add(pulsarClient.newProducer(Schema.JSON(new SchemaDefinition<>(MyClass.class))).topic(topicJson).create()); - producers.add(pulsarClient.newProducer(Schema.AVRO(new SchemaDefinition<>(MyClass.class).alwaysAllNull(false))).topic(topicAvroNotNull).create()); - producers.add(pulsarClient.newProducer(Schema.JSON(new SchemaDefinition<>(MyClass.class).alwaysAllNull(false))).topic(topicJsonNotNull).create()); + producers.add(pulsarClient.newProducer(Schema.AVRO(new SchemaDefinition<>(MyClass.class).alwaysAllowNull(false))).topic(topicAvroNotNull).create()); + producers.add(pulsarClient.newProducer(Schema.JSON(new SchemaDefinition<>(MyClass.class).alwaysAllowNull(false))).topic(topicJsonNotNull).create()); } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java index 64cd2665e9c33..80b3e360aedf7 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java @@ -39,9 +39,9 @@ public class SchemaDefinition { */ private final Class clazz; /** - * The flag of schema type always null + * The flag of schema type always allow null */ - private boolean alwaysAllNull = true; + private boolean alwaysAllowNull = true; /** * The schema info properties */ @@ -50,32 +50,31 @@ public class SchemaDefinition { public SchemaDefinition(Class clazz) { - properties.put("alwaysAllNull", "true"); + properties.put("__alwaysAllowNull", "true"); this.clazz = clazz; } /** - * Set schema whether always null or not + * Set schema whether always allow null or not * - * @param alwaysAllNull definition null or not + * @param alwaysAllowNull definition null or not * @return record schema definition */ - public SchemaDefinition alwaysAllNull(boolean alwaysAllNull) { + public SchemaDefinition alwaysAllowNull(boolean alwaysAllowNull) { - this.alwaysAllNull = alwaysAllNull; - properties.put("alwaysAllNull", this.alwaysAllNull ? "true" : "false"); + this.alwaysAllowNull = alwaysAllowNull; + properties.put("__alwaysAllowNull", this.alwaysAllowNull ? "true" : "false"); return this; } - /** - * get schema whether always null or not + * get schema whether always allow null or not * * @return schema always null or not */ - public boolean getAlwaysAllNull() { + public boolean getAlwaysAllowNull() { - return alwaysAllNull; + return alwaysAllowNull; } /** @@ -98,7 +97,7 @@ public Class getClazz() { public SchemaDefinition properties(Map properties) { this.properties = properties; - properties.put("alwaysAllNull", alwaysAllNull ? "true" : "false"); + properties.put("__alwaysAllowNull", alwaysAllowNull ? "true" : "false"); return this; } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index 9f4d6f0e15dda..5204b31b99222 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -103,7 +103,7 @@ private static org.apache.avro.Schema createAvroSchema(SchemaDefinition s Class clazz = schemaDefinition.getClazz(); - return schemaDefinition.getAlwaysAllNull() ? ReflectData.AllowNull.get().getSchema(clazz) : ReflectData.get().getSchema(clazz); + return schemaDefinition.getAlwaysAllowNull() ? ReflectData.AllowNull.get().getSchema(clazz) : ReflectData.get().getSchema(clazz); } public static AvroSchema of(SchemaDefinition schemaDefinition) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java index d731deb17fb13..f146b93cb6870 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java @@ -59,9 +59,9 @@ public class JSONSchema implements Schema { private JSONSchema(SchemaDefinition schemaDefinition) { this.pojo = schemaDefinition.getClazz(); this.properties = schemaDefinition.getProperties(); - boolean alwaysNull = schemaDefinition.getAlwaysAllNull(); + boolean alwaysAllowNull = schemaDefinition.getAlwaysAllowNull(); - this.schema = alwaysNull ? ReflectData.AllowNull.get().getSchema(pojo) : ReflectData.get().getSchema(pojo); + this.schema = alwaysAllowNull ? ReflectData.AllowNull.get().getSchema(pojo) : ReflectData.get().getSchema(pojo); this.schemaInfo = new SchemaInfo(); this.schemaInfo.setName(""); this.schemaInfo.setProperties(properties); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index 049151a975a1e..4d8471799a44d 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -39,7 +39,7 @@ public class AvroSchemaTest { @Test public void testNotAllowNullSchema() { - AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); + AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); @@ -61,7 +61,7 @@ public void testNotAllowNullSchema() { @Test public void testAllowNullSchema() { - AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(true)); + AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(true)); assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); @@ -83,7 +83,7 @@ public void testAllowNullSchema() { @Test public void testNotAllowNullEncodeAndDecode() { - AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); + AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); Foo foo1 = new Foo(); foo1.setField1("foo1"); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java index 3782e2c781568..443283618ee2d 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java @@ -44,7 +44,7 @@ public class JSONSchemaTest { @Test public void testNotAllowNullSchema() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema()); @@ -118,7 +118,7 @@ public void testAllowNullEncodeAndDecode() { @Test public void testNotAllowNullEncodeAndDecode() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); Foo foo1 = new Foo(); foo1.setField1("foo1"); @@ -174,8 +174,8 @@ public void testAllowNullNestedClasses() { @Test public void testNotAllowNullNestedClasses() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBar.class).alwaysAllNull(false)); - JSONSchema listJsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBarList.class).alwaysAllNull(false)); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBar.class).alwaysAllowNull(false)); + JSONSchema listJsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBarList.class).alwaysAllowNull(false)); Bar bar = new Bar(); bar.setField1(true); @@ -281,25 +281,25 @@ public void testAllowNullCorrectPolymorphism() { derivedDerivedFoo.setDerivedFoo(derivedFoo); // schema for base class - JSONSchema baseJsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); + JSONSchema baseJsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(foo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedFoo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedDerivedFoo)), foo); // schema for derived class - JSONSchema derivedJsonSchema = JSONSchema.of(new SchemaDefinition<>(DerivedFoo.class).alwaysAllNull(false)); + JSONSchema derivedJsonSchema = JSONSchema.of(new SchemaDefinition<>(DerivedFoo.class).alwaysAllowNull(false)); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedFoo)), derivedFoo); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedDerivedFoo)), derivedFoo); //schema for derived derived class JSONSchema derivedDerivedJsonSchema - = JSONSchema.of(new SchemaDefinition<>(SchemaTestUtils.DerivedDerivedFoo.class).alwaysAllNull(false)); + = JSONSchema.of(new SchemaDefinition<>(SchemaTestUtils.DerivedDerivedFoo.class).alwaysAllowNull(false)); Assert.assertEquals(derivedDerivedJsonSchema.decode(derivedDerivedJsonSchema.encode(derivedDerivedFoo)), derivedDerivedFoo); } @Test(expectedExceptions = SchemaSerializationException.class) public void testNotAllowNullDecodeWithInvalidContent() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); + JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); jsonSchema.decode(new byte[0]); } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java index a903b31c9550d..973a1e58b7e86 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java @@ -61,12 +61,12 @@ public void testAllowNullAvroSchemaCreate() { @Test public void testNotAllowNullAvroSchemaCreate() { - AvroSchema fooSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); - AvroSchema barSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false)); + AvroSchema fooSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); + AvroSchema barSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false)); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); Schema> keyValueSchema2 = Schema.KeyValue(AvroSchema.of(new SchemaDefinition<> - (Foo.class).alwaysAllNull(false)),AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false))); + (Foo.class).alwaysAllowNull(false)),AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false))); assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); @@ -120,15 +120,15 @@ public void testAllowNullJsonSchemaCreate() { @Test public void testNotAllowNullJsonSchemaCreate() { - JSONSchema fooSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); - JSONSchema barSchema = JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false)); + JSONSchema fooSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); + JSONSchema barSchema = JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false)); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); - Schema> keyValueSchema2 = Schema.KeyValue(JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)), - JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false))); + Schema> keyValueSchema2 = Schema.KeyValue(JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)), + JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false))); - Schema> keyValueSchema3 = Schema.KeyValue(JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)), - JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false))); + Schema> keyValueSchema3 = Schema.KeyValue(JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)), + JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false))); assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); @@ -182,7 +182,7 @@ public void testAllowNullSchemaEncodeAndDecode() { @Test public void testNotAllowNullSchemaEncodeAndDecode() { Schema keyValueSchema = Schema.KeyValue(JSONSchema.of(new SchemaDefinition<>(Foo.class) - .alwaysAllNull(false)),JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false))); + .alwaysAllowNull(false)),JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false))); Bar bar = new Bar(); bar.setField1(true); @@ -236,8 +236,8 @@ public void testAllowNullBytesSchemaEncodeAndDecode() { @Test public void testNotAllowNullBytesSchemaEncodeAndDecode() { - AvroSchema fooAvroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(false)); - AvroSchema barAvroSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllNull(false)); + AvroSchema fooAvroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); + AvroSchema barAvroSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false)); Bar bar = new Bar(); bar.setField1(true); diff --git a/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java b/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java index 68b13241e358c..109af0112ba65 100644 --- a/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java +++ b/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java @@ -109,7 +109,7 @@ public void TestOpenAndWriteSink() throws Exception { obj.setAddress("address_value"); obj.setAge(30); obj.setFlag(true); - AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(true)); + AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(true)); byte[] bytes = schema.encode(obj); ByteBuf payload = Unpooled.copiedBuffer(bytes); diff --git a/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java b/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java index 9aed97ad79c09..a3a0421fd96c1 100644 --- a/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java +++ b/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java @@ -96,7 +96,7 @@ public void TestOpenAndWriteSink() throws Exception { obj.setField1("ValueOfField1"); obj.setField2("ValueOfField1"); obj.setField3(3); - AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(true)); + AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(true)); byte[] bytes = schema.encode(obj); ByteBuf payload = Unpooled.copiedBuffer(bytes); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index a2b6de19b9663..04e27fb5300a7 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -162,7 +162,7 @@ private void runSinkTester(SinkTester tester, boolean builtin) throws Exception // produce messages Map kvs; if (tester instanceof JdbcSinkTester) { - kvs = produceSchemaMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(new SchemaDefinition<>(JdbcSinkTester.Foo.class).alwaysAllNull(true))); + kvs = produceSchemaMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(new SchemaDefinition<>(JdbcSinkTester.Foo.class).alwaysAllowNull(true))); } else { kvs = produceMessagesToInputTopic(inputTopicName, numMessages); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java index d41ecaeea4917..4d5e46d2a439b 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java @@ -61,7 +61,7 @@ public static class Foo { private static final String NAME = "jdbc"; private static final String MYSQL = "mysql"; - private AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllNull(true)); + private AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(true)); private String tableName = "test"; private Connection connection; From 08e2037b6b7f214b31ee8058f258e186faf3b579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Sat, 9 Mar 2019 11:22:36 +0800 Subject: [PATCH 20/35] Restore code format --- .../apache/pulsar/client/impl/schema/KeyValueSchema.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java index 3c4205d119638..0689b7e78e827 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java @@ -62,15 +62,15 @@ public KeyValueSchema(Schema keySchema, // set schemaInfo this.schemaInfo = new SchemaInfo() - .setName("KeyValue") - .setType(SchemaType.KEY_VALUE); + .setName("KeyValue") + .setType(SchemaType.KEY_VALUE); byte[] keySchemaInfo = keySchema.getSchemaInfo().getSchema(); byte[] valueSchemaInfo = valueSchema.getSchemaInfo().getSchema(); ByteBuffer byteBuffer = ByteBuffer.allocate(4 + keySchemaInfo.length + 4 + valueSchemaInfo.length); byteBuffer.putInt(keySchemaInfo.length).put(keySchemaInfo) - .putInt(valueSchemaInfo.length).put(valueSchemaInfo); + .putInt(valueSchemaInfo.length).put(valueSchemaInfo); this.schemaInfo.setSchema(byteBuffer.array()); } From 435f514cb94ed742f77a0c9b2354cf3e99d2cf49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Sat, 9 Mar 2019 14:55:00 +0800 Subject: [PATCH 21/35] Modify the keyValueSchemaTest --- .../apache/pulsar/client/impl/schema/KeyValueSchemaTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java index 3b84f54418276..5f6e5716e4612 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java @@ -224,8 +224,8 @@ public void testAllowNullBytesSchemaEncodeAndDecode() { byte[] fooBytes = fooAvroSchema.encode(foo); byte[] barBytes = barAvroSchema.encode(bar); - byte[] encodeBytes = Schema.KV_BYTES.encode(new KeyValue<>(fooBytes, barBytes)); - KeyValue decodeKV = Schema.KV_BYTES.decode(encodeBytes); + byte[] encodeBytes = Schema.KV_BYTES().encode(new KeyValue<>(fooBytes, barBytes)); + KeyValue decodeKV = Schema.KV_BYTES().decode(encodeBytes); Foo fooBack = fooAvroSchema.decode(decodeKV.getKey()); Bar barBack = barAvroSchema.decode(decodeKV.getValue()); From b02504e15c0e4a62c301771858f03ce3accd2677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 14 Mar 2019 10:41:36 +0800 Subject: [PATCH 22/35] add schema defination builder --- .../service/schema/ClientGetSchemaTest.java | 8 +- .../JsonSchemaCompatibilityCheckTest.java | 4 +- .../org/apache/pulsar/client/api/Schema.java | 8 +- .../client/api/schema/SchemaDefinition.java | 66 ++++------------ .../api/schema/SchemaDefinitionBuilder.java | 63 +++++++++++++++ .../internal/DefaultImplementation.java | 14 ++-- .../pulsar/client/impl/schema/AvroSchema.java | 6 +- .../pulsar/client/impl/schema/JSONSchema.java | 4 +- .../schema/SchemaDefinitionBuilderImpl.java | 79 +++++++++++++++++++ .../impl/schema/SchemaDefinitionImpl.java | 78 ++++++++++++++++++ .../client/impl/schema/AvroSchemaTest.java | 9 ++- .../client/impl/schema/JSONSchemaTest.java | 32 ++++---- .../impl/schema/KeyValueSchemaTest.java | 40 +++++----- ...MultiVersionGenericSchemaProviderTest.java | 3 +- .../SampleAsyncProducerWithSchema.java | 3 +- .../tutorial/SampleConsumerWithSchema.java | 3 +- .../sink/HbaseGenericRecordSinkTest.java | 2 +- .../apache/pulsar/io/jdbc/JdbcSinkTest.java | 2 +- .../tests/integration/io/JdbcSinkTester.java | 2 +- 19 files changed, 309 insertions(+), 117 deletions(-) create mode 100644 pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinitionBuilder.java create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionImpl.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java index 395eacf3322c9..07e9de8d651c8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/ClientGetSchemaTest.java @@ -65,10 +65,10 @@ protected void setup() throws Exception { producers.add(pulsarClient.newProducer(Schema.STRING).topic(topicString).create()); producers.add(pulsarClient.newProducer(Schema.AVRO(MyClass.class)).topic(topicAvro).create()); producers.add(pulsarClient.newProducer(Schema.JSON(MyClass.class)).topic(topicJson).create()); - producers.add(pulsarClient.newProducer(Schema.AVRO(new SchemaDefinition<>(MyClass.class))).topic(topicAvro).create()); - producers.add(pulsarClient.newProducer(Schema.JSON(new SchemaDefinition<>(MyClass.class))).topic(topicJson).create()); - producers.add(pulsarClient.newProducer(Schema.AVRO(new SchemaDefinition<>(MyClass.class).alwaysAllowNull(false))).topic(topicAvroNotNull).create()); - producers.add(pulsarClient.newProducer(Schema.JSON(new SchemaDefinition<>(MyClass.class).alwaysAllowNull(false))).topic(topicJsonNotNull).create()); + producers.add(pulsarClient.newProducer(Schema.AVRO(SchemaDefinition.builder(MyClass.class).build())).topic(topicAvro).create()); + producers.add(pulsarClient.newProducer(Schema.JSON(SchemaDefinition.builder(MyClass.class).build())).topic(topicJson).create()); + producers.add(pulsarClient.newProducer(Schema.AVRO(SchemaDefinition.builder(MyClass.class).withAlwaysAllowNull(false).build())).topic(topicAvroNotNull).create()); + producers.add(pulsarClient.newProducer(Schema.JSON(SchemaDefinition.builder(MyClass.class).withAlwaysAllowNull(false).build())).topic(topicJsonNotNull).create()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java index 24786ca464ed5..112ab4d3605ad 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheckTest.java @@ -52,11 +52,11 @@ public SchemaCompatibilityCheck getSchemaCheck() { public void testJsonSchemaBackwardsCompatibility() throws JsonProcessingException { SchemaData from = SchemaData.builder().data(OldJSONSchema.of(Foo.class).getSchemaInfo().getSchema()).build(); - SchemaData to = SchemaData.builder().data(JSONSchema.of(new SchemaDefinition<>(Foo.class)).getSchemaInfo().getSchema()).build(); + SchemaData to = SchemaData.builder().data(JSONSchema.of(SchemaDefinition.builder(Foo.class).build()).getSchemaInfo().getSchema()).build(); JsonSchemaCompatibilityCheck jsonSchemaCompatibilityCheck = new JsonSchemaCompatibilityCheck(); Assert.assertTrue(jsonSchemaCompatibilityCheck.isCompatible(from, to, SchemaCompatibilityStrategy.FULL)); - from = SchemaData.builder().data(JSONSchema.of(new SchemaDefinition<>(Foo.class)).getSchemaInfo().getSchema()).build(); + from = SchemaData.builder().data(JSONSchema.of(SchemaDefinition.builder(Foo.class).build()).getSchemaInfo().getSchema()).build(); to = SchemaData.builder().data(OldJSONSchema.of(Foo.class).getSchemaInfo().getSchema()).build(); Assert.assertTrue(jsonSchemaCompatibilityCheck.isCompatible(from, to, SchemaCompatibilityStrategy.FULL)); } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java index 1ded4186b678e..c201ba3ffb7d7 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Schema.java @@ -174,7 +174,7 @@ static Schema PROTOBUF(Cla * @return a Schema instance */ static Schema AVRO(Class clazz) { - return DefaultImplementation.newAvroSchema(new SchemaDefinition<>(clazz)); + return DefaultImplementation.newAvroSchema(SchemaDefinition.builder(clazz).build()); } /** @@ -195,7 +195,7 @@ static Schema AVRO(SchemaDefinition schemaDefinition) { * @return a Schema instance */ static Schema JSON(Class clazz) { - return DefaultImplementation.newJSONSchema(new SchemaDefinition<>(clazz)); + return DefaultImplementation.newJSONSchema(SchemaDefinition.builder(clazz).build()); } /** @@ -245,9 +245,9 @@ static Schema AUTO() { /** * Create a schema instance that automatically deserialize messages * based on the current topic schema. - * + *

* The messages values are deserialized into a {@link GenericRecord} object. - * + *

* Currently this is only supported with Avro and JSON schema types. * * @return the auto schema instance diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java index 80b3e360aedf7..05b76b9776a44 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java @@ -21,6 +21,9 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; +import org.apache.pulsar.client.api.ClientBuilder; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.internal.DefaultImplementation; import java.util.HashMap; import java.util.Map; @@ -29,75 +32,38 @@ * A schema definition * {@link org.apache.pulsar.client.api.Schema} for the schema definition value. */ -@Data -@EqualsAndHashCode -@ToString -public class SchemaDefinition { - /** - * the schema definition class - */ - private final Class clazz; - /** - * The flag of schema type always allow null - */ - private boolean alwaysAllowNull = true; - /** - * The schema info properties - */ - private Map properties = new HashMap<>(); - - - public SchemaDefinition(Class clazz) { - - properties.put("__alwaysAllowNull", "true"); - this.clazz = clazz; - - } +public interface SchemaDefinition { /** - * Set schema whether always allow null or not + * Get a new builder instance that can used to configure and build a {@link SchemaDefinition} instance. * - * @param alwaysAllowNull definition null or not - * @return record schema definition + * @param clazz the class of the pojo + * @return the {@link SchemaDefinition} */ - public SchemaDefinition alwaysAllowNull(boolean alwaysAllowNull) { - - this.alwaysAllowNull = alwaysAllowNull; - properties.put("__alwaysAllowNull", this.alwaysAllowNull ? "true" : "false"); - return this; + static SchemaDefinitionBuilder builder(Class clazz) { + return DefaultImplementation.newSchemaDefinitionBuilder(clazz); } + /** * get schema whether always allow null or not * * @return schema always null or not */ - public boolean getAlwaysAllowNull() { - - return alwaysAllowNull; - } + boolean getAlwaysAllowNull(); /** * Get schema class * * @return schema class */ - public Class getClazz() { - - return clazz; - } - - + Class getClazz(); /** - * Set schema info properties + * Get schema class * - * @param properties schema info properties - * @return record schema definition + * @return schema class */ - public SchemaDefinition properties(Map properties) { + Map getProperties(); + - this.properties = properties; - properties.put("__alwaysAllowNull", alwaysAllowNull ? "true" : "false"); - return this; - } } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinitionBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinitionBuilder.java new file mode 100644 index 0000000000000..7a2ab5bc201b7 --- /dev/null +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinitionBuilder.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api.schema; + +import java.util.Map; + +/** + * Builder to build schema definition {@link SchemaDefinition}. + */ +public interface SchemaDefinitionBuilder { + + static final String ALWAYS_ALLOW_NULL = "__alwaysAllowNull"; + + /** + * Set schema whether always allow null or not + * + * @param alwaysAllowNull definition null or not + * @return record schema definition builder + */ + SchemaDefinitionBuilder withAlwaysAllowNull(boolean alwaysAllowNull); + + /** + * Set schema info properties + * + * @param properties schema info properties + * @return record schema definition + */ + SchemaDefinitionBuilder withProperties(Map properties); + + /** + * Set schema info properties + * + * @param key property key + * @param value property value + * + * @return record schema definition + */ + SchemaDefinitionBuilder addProperty(String key, String value); + + /** + * Build the schema definition. + * + * @return the schema definition. + */ + SchemaDefinition build(); + +} diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java index 97e01a6747383..d5f20624267bc 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java @@ -37,10 +37,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.PulsarClientException.UnsupportedAuthenticationException; -import org.apache.pulsar.client.api.schema.GenericSchema; -import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; -import org.apache.pulsar.client.api.schema.SchemaDefinition; -import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.*; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -72,6 +69,13 @@ public class DefaultImplementation { private static final Constructor AUTHENTICATION_TLS_String_String = getConstructor( "org.apache.pulsar.client.impl.auth.AuthenticationTls", String.class, String.class); + private static final Constructor SCHEMA_DEFINITION_BUILDER_CONSTRUCTOR = getConstructor( + "org.apache.pulsar.client.impl.schema.SchemaDefinitionBuilderImpl", Class.class); + + public static SchemaDefinitionBuilder newSchemaDefinitionBuilder(Class clazz) { + return catchExceptions(() -> (SchemaDefinitionBuilder) SCHEMA_DEFINITION_BUILDER_CONSTRUCTOR.newInstance(clazz)); + } + public static ClientBuilder newClientBuilder() { return catchExceptions(() -> CLIENT_BUILDER_IMPL.newInstance()); } @@ -183,7 +187,7 @@ public static Schema newDoubleSchema() { .newInstance()); } - public static Schema newAvroSchema(SchemaDefinition schemaDefinition) { + public static Schema newAvroSchema(SchemaDefinition schemaDefinition) { return catchExceptions( () -> (Schema) getStaticMethod("org.apache.pulsar.client.impl.schema.AvroSchema", "of", SchemaDefinition.class) .invoke(null,schemaDefinition)); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index 5204b31b99222..1265a1b721208 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -100,9 +100,7 @@ public SchemaInfo getSchemaInfo() { } private static org.apache.avro.Schema createAvroSchema(SchemaDefinition schemaDefinition) { - Class clazz = schemaDefinition.getClazz(); - return schemaDefinition.getAlwaysAllowNull() ? ReflectData.AllowNull.get().getSchema(clazz) : ReflectData.get().getSchema(clazz); } @@ -111,11 +109,11 @@ public static AvroSchema of(SchemaDefinition schemaDefinition) { } public static AvroSchema of(Class pojo) { - return AvroSchema.of(new SchemaDefinition<>(pojo)); + return AvroSchema.of(SchemaDefinition.builder(pojo).build()); } public static AvroSchema of(Class pojo, Map properties) { - SchemaDefinition schemaDefinition = new SchemaDefinition<>(pojo).properties(properties); + SchemaDefinition schemaDefinition = SchemaDefinition.builder(pojo).withProperties(properties).build(); return new AvroSchema<>(createAvroSchema(schemaDefinition), schemaDefinition); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java index f146b93cb6870..ae8f780ef1f74 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/JSONSchema.java @@ -122,10 +122,10 @@ public static JSONSchema of(SchemaDefinition schemaDefinition) { } public static JSONSchema of(Class pojo) { - return new JSONSchema<>(new SchemaDefinition<>(pojo)); + return new JSONSchema<>(SchemaDefinition.builder(pojo).build()); } public static JSONSchema of(Class pojo, Map properties) { - return new JSONSchema<>(new SchemaDefinition<>(pojo).properties(properties)); + return new JSONSchema<>(SchemaDefinition.builder(pojo).withProperties(properties).build()); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java new file mode 100644 index 0000000000000..4ecbd37d4fa69 --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl.schema; + +import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.api.schema.SchemaDefinitionBuilder; + +import java.util.HashMap; +import java.util.Map; + +/** + * Builder to build {@link org.apache.pulsar.client.api.schema.GenericRecord}. + */ +public class SchemaDefinitionBuilderImpl implements SchemaDefinitionBuilder { + + /** + * the schema definition class + */ + private Class clazz; + /** + * The flag of schema type always allow null + * + * If it's true, will make all of the pojo field generate schema + * define default can be null,false default can't be null, but it's + * false youcan define the field by yourself by the annotation@Nullable + * + */ + private boolean alwaysAllowNull = true; + + /** + * The schema info properties + */ + private Map properties = new HashMap<>(); + + public SchemaDefinitionBuilderImpl(Class clazz){ + this.clazz = clazz; + } + + @Override + public SchemaDefinitionBuilder withAlwaysAllowNull(boolean alwaysAllowNull) { + this.alwaysAllowNull = alwaysAllowNull; + return this; + } + + @Override + public SchemaDefinitionBuilder addProperty(String key, String value) { + this.properties.put(key, value); + return this; + } + + + @Override + public SchemaDefinitionBuilder withProperties(Map properties) { + this.properties = properties; + return this; + } + + @Override + public SchemaDefinition build() { + properties.put(ALWAYS_ALLOW_NULL, this.alwaysAllowNull ? "true" : "false"); + return new SchemaDefinitionImpl<>(clazz, alwaysAllowNull, properties); + } +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionImpl.java new file mode 100644 index 0000000000000..c0ce70bff7b1c --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionImpl.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl.schema; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.api.schema.SchemaDefinitionBuilder; + +import java.util.HashMap; +import java.util.Map; + +/** + * A schema definition + * {@link org.apache.pulsar.client.api.Schema} for the schema definition value. + */ +public class SchemaDefinitionImpl implements SchemaDefinition{ + + private final Class clazz; + + private boolean alwaysAllowNull; + + private Map properties; + + public SchemaDefinitionImpl(Class clazz, boolean alwaysAllowNull, Map properties) { + this.alwaysAllowNull = alwaysAllowNull; + this.properties = properties; + this.clazz = clazz; + } + /** + * get schema whether always allow null or not + * + * @return schema always null or not + */ + public boolean getAlwaysAllowNull() { + + return alwaysAllowNull; + } + + /** + * Get schema class + * + * @return schema class + */ + public Class getClazz() { + + return clazz; + } + /** + * Get schema class + * + * @return schema class + */ + public Map getProperties() { + + return properties; + } + + + +} diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index 4d8471799a44d..60bca0400a3b2 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -27,6 +27,7 @@ import org.apache.avro.Schema; import org.apache.pulsar.client.api.SchemaSerializationException; import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.api.schema.SchemaDefinitionBuilder; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Bar; import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Foo; import org.apache.pulsar.common.schema.SchemaType; @@ -39,7 +40,7 @@ public class AvroSchemaTest { @Test public void testNotAllowNullSchema() { - AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); + AvroSchema avroSchema = AvroSchema.of(SchemaDefinition.builder(Foo.class).withAlwaysAllowNull(false).build()); assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); @@ -61,7 +62,7 @@ public void testNotAllowNullSchema() { @Test public void testAllowNullSchema() { - AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(true)); + AvroSchema avroSchema = AvroSchema.of(SchemaDefinition.builder(Foo.class).build()); assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); @@ -83,7 +84,7 @@ public void testAllowNullSchema() { @Test public void testNotAllowNullEncodeAndDecode() { - AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); + AvroSchema avroSchema = AvroSchema.of(SchemaDefinition.builder(Foo.class).withAlwaysAllowNull(false).build()); Foo foo1 = new Foo(); foo1.setField1("foo1"); @@ -112,7 +113,7 @@ public void testNotAllowNullEncodeAndDecode() { @Test public void testAllowNullEncodeAndDecode() { - AvroSchema avroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class)); + AvroSchema avroSchema = AvroSchema.of(SchemaDefinition.builder(Foo.class).build()); Foo foo1 = new Foo(); foo1.setField1("foo1"); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java index 443283618ee2d..00dfa8917e5da 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java @@ -44,7 +44,7 @@ public class JSONSchemaTest { @Test public void testNotAllowNullSchema() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); + JSONSchema jsonSchema = JSONSchema.of(SchemaDefinition.builder(Foo.class).withAlwaysAllowNull(false).build()); Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema()); @@ -66,7 +66,7 @@ public void testNotAllowNullSchema() { @Test public void testAllowNullSchema() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class)); + JSONSchema jsonSchema = JSONSchema.of(SchemaDefinition.builder(Foo.class).build()); Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON); Schema.Parser parser = new Schema.Parser(); String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema()); @@ -88,7 +88,7 @@ public void testAllowNullSchema() { @Test public void testAllowNullEncodeAndDecode() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class)); + JSONSchema jsonSchema = JSONSchema.of(SchemaDefinition.builder(Foo.class).build()); Bar bar = new Bar(); bar.setField1(true); @@ -118,7 +118,7 @@ public void testAllowNullEncodeAndDecode() { @Test public void testNotAllowNullEncodeAndDecode() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); + JSONSchema jsonSchema = JSONSchema.of(SchemaDefinition.builder(Foo.class).withAlwaysAllowNull(false).build()); Foo foo1 = new Foo(); foo1.setField1("foo1"); @@ -147,8 +147,8 @@ public void testNotAllowNullEncodeAndDecode() { @Test public void testAllowNullNestedClasses() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBar.class)); - JSONSchema listJsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBarList.class)); + JSONSchema jsonSchema = JSONSchema.of(SchemaDefinition.builder(NestedBar.class).build()); + JSONSchema listJsonSchema = JSONSchema.of(SchemaDefinition.builder(NestedBarList.class).build()); Bar bar = new Bar(); bar.setField1(true); @@ -174,8 +174,8 @@ public void testAllowNullNestedClasses() { @Test public void testNotAllowNullNestedClasses() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBar.class).alwaysAllowNull(false)); - JSONSchema listJsonSchema = JSONSchema.of(new SchemaDefinition<>(NestedBarList.class).alwaysAllowNull(false)); + JSONSchema jsonSchema = JSONSchema.of(SchemaDefinition.builder(NestedBar.class).withAlwaysAllowNull(false).build()); + JSONSchema listJsonSchema = JSONSchema.of(SchemaDefinition.builder(NestedBarList.class).withAlwaysAllowNull(false).build()); Bar bar = new Bar(); bar.setField1(true); @@ -229,25 +229,25 @@ public void testNotAllowNullCorrectPolymorphism() { derivedDerivedFoo.setDerivedFoo(derivedFoo); // schema for base class - JSONSchema baseJsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class)); + JSONSchema baseJsonSchema = JSONSchema.of(SchemaDefinition.builder(Foo.class).build()); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(foo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedFoo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedDerivedFoo)), foo); // schema for derived class - JSONSchema derivedJsonSchema = JSONSchema.of(new SchemaDefinition<>(DerivedFoo.class)); + JSONSchema derivedJsonSchema = JSONSchema.of(SchemaDefinition.builder(DerivedFoo.class).build()); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedFoo)), derivedFoo); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedDerivedFoo)), derivedFoo); //schema for derived derived class JSONSchema derivedDerivedJsonSchema - = JSONSchema.of(new SchemaDefinition<>(SchemaTestUtils.DerivedDerivedFoo.class)); + = JSONSchema.of(SchemaDefinition.builder(SchemaTestUtils.DerivedDerivedFoo.class).build()); Assert.assertEquals(derivedDerivedJsonSchema.decode(derivedDerivedJsonSchema.encode(derivedDerivedFoo)), derivedDerivedFoo); } @Test(expectedExceptions = SchemaSerializationException.class) public void testAllowNullDecodeWithInvalidContent() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class)); + JSONSchema jsonSchema = JSONSchema.of(SchemaDefinition.builder(Foo.class).build()); jsonSchema.decode(new byte[0]); } @@ -281,25 +281,25 @@ public void testAllowNullCorrectPolymorphism() { derivedDerivedFoo.setDerivedFoo(derivedFoo); // schema for base class - JSONSchema baseJsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); + JSONSchema baseJsonSchema = JSONSchema.of(SchemaDefinition.builder(Foo.class).withAlwaysAllowNull(false).build()); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(foo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedFoo)), foo); Assert.assertEquals(baseJsonSchema.decode(baseJsonSchema.encode(derivedDerivedFoo)), foo); // schema for derived class - JSONSchema derivedJsonSchema = JSONSchema.of(new SchemaDefinition<>(DerivedFoo.class).alwaysAllowNull(false)); + JSONSchema derivedJsonSchema = JSONSchema.of(SchemaDefinition.builder(DerivedFoo.class).withAlwaysAllowNull(false).build()); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedFoo)), derivedFoo); Assert.assertEquals(derivedJsonSchema.decode(derivedJsonSchema.encode(derivedDerivedFoo)), derivedFoo); //schema for derived derived class JSONSchema derivedDerivedJsonSchema - = JSONSchema.of(new SchemaDefinition<>(SchemaTestUtils.DerivedDerivedFoo.class).alwaysAllowNull(false)); + = JSONSchema.of(SchemaDefinition.builder(SchemaTestUtils.DerivedDerivedFoo.class).withAlwaysAllowNull(false).build()); Assert.assertEquals(derivedDerivedJsonSchema.decode(derivedDerivedJsonSchema.encode(derivedDerivedFoo)), derivedDerivedFoo); } @Test(expectedExceptions = SchemaSerializationException.class) public void testNotAllowNullDecodeWithInvalidContent() { - JSONSchema jsonSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); + JSONSchema jsonSchema = JSONSchema.of(SchemaDefinition.builder(Foo.class).withAlwaysAllowNull(false).build()); jsonSchema.decode(new byte[0]); } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java index 5f6e5716e4612..3c83167bdf9f4 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/KeyValueSchemaTest.java @@ -36,8 +36,8 @@ public class KeyValueSchemaTest { @Test public void testAllowNullAvroSchemaCreate() { - AvroSchema fooSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class)); - AvroSchema barSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class)); + AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder(Foo.class).build()); + AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder(Bar.class).build()); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.AVRO); @@ -61,12 +61,12 @@ public void testAllowNullAvroSchemaCreate() { @Test public void testNotAllowNullAvroSchemaCreate() { - AvroSchema fooSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); - AvroSchema barSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false)); + AvroSchema fooSchema = AvroSchema.of(SchemaDefinition.builder(Foo.class).withAlwaysAllowNull(false).build()); + AvroSchema barSchema = AvroSchema.of(SchemaDefinition.builder(Bar.class).withAlwaysAllowNull(false).build()); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); - Schema> keyValueSchema2 = Schema.KeyValue(AvroSchema.of(new SchemaDefinition<> - (Foo.class).alwaysAllowNull(false)),AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false))); + Schema> keyValueSchema2 = Schema.KeyValue(AvroSchema.of(SchemaDefinition.builder(Foo.class).withAlwaysAllowNull(false).build()), + AvroSchema.of(SchemaDefinition.builder(Bar.class).withAlwaysAllowNull(false).build())); assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); @@ -87,8 +87,8 @@ public void testNotAllowNullAvroSchemaCreate() { @Test public void testAllowNullJsonSchemaCreate() { - JSONSchema fooSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class)); - JSONSchema barSchema = JSONSchema.of(new SchemaDefinition<>(Bar.class)); + JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder(Foo.class).build()); + JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder(Bar.class).build()); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); Schema> keyValueSchema2 = Schema.KeyValue(Foo.class, Bar.class, SchemaType.JSON); @@ -120,15 +120,15 @@ public void testAllowNullJsonSchemaCreate() { @Test public void testNotAllowNullJsonSchemaCreate() { - JSONSchema fooSchema = JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); - JSONSchema barSchema = JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false)); + JSONSchema fooSchema = JSONSchema.of(SchemaDefinition.builder(Foo.class).withAlwaysAllowNull(false).build()); + JSONSchema barSchema = JSONSchema.of(SchemaDefinition.builder(Bar.class).withAlwaysAllowNull(false).build()); Schema> keyValueSchema1 = Schema.KeyValue(fooSchema, barSchema); - Schema> keyValueSchema2 = Schema.KeyValue(JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)), - JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false))); + Schema> keyValueSchema2 = Schema.KeyValue(JSONSchema.of(SchemaDefinition.builder(Foo.class).withAlwaysAllowNull(false).build()), + JSONSchema.of(SchemaDefinition.builder(Bar.class).withAlwaysAllowNull(false).build())); - Schema> keyValueSchema3 = Schema.KeyValue(JSONSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)), - JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false))); + Schema> keyValueSchema3 = Schema.KeyValue(JSONSchema.of(SchemaDefinition.builder(Foo.class).withAlwaysAllowNull(false).build()), + JSONSchema.of(SchemaDefinition.builder(Bar.class).withAlwaysAllowNull(false).build())); assertEquals(keyValueSchema1.getSchemaInfo().getType(), SchemaType.KEY_VALUE); assertEquals(keyValueSchema2.getSchemaInfo().getType(), SchemaType.KEY_VALUE); @@ -181,8 +181,8 @@ public void testAllowNullSchemaEncodeAndDecode() { @Test public void testNotAllowNullSchemaEncodeAndDecode() { - Schema keyValueSchema = Schema.KeyValue(JSONSchema.of(new SchemaDefinition<>(Foo.class) - .alwaysAllowNull(false)),JSONSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false))); + Schema keyValueSchema = Schema.KeyValue(JSONSchema.of(SchemaDefinition.builder(Foo.class).withAlwaysAllowNull(false).build()), + JSONSchema.of(SchemaDefinition.builder(Bar.class).withAlwaysAllowNull(false).build())); Bar bar = new Bar(); bar.setField1(true); @@ -207,8 +207,8 @@ public void testNotAllowNullSchemaEncodeAndDecode() { @Test public void testAllowNullBytesSchemaEncodeAndDecode() { - AvroSchema fooAvroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class)); - AvroSchema barAvroSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class)); + AvroSchema fooAvroSchema = AvroSchema.of(SchemaDefinition.builder(Foo.class).build()); + AvroSchema barAvroSchema = AvroSchema.of(SchemaDefinition.builder(Bar.class).build()); Bar bar = new Bar(); bar.setField1(true); @@ -236,8 +236,8 @@ public void testAllowNullBytesSchemaEncodeAndDecode() { @Test public void testNotAllowNullBytesSchemaEncodeAndDecode() { - AvroSchema fooAvroSchema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(false)); - AvroSchema barAvroSchema = AvroSchema.of(new SchemaDefinition<>(Bar.class).alwaysAllowNull(false)); + AvroSchema fooAvroSchema = AvroSchema.of(SchemaDefinition.builder(Foo.class).withAlwaysAllowNull(false).build()); + AvroSchema barAvroSchema = AvroSchema.of(SchemaDefinition.builder(Bar.class).withAlwaysAllowNull(false).build()); Bar bar = new Bar(); bar.setField1(true); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionGenericSchemaProviderTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionGenericSchemaProviderTest.java index 2eb65ce3f0a8e..bfc03367084cf 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionGenericSchemaProviderTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionGenericSchemaProviderTest.java @@ -29,6 +29,7 @@ import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.client.impl.schema.SchemaTestUtils; +import org.apache.pulsar.client.tutorial.JsonPojo; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.schema.SchemaInfo; import org.testng.annotations.BeforeMethod; @@ -55,7 +56,7 @@ public void setup() { @Test public void testGetSchema() { CompletableFuture> completableFuture = new CompletableFuture<>(); - SchemaInfo schemaInfo = AvroSchema.of(new SchemaDefinition<>(SchemaTestUtils.Foo.class)).getSchemaInfo(); + SchemaInfo schemaInfo = AvroSchema.of(SchemaDefinition.builder(SchemaTestUtils.Foo.class).build()).getSchemaInfo(); completableFuture.complete(Optional.of(schemaInfo)); when(schemaProvider.getPulsarClient().getLookup() .getSchema( diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java index d148d2c2bad38..59591c2c70f1a 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java @@ -29,6 +29,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.JSONSchema; +import org.apache.pulsar.client.impl.schema.SchemaTestUtils; @Slf4j public class SampleAsyncProducerWithSchema { @@ -36,7 +37,7 @@ public class SampleAsyncProducerWithSchema { public static void main(String[] args) throws IOException { PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("http://localhost:8080").build(); - Producer producer = pulsarClient.newProducer(JSONSchema.of(new SchemaDefinition<>(JsonPojo.class))).topic("persistent://my-property/use/my-ns/my-topic") + Producer producer = pulsarClient.newProducer(JSONSchema.of(SchemaDefinition.builder(JsonPojo.class).build())).topic("persistent://my-property/use/my-ns/my-topic") .sendTimeout(3, TimeUnit.SECONDS).create(); List> futures = Lists.newArrayList(); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java index d851e171d4863..7fa12449d13cb 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java @@ -25,13 +25,14 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.JSONSchema; +import org.apache.pulsar.client.impl.testMySelf.ConsumerSchema; public class SampleConsumerWithSchema { public static void main(String[] args) throws PulsarClientException, JsonProcessingException { PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("http://localhost:8080").build(); - Consumer consumer = pulsarClient.newConsumer(JSONSchema.of(new SchemaDefinition<>(JsonPojo.class))) // + Consumer consumer = pulsarClient.newConsumer(JSONSchema.of(SchemaDefinition.builder(JsonPojo.class).build())) // .topic("persistent://my-property/use/my-ns/my-topic") // .subscriptionName("my-subscription-name").subscribe(); diff --git a/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java b/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java index 109af0112ba65..5ca9a27652b37 100644 --- a/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java +++ b/pulsar-io/hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java @@ -109,7 +109,7 @@ public void TestOpenAndWriteSink() throws Exception { obj.setAddress("address_value"); obj.setAge(30); obj.setFlag(true); - AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(true)); + AvroSchema schema = AvroSchema.of(SchemaDefinition.builder(Foo.class).build()); byte[] bytes = schema.encode(obj); ByteBuf payload = Unpooled.copiedBuffer(bytes); diff --git a/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java b/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java index a3a0421fd96c1..f514e81e4f4d9 100644 --- a/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java +++ b/pulsar-io/jdbc/src/test/java/org/apache/pulsar/io/jdbc/JdbcSinkTest.java @@ -96,7 +96,7 @@ public void TestOpenAndWriteSink() throws Exception { obj.setField1("ValueOfField1"); obj.setField2("ValueOfField1"); obj.setField3(3); - AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(true)); + AvroSchema schema = AvroSchema.of(SchemaDefinition.builder(Foo.class).build()); byte[] bytes = schema.encode(obj); ByteBuf payload = Unpooled.copiedBuffer(bytes); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java index 4d5e46d2a439b..5a6c27333e50f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcSinkTester.java @@ -61,7 +61,7 @@ public static class Foo { private static final String NAME = "jdbc"; private static final String MYSQL = "mysql"; - private AvroSchema schema = AvroSchema.of(new SchemaDefinition<>(Foo.class).alwaysAllowNull(true)); + private AvroSchema schema = AvroSchema.of(SchemaDefinition.builder(Foo.class).build()); private String tableName = "test"; private Connection connection; From 7c78762e77d6421ed94278174b0895f6c2316565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 14 Mar 2019 11:19:55 +0800 Subject: [PATCH 23/35] delete package dependency --- .../apache/pulsar/client/tutorial/SampleConsumerWithSchema.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java index 7fa12449d13cb..2fc4a1f927042 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleConsumerWithSchema.java @@ -25,7 +25,6 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.JSONSchema; -import org.apache.pulsar.client.impl.testMySelf.ConsumerSchema; public class SampleConsumerWithSchema { public static void main(String[] args) throws PulsarClientException, JsonProcessingException { From 7ced54a8045e880bea07a9a92169e22a21be775c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 14 Mar 2019 11:23:47 +0800 Subject: [PATCH 24/35] delete package dependency --- .../apache/pulsar/client/api/schema/SchemaDefinition.java | 7 +------ .../pulsar/client/impl/schema/SchemaDefinitionImpl.java | 6 +----- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java index 05b76b9776a44..23b4650b6c84c 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinition.java @@ -18,14 +18,9 @@ */ package org.apache.pulsar.client.api.schema; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; -import org.apache.pulsar.client.api.ClientBuilder; -import org.apache.pulsar.client.api.PulsarClient; + import org.apache.pulsar.client.internal.DefaultImplementation; -import java.util.HashMap; import java.util.Map; /** diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionImpl.java index c0ce70bff7b1c..09efa686901f1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionImpl.java @@ -18,13 +18,9 @@ */ package org.apache.pulsar.client.impl.schema; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.ToString; + import org.apache.pulsar.client.api.schema.SchemaDefinition; -import org.apache.pulsar.client.api.schema.SchemaDefinitionBuilder; -import java.util.HashMap; import java.util.Map; /** From 96ffcf96357cad7182afe69038b58e0e21a4f9df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 14 Mar 2019 11:40:49 +0800 Subject: [PATCH 25/35] Modify the schema defination of topic schema --- .../java/org/apache/pulsar/functions/source/TopicSchema.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java index b4cf6f7b9d5ec..f248e90bc116a 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java @@ -125,10 +125,10 @@ private static Schema newSchemaInstance(Class clazz, SchemaType type) return (Schema) Schema.STRING; case AVRO: - return AvroSchema.of(new SchemaDefinition<>(clazz)); + return AvroSchema.of(SchemaDefinition.builder(clazz).build()); case JSON: - return JSONSchema.of(new SchemaDefinition<>(clazz)); + return JSONSchema.of(SchemaDefinition.builder(clazz).build()); case KEY_VALUE: return (Schema)Schema.KV_BYTES(); From 4ed1db8fb061e0b92c7530bffdd14f5d097ac611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 14 Mar 2019 13:20:08 +0800 Subject: [PATCH 26/35] Remove the dependency --- .../pulsar/client/tutorial/SampleAsyncProducerWithSchema.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java index 59591c2c70f1a..e19c4e71c02c1 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/tutorial/SampleAsyncProducerWithSchema.java @@ -29,8 +29,6 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.JSONSchema; -import org.apache.pulsar.client.impl.schema.SchemaTestUtils; - @Slf4j public class SampleAsyncProducerWithSchema { From 12ba1cbd13b679802aa0f5bf125d6c2e552c8f5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 14 Mar 2019 13:58:14 +0800 Subject: [PATCH 27/35] Modify the test --- .../api/SimpleTypedProducerConsumerTest.java | 20 +++++++++---------- .../schema/SchemaDefinitionBuilderImpl.java | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java index 4b1742b77f305..b801db8234f3b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java @@ -64,7 +64,7 @@ public void testJsonProducerAndConsumer() throws Exception { log.info("-- Starting {} test --", methodName); JSONSchema jsonSchema = - JSONSchema.of(new SchemaDefinition<>(JsonEncodedPojo.class)); + JSONSchema.of(SchemaDefinition.builder(JsonEncodedPojo.class).build()); Consumer consumer = pulsarClient .newConsumer(jsonSchema) @@ -109,7 +109,7 @@ public void testJsonProducerAndConsumerWithPrestoredSchema() throws Exception { log.info("-- Starting {} test --", methodName); JSONSchema jsonSchema = - JSONSchema.of(new SchemaDefinition<>(JsonEncodedPojo.class)); + JSONSchema.of(SchemaDefinition.builder(JsonEncodedPojo.class).build()); pulsar.getSchemaRegistryService() .putSchemaIfAbsent("my-property/my-ns/my-topic1", @@ -167,7 +167,7 @@ public void testJsonConsumerWithWrongCorruptedSchema() throws Exception { ).get(); Consumer consumer = pulsarClient - .newConsumer(JSONSchema.of(new SchemaDefinition<>(JsonEncodedPojo.class))) + .newConsumer(JSONSchema.of(SchemaDefinition.builder(JsonEncodedPojo.class).build())) .topic("persistent://my-property/use/my-ns/my-topic1") .subscriptionName("my-subscriber-name") .subscribe(); @@ -195,7 +195,7 @@ public void testJsonProducerWithWrongCorruptedSchema() throws Exception { ).get(); Producer producer = pulsarClient - .newProducer(JSONSchema.of(new SchemaDefinition<>(JsonEncodedPojo.class))) + .newProducer(JSONSchema.of(SchemaDefinition.builder(JsonEncodedPojo.class).build())) .topic("persistent://my-property/use/my-ns/my-topic1") .create(); @@ -274,7 +274,7 @@ public void testProtobufConsumerWithWrongPrestoredSchema() throws Exception { ).get(); Consumer consumer = pulsarClient - .newConsumer(AvroSchema.of(new SchemaDefinition<>(org.apache.pulsar.client.api.schema.proto.Test.TestMessageWrong.class))) + .newConsumer(AvroSchema.of(SchemaDefinition.builder(org.apache.pulsar.client.api.schema.proto.Test.TestMessageWrong.class).build())) .topic("persistent://my-property/use/my-ns/my-topic1") .subscriptionName("my-subscriber-name") .subscribe(); @@ -287,7 +287,7 @@ public void testAvroProducerAndConsumer() throws Exception { log.info("-- Starting {} test --", methodName); AvroSchema avroSchema = - AvroSchema.of(new SchemaDefinition<>(AvroEncodedPojo.class)); + AvroSchema.of(SchemaDefinition.builder(AvroEncodedPojo.class).build()); Consumer consumer = pulsarClient .newConsumer(avroSchema) @@ -356,7 +356,7 @@ public void testAvroConsumerWithWrongPrestoredSchema() throws Exception { ).get(); Consumer consumer = pulsarClient - .newConsumer(AvroSchema.of(new SchemaDefinition<>(AvroEncodedPojo.class))) + .newConsumer(AvroSchema.of(SchemaDefinition.builder(AvroEncodedPojo.class).build())) .topic("persistent://my-property/use/my-ns/my-topic1") .subscriptionName("my-subscriber-name") .subscribe(); @@ -455,7 +455,7 @@ public void testAvroProducerAndAutoSchemaConsumer() throws Exception { log.info("-- Starting {} test --", methodName); AvroSchema avroSchema = - AvroSchema.of(new SchemaDefinition<>(AvroEncodedPojo.class)); + AvroSchema.of(SchemaDefinition.builder(AvroEncodedPojo.class).build()); Producer producer = pulsarClient .newProducer(avroSchema) @@ -503,7 +503,7 @@ public void testAvroProducerAndAutoSchemaReader() throws Exception { log.info("-- Starting {} test --", methodName); AvroSchema avroSchema = - AvroSchema.of(new SchemaDefinition<>(AvroEncodedPojo.class)); + AvroSchema.of(SchemaDefinition.builder(AvroEncodedPojo.class).build()); Producer producer = pulsarClient .newProducer(avroSchema) @@ -549,7 +549,7 @@ public void testAutoBytesProducer() throws Exception { log.info("-- Starting {} test --", methodName); AvroSchema avroSchema = - AvroSchema.of(new SchemaDefinition<>(AvroEncodedPojo.class)); + AvroSchema.of(SchemaDefinition.builder(AvroEncodedPojo.class).build()); try (Producer producer = pulsarClient .newProducer(avroSchema) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java index 4ecbd37d4fa69..3ed431df3196c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java @@ -38,7 +38,7 @@ public class SchemaDefinitionBuilderImpl implements SchemaDefinitionBuilder Date: Thu, 14 Mar 2019 14:03:46 +0800 Subject: [PATCH 28/35] move the ALWAYS_ALLOW_NULL field location --- .../pulsar/client/api/schema/SchemaDefinitionBuilder.java | 4 +--- .../client/impl/schema/SchemaDefinitionBuilderImpl.java | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinitionBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinitionBuilder.java index 7a2ab5bc201b7..4cdc5bc89c02d 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinitionBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaDefinitionBuilder.java @@ -24,9 +24,7 @@ * Builder to build schema definition {@link SchemaDefinition}. */ public interface SchemaDefinitionBuilder { - - static final String ALWAYS_ALLOW_NULL = "__alwaysAllowNull"; - + /** * Set schema whether always allow null or not * diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java index 3ed431df3196c..a6c4bb03f4d86 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/SchemaDefinitionBuilderImpl.java @@ -29,6 +29,8 @@ */ public class SchemaDefinitionBuilderImpl implements SchemaDefinitionBuilder { + public static final String ALWAYS_ALLOW_NULL = "__alwaysAllowNull"; + /** * the schema definition class */ From 0271117c5828402b5dd46e0187eab4392af8dfa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Thu, 14 Mar 2019 15:51:48 +0800 Subject: [PATCH 29/35] modify the test --- .../sql/presto/TestPulsarConnector.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index 522233c07df21..f81d10fbdd9ef 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -220,21 +220,21 @@ public static class Boo { partitionedTopicsToPartitions.put(PARTITIONED_TOPIC_6.toString(), 7); topicsToSchemas = new HashMap<>(); - topicsToSchemas.put(TOPIC_1.getSchemaName(), AvroSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); - topicsToSchemas.put(TOPIC_2.getSchemaName(), AvroSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); - topicsToSchemas.put(TOPIC_3.getSchemaName(), AvroSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); - topicsToSchemas.put(TOPIC_4.getSchemaName(), JSONSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); - topicsToSchemas.put(TOPIC_5.getSchemaName(), JSONSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); - topicsToSchemas.put(TOPIC_6.getSchemaName(), JSONSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); + topicsToSchemas.put(TOPIC_1.getSchemaName(), AvroSchema.of(SchemaDefinition.builder(TestPulsarMetadata.Foo.class).build()).getSchemaInfo()); + topicsToSchemas.put(TOPIC_2.getSchemaName(), AvroSchema.of(SchemaDefinition.builder(TestPulsarMetadata.Foo.class).build()).getSchemaInfo()); + topicsToSchemas.put(TOPIC_3.getSchemaName(), AvroSchema.of(SchemaDefinition.builder(TestPulsarMetadata.Foo.class).build()).getSchemaInfo()); + topicsToSchemas.put(TOPIC_4.getSchemaName(), JSONSchema.of(SchemaDefinition.builder(TestPulsarMetadata.Foo.class).build()).getSchemaInfo()); + topicsToSchemas.put(TOPIC_5.getSchemaName(), JSONSchema.of(SchemaDefinition.builder(TestPulsarMetadata.Foo.class).build()).getSchemaInfo()); + topicsToSchemas.put(TOPIC_6.getSchemaName(), JSONSchema.of(SchemaDefinition.builder(TestPulsarMetadata.Foo.class).build()).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_1.getSchemaName(), AvroSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_1.getSchemaName(), AvroSchema.of(SchemaDefinition.builder(TestPulsarMetadata.Foo.class).build()).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_2.getSchemaName(), AvroSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_3.getSchemaName(), AvroSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_4.getSchemaName(), JSONSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_5.getSchemaName(), JSONSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); - topicsToSchemas.put(PARTITIONED_TOPIC_6.getSchemaName(), JSONSchema.of(new SchemaDefinition<>(TestPulsarMetadata.Foo.class)).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_2.getSchemaName(), AvroSchema.of(SchemaDefinition.builder(TestPulsarMetadata.Foo.class).build()).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_3.getSchemaName(), AvroSchema.of(SchemaDefinition.builder(TestPulsarMetadata.Foo.class).build()).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_4.getSchemaName(), JSONSchema.of(SchemaDefinition.builder(TestPulsarMetadata.Foo.class).build()).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_5.getSchemaName(), JSONSchema.of(SchemaDefinition.builder(TestPulsarMetadata.Foo.class).build()).getSchemaInfo()); + topicsToSchemas.put(PARTITIONED_TOPIC_6.getSchemaName(), JSONSchema.of(SchemaDefinition.builder(TestPulsarMetadata.Foo.class).build()).getSchemaInfo()); fooTypes = new HashMap<>(); fooTypes.put("field1", IntegerType.INTEGER); @@ -623,7 +623,7 @@ private static List getTopicEntries(String topicSchemaName) { .setProducerName("test-producer").setSequenceId(i) .setPublishTime(currentTimeMs + i).build(); - Schema schema = topicsToSchemas.get(topicSchemaName).getType() == SchemaType.AVRO ? AvroSchema.of(new SchemaDefinition<>(Foo.class)) : JSONSchema.of(new SchemaDefinition<>(Foo.class)); + Schema schema = topicsToSchemas.get(topicSchemaName).getType() == SchemaType.AVRO ? AvroSchema.of(SchemaDefinition.builder(Foo.class).build()) : JSONSchema.of(SchemaDefinition.builder(Foo.class).build()); org.apache.pulsar.shade.io.netty.buffer.ByteBuf payload = org.apache.pulsar.shade.io.netty.buffer.Unpooled .copiedBuffer(schema.encode(foo)); From 4ef4be4459f913cc2686e026a0998e30880a00c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Fri, 15 Mar 2019 18:17:14 +0800 Subject: [PATCH 30/35] Add genericity --- .../java/org/apache/pulsar/functions/source/TopicSchema.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java index bf47334fc3521..2ac42f46641f1 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java @@ -125,10 +125,10 @@ private static Schema newSchemaInstance(Class clazz, SchemaType type) return (Schema) Schema.STRING; case AVRO: - return AvroSchema.of(SchemaDefinition.builder().withPojo(clazz).build()); + return AvroSchema.of(SchemaDefinition.builder().withPojo(clazz).build()); case JSON: - return JSONSchema.of(SchemaDefinition.builder().withPojo(clazz).build()); + return JSONSchema.of(SchemaDefinition.builder().withPojo(clazz).build()); case KEY_VALUE: return (Schema)Schema.KV_BYTES(); From ac4f04740fada2c8f24cc69b1ff9e6ecd7aacc2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Sat, 16 Mar 2019 09:23:14 +0800 Subject: [PATCH 31/35] modify the test --- .../org/apache/pulsar/client/impl/schema/StructSchema.java | 4 ++-- .../org/apache/pulsar/sql/presto/TestPulsarConnector.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java index 9477d0f2ae3df..31156d4d49925 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java @@ -64,8 +64,8 @@ public SchemaInfo getSchemaInfo() { } protected static org.apache.avro.Schema createAvroSchema(SchemaDefinition schemaDefinition) { - Class clazz = schemaDefinition.getPojo(); - return schemaDefinition.getAlwaysAllowNull() ? ReflectData.AllowNull.get().getSchema(clazz) : ReflectData.get().getSchema(clazz); + Class pojo = schemaDefinition.getPojo(); + return schemaDefinition.getAlwaysAllowNull() ? ReflectData.AllowNull.get().getSchema(pojo) : ReflectData.get().getSchema(pojo); } protected static org.apache.avro.Schema parseAvroSchema(String jsonDef) { diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java index 61038a3de47d7..7a62f675a91eb 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java @@ -623,7 +623,7 @@ private static List getTopicEntries(String topicSchemaName) { .setProducerName("test-producer").setSequenceId(i) .setPublishTime(currentTimeMs + i).build(); - Schema schema = topicsToSchemas.get(topicSchemaName).getType() == SchemaType.AVRO ? AvroSchema.of(SchemaDefinition.builder(Foo.class).build()) : JSONSchema.of(SchemaDefinition.builder(Foo.class).build()); + Schema schema = topicsToSchemas.get(topicSchemaName).getType() == SchemaType.AVRO ? AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()) : JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); org.apache.pulsar.shade.io.netty.buffer.ByteBuf payload = org.apache.pulsar.shade.io.netty.buffer.Unpooled .copiedBuffer(schema.encode(foo)); @@ -873,7 +873,7 @@ public void run() { .setProducerName("test-producer").setSequenceId(positions.get(topic)) .setPublishTime(System.currentTimeMillis()).build(); - Schema schema = topicsToSchemas.get(schemaName).getType() == SchemaType.AVRO ? AvroSchema.of(new SchemaDefinition<>(Foo.class)) : JSONSchema.of(new SchemaDefinition<>(Foo.class)); + Schema schema = topicsToSchemas.get(schemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class) : JSONSchema.of(Foo.class); org.apache.pulsar.shade.io.netty.buffer.ByteBuf payload = org.apache.pulsar.shade.io.netty.buffer.Unpooled .copiedBuffer(schema.encode(foo)); From 841dad54a94a9024d4f088c9245c0635eef65b8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Sat, 16 Mar 2019 11:35:31 +0800 Subject: [PATCH 32/35] modify the test --- .../apache/pulsar/tests/integration/presto/TestBasicPresto.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java index 754361d830a8a..48fc3c976cd61 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java @@ -87,7 +87,7 @@ public void testSimpleSQLQuery(boolean isBatched) throws Exception { final String stocksTopic = "stocks"; @Cleanup - Producer producer = pulsarClient.newProducer(JSONSchema.of(new SchemaDefinition<>(Stock.class))) + Producer producer = pulsarClient.newProducer(JSONSchema.of(Stock.class)) .topic(stocksTopic) .enableBatching(isBatched) .create(); From eab205f429183a9efee55305fbd3479de2fc84b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Sat, 16 Mar 2019 15:18:05 +0800 Subject: [PATCH 33/35] modify the test --- .../pulsar/tests/integration/functions/PulsarFunctionsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 04e27fb5300a7..0bb230ccca5e2 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -162,7 +162,7 @@ private void runSinkTester(SinkTester tester, boolean builtin) throws Exception // produce messages Map kvs; if (tester instanceof JdbcSinkTester) { - kvs = produceSchemaMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(new SchemaDefinition<>(JdbcSinkTester.Foo.class).alwaysAllowNull(true))); + kvs = produceSchemaMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcSinkTester.Foo.class)); } else { kvs = produceMessagesToInputTopic(inputTopicName, numMessages); } From 7c59bac8d925e2fbc9fd0b5c74fe3c4cf322d9a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Sun, 17 Mar 2019 16:23:11 +0800 Subject: [PATCH 34/35] remove the dependency --- .../apache/pulsar/tests/integration/presto/TestBasicPresto.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java index 48fc3c976cd61..48cffb7022022 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestBasicPresto.java @@ -22,7 +22,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.tests.integration.docker.ContainerExecException; import org.apache.pulsar.tests.integration.docker.ContainerExecResult; From 36a89771d2054eba91bba5e44cb3918301d7400d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9B=E6=90=8F?= <237716289@qq.com> Date: Mon, 18 Mar 2019 08:49:06 +0800 Subject: [PATCH 35/35] remove the dependency --- .../pulsar/tests/integration/functions/PulsarFunctionsTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 0bb230ccca5e2..18eaba945fb88 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -40,7 +40,6 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.FunctionStats;