From e3c344b71159305a69e9d8a7a3748b2fa0efe153 Mon Sep 17 00:00:00 2001 From: guangning Date: Tue, 31 Dec 2019 15:33:38 +0800 Subject: [PATCH 01/36] Add avro parse --- pom.xml | 5 ++++ .../apache/pulsar/functions/api/Record.java | 5 ++++ pulsar-io/kafka-connect-adaptor/pom.xml | 7 ++++++ .../io/kafka/connect/KafkaConnectSource.java | 24 +++++++++++++++++++ 4 files changed, 41 insertions(+) diff --git a/pom.xml b/pom.xml index 471abc248f27a..eb3367f59706b 100644 --- a/pom.xml +++ b/pom.xml @@ -220,6 +220,7 @@ flexible messaging model and an intuitive client API. 3.25.0-GA 2.3.1 1.5.0 + 5.2.2 0.6.1 @@ -1715,5 +1716,9 @@ flexible messaging model and an intuitive client API. spring-plugins-release https://repo.spring.io/plugins-release/ + + confluent + http://packages.confluent.io/maven/ + diff --git a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java index 3c3d7e8d78efe..9280cd71f1874 100644 --- a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java +++ b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java @@ -19,6 +19,7 @@ package org.apache.pulsar.functions.api; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Schema; import java.util.Collections; import java.util.Map; @@ -43,6 +44,10 @@ default Optional getKey() { return Optional.empty(); } + default Schema getSchema() { + return null; + } + /** * Retrieves the actual data of the record. * diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 854489b473637..250c73c8028a3 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -97,6 +97,13 @@ test-jar + + + io.confluent + kafka-connect-avro-converter + ${confluent.version} + + diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index ed059bfa794fc..8d50f7e6bc84c 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -34,6 +34,8 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; + +import io.confluent.connect.avro.AvroData; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.connect.runtime.TaskConfig; @@ -45,6 +47,8 @@ import org.apache.kafka.connect.storage.OffsetStorageReader; import org.apache.kafka.connect.storage.OffsetStorageReaderImpl; import org.apache.kafka.connect.storage.OffsetStorageWriter; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.schema.KeyValueSchema; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.core.Source; @@ -188,14 +192,27 @@ private class KafkaSourceRecord implements Record> { @Getter Optional destinationTopic; + private final AvroData avroData; + + private final org.apache.avro.Schema keyAvroSchema; + + private final org.apache.avro.Schema valueAvroSchema; + KafkaSourceRecord(SourceRecord srcRecord) { byte[] keyBytes = keyConverter.fromConnectData( srcRecord.topic(), srcRecord.keySchema(), srcRecord.key()); byte[] valueBytes = valueConverter.fromConnectData( srcRecord.topic(), srcRecord.valueSchema(), srcRecord.value()); + this.avroData = new AvroData(1000); this.key = keyBytes != null ? Optional.of(Base64.getEncoder().encodeToString(keyBytes)) : Optional.empty(); this.value = new KeyValue(keyBytes, valueBytes); + keyAvroSchema = (org.apache.avro.Schema) this.avroData.fromConnectData( + srcRecord.keySchema(), keyBytes); + valueAvroSchema = (org.apache.avro.Schema) this.avroData.fromConnectData( + srcRecord.valueSchema(), valueBytes); + this.avroData.fromConnectData(srcRecord.valueSchema(), valueBytes); + this.topicName = Optional.of(srcRecord.topic()); this.eventTime = Optional.ofNullable(srcRecord.timestamp()); this.partitionId = Optional.of(srcRecord.sourcePartition() @@ -206,6 +223,13 @@ private class KafkaSourceRecord implements Record> { this.destinationTopic = Optional.of(topicNamespace + "/" + srcRecord.topic()); } + @Override + public Schema> getSchema() { + return new KeyValueSchema<>( + keyAvroSchema, valueAvroSchema + ); + } + @Override public Optional getRecordSequence() { return RECORD_SEQUENCE; From a0699dfc1559484b275eeffade47adf3bb194f08 Mon Sep 17 00:00:00 2001 From: guangning Date: Wed, 1 Jan 2020 10:11:39 +0800 Subject: [PATCH 02/36] Add Class KafkaSchema --- .../io/kafka/connect/KafkaConnectSource.java | 22 +++- .../io/kafka/connect/schema/KafkaSchema.java | 121 ++++++++++++++++++ 2 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchema.java diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index 8d50f7e6bc84c..f11429d6df3f6 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -53,6 +53,7 @@ import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.core.Source; import org.apache.pulsar.io.core.SourceContext; +import org.apache.pulsar.io.kafka.connect.schema.KafkaSchema; /** * A pulsar source that runs @@ -198,10 +199,18 @@ private class KafkaSourceRecord implements Record> { private final org.apache.avro.Schema valueAvroSchema; + private final KafkaSchema keySchema; + + private final KafkaSchema valueSchema; + + private byte[] keyBytes; + + private byte[] valueBytes; + KafkaSourceRecord(SourceRecord srcRecord) { - byte[] keyBytes = keyConverter.fromConnectData( + keyBytes = keyConverter.fromConnectData( srcRecord.topic(), srcRecord.keySchema(), srcRecord.key()); - byte[] valueBytes = valueConverter.fromConnectData( + valueBytes = valueConverter.fromConnectData( srcRecord.topic(), srcRecord.valueSchema(), srcRecord.value()); this.avroData = new AvroData(1000); this.key = keyBytes != null ? Optional.of(Base64.getEncoder().encodeToString(keyBytes)) : Optional.empty(); @@ -211,7 +220,10 @@ private class KafkaSourceRecord implements Record> { srcRecord.keySchema(), keyBytes); valueAvroSchema = (org.apache.avro.Schema) this.avroData.fromConnectData( srcRecord.valueSchema(), valueBytes); - this.avroData.fromConnectData(srcRecord.valueSchema(), valueBytes); + keySchema = new KafkaSchema(); + keySchema.setAvroSchema(true, this.avroData, keyAvroSchema, keyConverter); + valueSchema = new KafkaSchema(); + valueSchema.setAvroSchema(false, this.avroData, valueAvroSchema, valueConverter); this.topicName = Optional.of(srcRecord.topic()); this.eventTime = Optional.ofNullable(srcRecord.timestamp()); @@ -225,9 +237,7 @@ private class KafkaSourceRecord implements Record> { @Override public Schema> getSchema() { - return new KeyValueSchema<>( - keyAvroSchema, valueAvroSchema - ); + return KeyValueSchema.of(keySchema, valueSchema); } @Override diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchema.java new file mode 100644 index 0000000000000..fc308244a89dc --- /dev/null +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchema.java @@ -0,0 +1,121 @@ +package org.apache.pulsar.io.kafka.connect.schema; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroConverter; +import io.confluent.connect.avro.AvroData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.EncoderFactory; +import org.apache.kafka.connect.json.JsonConverter; +import org.apache.kafka.connect.json.JsonDeserializer; +import org.apache.kafka.connect.storage.Converter; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SchemaSerializationException; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; + +import java.io.ByteArrayOutputStream; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Collections; + +import static java.nio.charset.StandardCharsets.UTF_8; + +public class KafkaSchema implements Schema { + + private AvroData avroData = null; + private final JsonDeserializer jsonDeserializer = new JsonDeserializer(); + private Converter valueConverter = null; + private SchemaInfo schemaInfo = null; + private org.apache.avro.Schema avroSchema = null; + private final Method convertToConnectMethod; + + public KafkaSchema() { + try { + this.convertToConnectMethod = JsonConverter.class.getDeclaredMethod( + "convertToConnect", + org.apache.kafka.connect.data.Schema.class, + JsonNode.class + ); + this.convertToConnectMethod.setAccessible(true); + } catch (NoSuchMethodException e) { + throw new RuntimeException("Failed to locate `convertToConnect` method for JsonConverter", e); + } + } + + public void setAvroSchema(boolean isKey, + AvroData avroData, + org.apache.avro.Schema schema, + Converter converter) { + this.valueConverter = converter; + this.avroData = avroData; + this.avroSchema = schema; + this.schemaInfo = SchemaInfo.builder() + .name(converter instanceof JsonConverter ? "KafkaJson" : "KafkaAvro") + .type(converter instanceof JsonConverter ? SchemaType.JSON : SchemaType.AVRO) + .properties(Collections.emptyMap()) + .schema(schema.toString().getBytes(UTF_8)) + .build(); + if (converter instanceof AvroConverter) { + initializeAvroWriter(schema); + } + } + + @Override + public byte[] encode(byte[] data) { + if (null == valueConverter || valueConverter instanceof JsonConverter) { + return data; + } + + org.apache.kafka.connect.data.Schema connectSchema = avroData.toConnectSchema(avroSchema); + JsonNode jsonNode = jsonDeserializer.deserialize("", data); + + Object connectValue; + try { + connectValue = convertToConnectMethod.invoke( + null, + connectSchema, + jsonNode + ); + } catch (IllegalAccessException e) { + throw new SchemaSerializationException("Can not call JsonConverter#convertToConnect"); + } catch (InvocationTargetException e) { + throw new SchemaSerializationException(e.getCause()); + } + + Object avroValue = avroData.fromConnectData( + connectSchema, + connectValue + ); + + return writeAvroRecord((GenericRecord) avroValue); + } + + private GenericDatumWriter writer; + private BinaryEncoder encoder; + private ByteArrayOutputStream byteArrayOutputStream; + + synchronized void initializeAvroWriter(org.apache.avro.Schema schema) { + this.writer = new GenericDatumWriter<>(schema); + this.byteArrayOutputStream = new ByteArrayOutputStream(); + this.encoder = EncoderFactory.get().binaryEncoder(this.byteArrayOutputStream, this.encoder); + } + + synchronized byte[] writeAvroRecord(GenericRecord record) { + try { + this.writer.write(record, this.encoder); + this.encoder.flush(); + return this.byteArrayOutputStream.toByteArray(); + } catch (Exception e) { + throw new SchemaSerializationException(e); + } finally { + this.byteArrayOutputStream.reset(); + } + } + + @Override + public SchemaInfo getSchemaInfo() { + return schemaInfo; + } +} From 9533d3bd84e62fadbaf9f89a9749e0c42c6570fe Mon Sep 17 00:00:00 2001 From: guangning Date: Wed, 8 Jan 2020 17:31:21 +0800 Subject: [PATCH 03/36] Add cache for key and value schema --- .../io/kafka/connect/KafkaConnectSource.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index f11429d6df3f6..20c785fe3b236 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -35,9 +35,14 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import io.confluent.connect.avro.AvroData; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.concurrent.ConcurrentException; import org.apache.kafka.connect.runtime.TaskConfig; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; @@ -79,6 +84,9 @@ public class KafkaConnectSource implements Source> { // number of outstandingRecords that have been polled but not been acked private AtomicInteger outstandingRecords = new AtomicInteger(0); + private final Cache readerCache = CacheBuilder.newBuilder().maximumSize(100000) + .expireAfterAccess(30, TimeUnit.MINUTES).build(); + @Override public void open(Map config, SourceContext sourceContext) throws Exception { Map stringConfig = new HashMap<>(); @@ -216,14 +224,21 @@ private class KafkaSourceRecord implements Record> { this.key = keyBytes != null ? Optional.of(Base64.getEncoder().encodeToString(keyBytes)) : Optional.empty(); this.value = new KeyValue(keyBytes, valueBytes); + if (readerCache.getIfPresent("keySchema") == null || readerCache.getIfPresent("valueSchema") == null) { + keySchema = new KafkaSchema(); + valueSchema = new KafkaSchema(); + } else { + keySchema = readerCache.getIfPresent("keySchema"); + valueSchema = readerCache.getIfPresent("valueSchema"); + } keyAvroSchema = (org.apache.avro.Schema) this.avroData.fromConnectData( srcRecord.keySchema(), keyBytes); valueAvroSchema = (org.apache.avro.Schema) this.avroData.fromConnectData( srcRecord.valueSchema(), valueBytes); - keySchema = new KafkaSchema(); keySchema.setAvroSchema(true, this.avroData, keyAvroSchema, keyConverter); - valueSchema = new KafkaSchema(); valueSchema.setAvroSchema(false, this.avroData, valueAvroSchema, valueConverter); + readerCache.getIfPresent("valueSchema"); + this.topicName = Optional.of(srcRecord.topic()); this.eventTime = Optional.ofNullable(srcRecord.timestamp()); From b84eceabc5c22da3477b377b16a174e39ebf40ad Mon Sep 17 00:00:00 2001 From: guangning Date: Sun, 12 Jan 2020 19:44:49 +0800 Subject: [PATCH 04/36] Support debezium avro schema --- kafka-connect-avro-converter-shaded/pom.xml | 128 +++++++++++++++ pom.xml | 5 + .../pulsar/functions/sink/PulsarSink.java | 21 ++- pulsar-io/kafka-connect-adaptor/pom.xml | 29 +++- .../io/kafka/connect/KafkaConnectSource.java | 35 ++-- .../io/kafka/connect/schema/KafkaSchema.java | 36 +++-- .../kafka/connect/schema/KafkaSchemaTest.java | 149 ++++++++++++++++++ 7 files changed, 366 insertions(+), 37 deletions(-) create mode 100644 kafka-connect-avro-converter-shaded/pom.xml create mode 100644 pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaTest.java diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml new file mode 100644 index 0000000000000..100636771286f --- /dev/null +++ b/kafka-connect-avro-converter-shaded/pom.xml @@ -0,0 +1,128 @@ + + + + 4.0.0 + + pulsar + org.apache.pulsar + 2.6.0-SNAPSHOT + .. + + + kafka-connect-avro-converter-shaded + Apache Pulsar :: Kafka Connect Avro Converter shaded + + + + + io.confluent + kafka-connect-avro-converter + ${confluent.version} + + + org.codehaus.jackson + jackson-core-asl + ${kafka-avro-convert-jackson.version} + + + org.codehaus.jackson + jackson-mapper-asl + ${kafka-avro-convert-jackson.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + + + true + true + + + + io.confluent:* + io.confluent:kafka-avro-serializer + io.confluent:kafka-schema-registry-client + io.confluent:common-config + io.confluent:common-utils + org.apache.avro:* + + org.codehaus.jackson:jackson-core-asl + org.codehaus.jackson:jackson-mapper-asl + com.thoughtworks.paranamer:paranamer + org.xerial.snappy:snappy-java + org.apache.commons:commons-compress + org.tukaani:xz + + + + + io.confluent + org.apache.pulsar.kafka.shade.io.confluent + + + org.apache.avro + org.apache.pulsar.kafka.shade.avro + + + org.codehaus.jackson + org.apache.pulsar.kafka.shade.org.codehaus.jackson + + + com.thoughtworks.paranamer + org.apache.pulsar.kafka.shade.com.thoughtworks.paranamer + + + org.xerial.snappy + org.apache.pulsar.kafka.shade.org.xerial.snappy + + + org.apache.commons + org.apache.pulsar.kafka.shade.org.apache.commons + + + org.tukaani + org.apache.pulsar.kafka.shade.org.tukaani + + + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index eb3367f59706b..ddeda8fc081da 100644 --- a/pom.xml +++ b/pom.xml @@ -117,6 +117,9 @@ flexible messaging model and an intuitive client API. pulsar-io + + kafka-connect-avro-converter-shaded + examples @@ -208,6 +211,8 @@ flexible messaging model and an intuitive client API. 25.1-jre 1.0 0.12.0 + 5.3.2 + 1.9.13 3.6.0 diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java index 43100e46a3759..40391bf775fd8 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java @@ -177,7 +177,17 @@ public PulsarSinkAtMostOnceProcessor(Schema schema) { @Override public TypedMessageBuilder newMessage(Record record) { - return getProducer(record.getDestinationTopic().orElse(pulsarSinkConfig.getTopic())).newMessage(); + if (record.getSchema() != null) { + return getProducer(record + .getDestinationTopic() + .orElse(pulsarSinkConfig.getTopic())) + .newMessage(record.getSchema()); + } else { + return getProducer(record + .getDestinationTopic() + .orElse(pulsarSinkConfig.getTopic())) + .newMessage(); + } } @Override @@ -215,11 +225,16 @@ public TypedMessageBuilder newMessage(Record record) { throw new RuntimeException("PartitionId needs to be specified for every record while in Effectively-once mode"); } - return getProducer( + Producer producer = getProducer( String.format("%s-%s",record.getDestinationTopic().orElse(pulsarSinkConfig.getTopic()), record.getPartitionId().get()), record.getPartitionId().get(), record.getDestinationTopic().orElse(pulsarSinkConfig.getTopic()) - ).newMessage(); + ); + if (record.getSchema() != null) { + return producer.newMessage(record.getSchema()); + } else { + return producer.newMessage(); + } } @Override diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 250c73c8028a3..8463c76d48153 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -68,6 +68,22 @@ ${project.version} + + org.apache.pulsar + kafka-connect-avro-converter-shaded + ${project.version} + + + io.confluent + * + + + org.apache.avro + * + + + + ${project.groupId} pulsar-broker @@ -95,13 +111,12 @@ ${project.version} test test-jar - - - - - io.confluent - kafka-connect-avro-converter - ${confluent.version} + + + org.apache.avro + * + + diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index 20c785fe3b236..e4cda6135d3b8 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -37,12 +37,9 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import io.confluent.connect.avro.AvroData; +import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroData; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.concurrent.ConcurrentException; import org.apache.kafka.connect.runtime.TaskConfig; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; @@ -203,9 +200,9 @@ private class KafkaSourceRecord implements Record> { private final AvroData avroData; - private final org.apache.avro.Schema keyAvroSchema; + private org.apache.pulsar.kafka.shade.avro.Schema keyAvroSchema; - private final org.apache.avro.Schema valueAvroSchema; + private org.apache.pulsar.kafka.shade.avro.Schema valueAvroSchema; private final KafkaSchema keySchema; @@ -224,23 +221,25 @@ private class KafkaSourceRecord implements Record> { this.key = keyBytes != null ? Optional.of(Base64.getEncoder().encodeToString(keyBytes)) : Optional.empty(); this.value = new KeyValue(keyBytes, valueBytes); - if (readerCache.getIfPresent("keySchema") == null || readerCache.getIfPresent("valueSchema") == null) { + this.topicName = Optional.of(srcRecord.topic()); + String keyName = this.topicName.get() + "-key"; + String valueName = this.topicName.get() + "-value"; + + if (readerCache.getIfPresent(keyName) == null + || readerCache.getIfPresent(valueName) == null) { keySchema = new KafkaSchema(); valueSchema = new KafkaSchema(); + readerCache.put(keyName, keySchema); + readerCache.put(valueName, valueSchema); + keyAvroSchema = this.avroData.fromConnectSchema(srcRecord.keySchema()); + valueAvroSchema = this.avroData.fromConnectSchema(srcRecord.valueSchema()); + keySchema.setAvroSchema(true, this.avroData, keyAvroSchema, keyConverter); + valueSchema.setAvroSchema(false, this.avroData, valueAvroSchema, valueConverter); } else { - keySchema = readerCache.getIfPresent("keySchema"); - valueSchema = readerCache.getIfPresent("valueSchema"); + keySchema = readerCache.getIfPresent(keyName); + valueSchema = readerCache.getIfPresent(valueName); } - keyAvroSchema = (org.apache.avro.Schema) this.avroData.fromConnectData( - srcRecord.keySchema(), keyBytes); - valueAvroSchema = (org.apache.avro.Schema) this.avroData.fromConnectData( - srcRecord.valueSchema(), valueBytes); - keySchema.setAvroSchema(true, this.avroData, keyAvroSchema, keyConverter); - valueSchema.setAvroSchema(false, this.avroData, valueAvroSchema, valueConverter); - readerCache.getIfPresent("valueSchema"); - - this.topicName = Optional.of(srcRecord.topic()); this.eventTime = Optional.ofNullable(srcRecord.timestamp()); this.partitionId = Optional.of(srcRecord.sourcePartition() .entrySet() diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchema.java index fc308244a89dc..b5589dd2ee3e2 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchema.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchema.java @@ -1,12 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.pulsar.io.kafka.connect.schema; import com.fasterxml.jackson.databind.JsonNode; -import io.confluent.connect.avro.AvroConverter; -import io.confluent.connect.avro.AvroData; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.EncoderFactory; +import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter; +import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroData; +import org.apache.pulsar.kafka.shade.avro.generic.GenericDatumWriter; +import org.apache.pulsar.kafka.shade.avro.generic.GenericRecord; +import org.apache.pulsar.kafka.shade.avro.io.BinaryEncoder; +import org.apache.pulsar.kafka.shade.avro.io.EncoderFactory; import org.apache.kafka.connect.json.JsonConverter; import org.apache.kafka.connect.json.JsonDeserializer; import org.apache.kafka.connect.storage.Converter; @@ -28,7 +46,7 @@ public class KafkaSchema implements Schema { private final JsonDeserializer jsonDeserializer = new JsonDeserializer(); private Converter valueConverter = null; private SchemaInfo schemaInfo = null; - private org.apache.avro.Schema avroSchema = null; + private org.apache.pulsar.kafka.shade.avro.Schema avroSchema = null; private final Method convertToConnectMethod; public KafkaSchema() { @@ -46,7 +64,7 @@ public KafkaSchema() { public void setAvroSchema(boolean isKey, AvroData avroData, - org.apache.avro.Schema schema, + org.apache.pulsar.kafka.shade.avro.Schema schema, Converter converter) { this.valueConverter = converter; this.avroData = avroData; @@ -96,7 +114,7 @@ public byte[] encode(byte[] data) { private BinaryEncoder encoder; private ByteArrayOutputStream byteArrayOutputStream; - synchronized void initializeAvroWriter(org.apache.avro.Schema schema) { + synchronized void initializeAvroWriter(org.apache.pulsar.kafka.shade.avro.Schema schema) { this.writer = new GenericDatumWriter<>(schema); this.byteArrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(this.byteArrayOutputStream, this.encoder); diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaTest.java new file mode 100644 index 0000000000000..b221f3b08070a --- /dev/null +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaTest.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.kafka.connect.schema; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.connect.json.JsonConverter; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; +import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter; +import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroData; +import org.apache.pulsar.kafka.shade.io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient; +import org.apache.pulsar.kafka.shade.io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +@Slf4j +public class KafkaSchemaTest { + + private final KafkaSchema kafkaSchema; + private final AvroData avroData; + private final Schema userJsonSchema = Schema.JSON( + SchemaDefinition.builder() + .withPojo(User.class) + .withAlwaysAllowNull(false) + .withSupportSchemaVersioning(true) + .build()); + + public KafkaSchemaTest() { + kafkaSchema = new KafkaSchema(); + avroData = new AvroData(1000); + } + + @Before + public void setup() { + } + + /** + * User class. + */ + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class User { + String name; + int age; + } + + @Test + public void testJsonConverter() throws Exception { + String AVRO_USER_SCHEMA_DEF = "{" + + "\"type\":\"record\"," + + "\"name\":\"User\"," + + "\"namespace\":\"org.apache.pulsar.io.kafka.connect.schema.KafkaSchemaTest\"," + + "\"fields\":[" + + "{\"name\":\"name\",\"type\":\"string\",\"default\":null}," + + "{\"name\":\"age\",\"type\":\"int\"}" + + "]}"; + org.apache.pulsar.kafka.shade.avro.Schema AVRO_USER_SCHEMA = new org.apache.pulsar.kafka.shade.avro.Schema + .Parser().parse(AVRO_USER_SCHEMA_DEF); + kafkaSchema.setAvroSchema( + false, + avroData, + AVRO_USER_SCHEMA, + new JsonConverter() + ); + + User user = new User("user-1", 100); + byte[] data = userJsonSchema.encode(user); + byte[] encodedData = kafkaSchema.encode(data); + assertSame(data, encodedData); + } + + @Test + public void testAvroConverter() throws Exception { + String AVRO_USER_SCHEMA_DEF = "{" + + "\"type\":\"record\"," + + "\"name\":\"User\"," + + "\"namespace\":\"org.apache.pulsar.io.kafka.connect.schema.KafkaSchemaTest\"," + + "\"fields\":[" + + "{\"name\":\"name\",\"type\":\"string\"}," + + "{\"name\":\"age\",\"type\":\"int\"}" + + "]}"; + org.apache.pulsar.kafka.shade.avro.Schema AVRO_USER_SCHEMA = new org.apache.pulsar.kafka.shade.avro.Schema + .Parser().parse(AVRO_USER_SCHEMA_DEF); + Map config = new HashMap<>(); + AvroConverter converter = new AvroConverter(new MockSchemaRegistryClient()); + config.put( + AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, + "mock"); + converter.configure(config, false); + kafkaSchema.setAvroSchema( + false, + avroData, + AVRO_USER_SCHEMA, + converter + ); + + log.info("Initialized with avro schema {}", AVRO_USER_SCHEMA); + + User user = new User("user-1", 100); + byte[] jsonData = userJsonSchema.encode(user); + + byte[] avroData = kafkaSchema.encode(jsonData); + + + Schema userAvroSchema = Schema.generic( + SchemaInfo.builder() + .name("") + .properties(Collections.emptyMap()) + .type(SchemaType.AVRO) + .schema(userJsonSchema.getSchemaInfo().getSchema()) + .build() + ); + GenericRecord userRecord = userAvroSchema.decode(avroData); + assertEquals(user.getName(), userRecord.getField("name")); + assertEquals(user.getAge(), userRecord.getField("age")); + + } +} + From f6165625664e87742ec0e695e3322732ca681d49 Mon Sep 17 00:00:00 2001 From: guangning Date: Sun, 12 Jan 2020 20:11:09 +0800 Subject: [PATCH 05/36] Delete no used dependency --- pulsar-io/kafka-connect-adaptor/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 8463c76d48153..5609cd129bf91 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -111,12 +111,6 @@ ${project.version} test test-jar - - - org.apache.avro - * - - From e9a8bc27e8d9e2fa6b49c7d43555e18a6d4e65c8 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Fri, 17 Apr 2020 02:13:35 +0800 Subject: [PATCH 06/36] debezium connect source support `org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter`, add integration test --- .../client/impl/schema/StructSchema.java | 6 + .../schema/generic/GenericAvroReader.java | 20 ++- .../schema/generic/GenericAvroSchema.java | 3 + .../schema/generic/GenericAvroReaderTest.java | 24 ++- .../pulsar/functions/instance/SinkRecord.java | 6 + .../pulsar/functions/sink/PulsarSink.java | 7 +- .../io/kafka/connect/KafkaConnectSource.java | 87 +++++----- .../io/kafka/connect/schema/KafkaSchema.java | 139 ---------------- .../schema/KafkaSchemaWrappedSchema.java | 66 ++++++++ .../kafka/connect/schema/KafkaSchemaTest.java | 149 ------------------ .../functions/PulsarFunctionsTest.java | 62 +++++--- .../io/DebeziumMySqlSourceTester.java | 4 +- .../tests/integration/io/SourceTester.java | 54 ++++++- 13 files changed, 272 insertions(+), 355 deletions(-) delete mode 100644 pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchema.java create mode 100644 pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java delete mode 100644 pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaTest.java diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java index f9faa0b1156bc..bbbda12e1ce9d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java @@ -41,6 +41,7 @@ import org.apache.pulsar.client.api.schema.SchemaInfoProvider; import org.apache.pulsar.client.api.schema.SchemaReader; import org.apache.pulsar.client.api.schema.SchemaWriter; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroReader; import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -78,6 +79,11 @@ public SchemaReader load(BytesSchemaVersion schemaVersion) { protected StructSchema(SchemaInfo schemaInfo) { this.schema = parseAvroSchema(new String(schemaInfo.getSchema(), UTF_8)); this.schemaInfo = schemaInfo; + + if (schemaInfo.getProperties().containsKey(GenericAvroReader.OFFSET_PROP)) { + this.schema.addProp(GenericAvroReader.OFFSET_PROP, + schemaInfo.getProperties().get(GenericAvroReader.OFFSET_PROP)); + } } public org.apache.avro.Schema getAvroSchema() { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReader.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReader.java index 2ac0c2f2695bf..9e90080aea607 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReader.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReader.java @@ -47,6 +47,10 @@ public class GenericAvroReader implements SchemaReader { private final List fields; private final Schema schema; private final byte[] schemaVersion; + + private int offset; + public final static String OFFSET_PROP = "AVRO_READ_OFFSET"; + public GenericAvroReader(Schema schema) { this(null, schema, null); } @@ -65,12 +69,22 @@ public GenericAvroReader(Schema writerSchema, Schema readerSchema, byte[] schema } this.byteArrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(this.byteArrayOutputStream, encoder); + + if (schema.getObjectProp(GenericAvroReader.OFFSET_PROP) != null) { + this.offset = Integer.parseInt(schema.getObjectProp(GenericAvroReader.OFFSET_PROP).toString()); + } else { + this.offset = 0; + } + } @Override public GenericAvroRecord read(byte[] bytes, int offset, int length) { try { - Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, offset, length, null); + if (offset == 0 && this.offset > 0) { + offset = this.offset; + } + Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, offset, length - offset, null); org.apache.avro.generic.GenericRecord avroRecord = (org.apache.avro.generic.GenericRecord)reader.read( null, @@ -101,5 +115,9 @@ public GenericRecord read(InputStream inputStream) { } } + public int getOffset() { + return offset; + } + private static final Logger log = LoggerFactory.getLogger(GenericAvroReader.class); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java index 98e646ea2ba50..a1f93b94474a3 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java @@ -73,6 +73,9 @@ protected SchemaReader loadReader(BytesSchemaVersion schemaVersio schemaInfo); Schema writerSchema = parseAvroSchema(schemaInfo.getSchemaDefinition()); Schema readerSchema = useProvidedSchemaAsReaderSchema ? schema : writerSchema; + readerSchema.addProp(GenericAvroReader.OFFSET_PROP, + schemaInfo.getProperties().getOrDefault(GenericAvroReader.OFFSET_PROP, "0")); + return new GenericAvroReader( writerSchema, readerSchema, diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReaderTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReaderTest.java index d77d0f2995414..52e387fd88f5f 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReaderTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReaderTest.java @@ -20,6 +20,8 @@ import static org.testng.Assert.assertEquals; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.SchemaDefinition; @@ -38,11 +40,12 @@ public class GenericAvroReaderTest { private AvroSchema fooSchemaNotNull; private AvroSchema fooSchema; private AvroSchema fooV2Schema; - + private AvroSchema fooOffsetSchema; @BeforeMethod public void setup() { fooSchema = AvroSchema.of(Foo.class); + fooV2Schema = AvroSchema.of(FooV2.class); fooSchemaNotNull = AvroSchema.of(SchemaDefinition .builder() @@ -50,6 +53,9 @@ public void setup() { .withPojo(Foo.class) .build()); + fooOffsetSchema = AvroSchema.of(Foo.class); + fooOffsetSchema.getAvroSchema().addProp(GenericAvroReader.OFFSET_PROP, 5); + foo = new Foo(); foo.setField1("foo1"); foo.setField2("bar1"); @@ -83,4 +89,20 @@ public void testGenericAvroReaderByReaderSchema() { assertEquals(genericRecordByReaderSchema.getField("field3"), 10); } + @Test + public void testOffsetSchema() { + byte[] fooBytes = fooOffsetSchema.encode(foo); + ByteBuf byteBuf = Unpooled.buffer(); + byteBuf.writeByte(0); + byteBuf.writeInt(10); + byteBuf.writeBytes(fooBytes); + + GenericAvroReader reader = new GenericAvroReader(fooOffsetSchema.getAvroSchema()); + assertEquals(reader.getOffset(), 5); + GenericRecord record = reader.read(byteBuf.array()); + assertEquals(record.getField("field1"), "foo1"); + assertEquals(record.getField("field2"), "bar1"); + assertEquals(record.getField("fieldUnableNull"), "notNull"); + } + } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java index 3e1d5c01a2d88..d45345d340e02 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java @@ -24,6 +24,7 @@ import lombok.AllArgsConstructor; import lombok.Data; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.functions.api.Record; @Data @@ -81,4 +82,9 @@ public void fail() { public Optional getDestinationTopic() { return sourceRecord.getDestinationTopic(); } + + @Override + public Schema getSchema() { + return sourceRecord.getSchema(); + } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java index 40391bf775fd8..d2aaad0d5efb8 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java @@ -31,7 +31,9 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TypedMessageBuilder; +import org.apache.pulsar.client.impl.schema.KeyValueSchema; import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.instance.FunctionResultRouter; import org.apache.pulsar.functions.instance.SinkRecord; @@ -178,6 +180,7 @@ public PulsarSinkAtMostOnceProcessor(Schema schema) { @Override public TypedMessageBuilder newMessage(Record record) { if (record.getSchema() != null) { + schema = record.getSchema(); return getProducer(record .getDestinationTopic() .orElse(pulsarSinkConfig.getTopic())) @@ -231,6 +234,7 @@ public TypedMessageBuilder newMessage(Record record) { record.getDestinationTopic().orElse(pulsarSinkConfig.getTopic()) ); if (record.getSchema() != null) { + schema = record.getSchema(); return producer.newMessage(record.getSchema()); } else { return producer.newMessage(); @@ -289,7 +293,8 @@ public void open(Map config, SinkContext sinkContext) throws Exc @Override public void write(Record record) { TypedMessageBuilder msg = pulsarSinkProcessor.newMessage(record); - if (record.getKey().isPresent()) { + if (record.getKey().isPresent() && !(record.getSchema() instanceof KeyValueSchema && + ((KeyValueSchema) record.getSchema()).getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED)) { msg.key(record.getKey().get()); } diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index e4cda6135d3b8..68110f63700ea 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -37,6 +37,8 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter; import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroData; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -55,7 +57,9 @@ import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.core.Source; import org.apache.pulsar.io.core.SourceContext; -import org.apache.pulsar.io.kafka.connect.schema.KafkaSchema; +import org.apache.pulsar.io.kafka.connect.schema.KafkaSchemaWrappedSchema; +import org.apache.pulsar.kafka.shade.io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient; +import org.apache.pulsar.kafka.shade.io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; /** * A pulsar source that runs @@ -81,8 +85,9 @@ public class KafkaConnectSource implements Source> { // number of outstandingRecords that have been polled but not been acked private AtomicInteger outstandingRecords = new AtomicInteger(0); - private final Cache readerCache = CacheBuilder.newBuilder().maximumSize(100000) - .expireAfterAccess(30, TimeUnit.MINUTES).build(); + private final Cache readerCache = + CacheBuilder.newBuilder().maximumSize(10000) + .expireAfterAccess(30, TimeUnit.MINUTES).build(); @Override public void open(Map config, SourceContext sourceContext) throws Exception { @@ -111,6 +116,14 @@ public void open(Map config, SourceContext sourceContext) throws .getDeclaredConstructor() .newInstance(); + if (keyConverter instanceof AvroConverter) { + keyConverter = new AvroConverter(new MockSchemaRegistryClient()); + config.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "mock"); + } + if (valueConverter instanceof AvroConverter) { + valueConverter = new AvroConverter(new MockSchemaRegistryClient()); + config.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "mock"); + } keyConverter.configure(config, true); valueConverter.configure(config, false); @@ -155,7 +168,6 @@ public synchronized Record> read() throws Exception { Record> processRecord = processSourceRecord(currentBatch.next()); if (processRecord.getValue().getValue() == null) { outstandingRecords.decrementAndGet(); - continue; } else { return processRecord; } @@ -198,46 +210,37 @@ private class KafkaSourceRecord implements Record> { @Getter Optional destinationTopic; - private final AvroData avroData; + KafkaSchemaWrappedSchema keySchema; - private org.apache.pulsar.kafka.shade.avro.Schema keyAvroSchema; + KafkaSchemaWrappedSchema valueSchema; - private org.apache.pulsar.kafka.shade.avro.Schema valueAvroSchema; + KafkaSourceRecord(SourceRecord srcRecord) { + AvroData avroData = new AvroData(1000); + byte[] keyBytes = keyConverter.fromConnectData( + srcRecord.topic(), srcRecord.keySchema(), srcRecord.key()); + this.key = keyBytes != null ? Optional.of( + Base64.getEncoder().encodeToString(keyBytes)) : Optional.empty(); - private final KafkaSchema keySchema; + byte[] valueBytes = valueConverter.fromConnectData( + srcRecord.topic(), srcRecord.valueSchema(), srcRecord.value()); - private final KafkaSchema valueSchema; + this.value = new KeyValue<>(keyBytes, valueBytes); - private byte[] keyBytes; + this.topicName = Optional.of(srcRecord.topic()); - private byte[] valueBytes; + keySchema = readerCache.getIfPresent(srcRecord.keySchema()); + valueSchema = readerCache.getIfPresent(srcRecord.valueSchema()); - KafkaSourceRecord(SourceRecord srcRecord) { - keyBytes = keyConverter.fromConnectData( - srcRecord.topic(), srcRecord.keySchema(), srcRecord.key()); - valueBytes = valueConverter.fromConnectData( - srcRecord.topic(), srcRecord.valueSchema(), srcRecord.value()); - this.avroData = new AvroData(1000); - this.key = keyBytes != null ? Optional.of(Base64.getEncoder().encodeToString(keyBytes)) : Optional.empty(); - this.value = new KeyValue(keyBytes, valueBytes); + if (keySchema == null) { + keySchema = new KafkaSchemaWrappedSchema( + avroData.fromConnectSchema(srcRecord.keySchema()), keyConverter); + readerCache.put(srcRecord.keySchema(), keySchema); + } - this.topicName = Optional.of(srcRecord.topic()); - String keyName = this.topicName.get() + "-key"; - String valueName = this.topicName.get() + "-value"; - - if (readerCache.getIfPresent(keyName) == null - || readerCache.getIfPresent(valueName) == null) { - keySchema = new KafkaSchema(); - valueSchema = new KafkaSchema(); - readerCache.put(keyName, keySchema); - readerCache.put(valueName, valueSchema); - keyAvroSchema = this.avroData.fromConnectSchema(srcRecord.keySchema()); - valueAvroSchema = this.avroData.fromConnectSchema(srcRecord.valueSchema()); - keySchema.setAvroSchema(true, this.avroData, keyAvroSchema, keyConverter); - valueSchema.setAvroSchema(false, this.avroData, valueAvroSchema, valueConverter); - } else { - keySchema = readerCache.getIfPresent(keyName); - valueSchema = readerCache.getIfPresent(valueName); + if (valueSchema == null) { + valueSchema = new KafkaSchemaWrappedSchema( + avroData.fromConnectSchema(srcRecord.valueSchema()), valueConverter); + readerCache.put(srcRecord.valueSchema(), valueSchema); } this.eventTime = Optional.ofNullable(srcRecord.timestamp()); @@ -250,8 +253,16 @@ private class KafkaSourceRecord implements Record> { } @Override - public Schema> getSchema() { - return KeyValueSchema.of(keySchema, valueSchema); + public Schema getSchema() { + // When use `org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter` + // as the key.converter and value.converter, make the `KeyValueSchema` encodingType + // use the `KeyValueEncodingType.SEPARATED`, then the pulsar client could get the original + // byte array which are converted by the AvroConverter, or consume the GenericRecord object. + if (keyConverter instanceof AvroConverter && valueConverter instanceof AvroConverter) { + return KeyValueSchema.of(keySchema, valueSchema, KeyValueEncodingType.SEPARATED); + } else { + return KeyValueSchema.of(Schema.BYTES, Schema.BYTES); + } } @Override diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchema.java deleted file mode 100644 index b5589dd2ee3e2..0000000000000 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchema.java +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.io.kafka.connect.schema; - -import com.fasterxml.jackson.databind.JsonNode; -import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter; -import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroData; -import org.apache.pulsar.kafka.shade.avro.generic.GenericDatumWriter; -import org.apache.pulsar.kafka.shade.avro.generic.GenericRecord; -import org.apache.pulsar.kafka.shade.avro.io.BinaryEncoder; -import org.apache.pulsar.kafka.shade.avro.io.EncoderFactory; -import org.apache.kafka.connect.json.JsonConverter; -import org.apache.kafka.connect.json.JsonDeserializer; -import org.apache.kafka.connect.storage.Converter; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.SchemaSerializationException; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; - -import java.io.ByteArrayOutputStream; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Collections; - -import static java.nio.charset.StandardCharsets.UTF_8; - -public class KafkaSchema implements Schema { - - private AvroData avroData = null; - private final JsonDeserializer jsonDeserializer = new JsonDeserializer(); - private Converter valueConverter = null; - private SchemaInfo schemaInfo = null; - private org.apache.pulsar.kafka.shade.avro.Schema avroSchema = null; - private final Method convertToConnectMethod; - - public KafkaSchema() { - try { - this.convertToConnectMethod = JsonConverter.class.getDeclaredMethod( - "convertToConnect", - org.apache.kafka.connect.data.Schema.class, - JsonNode.class - ); - this.convertToConnectMethod.setAccessible(true); - } catch (NoSuchMethodException e) { - throw new RuntimeException("Failed to locate `convertToConnect` method for JsonConverter", e); - } - } - - public void setAvroSchema(boolean isKey, - AvroData avroData, - org.apache.pulsar.kafka.shade.avro.Schema schema, - Converter converter) { - this.valueConverter = converter; - this.avroData = avroData; - this.avroSchema = schema; - this.schemaInfo = SchemaInfo.builder() - .name(converter instanceof JsonConverter ? "KafkaJson" : "KafkaAvro") - .type(converter instanceof JsonConverter ? SchemaType.JSON : SchemaType.AVRO) - .properties(Collections.emptyMap()) - .schema(schema.toString().getBytes(UTF_8)) - .build(); - if (converter instanceof AvroConverter) { - initializeAvroWriter(schema); - } - } - - @Override - public byte[] encode(byte[] data) { - if (null == valueConverter || valueConverter instanceof JsonConverter) { - return data; - } - - org.apache.kafka.connect.data.Schema connectSchema = avroData.toConnectSchema(avroSchema); - JsonNode jsonNode = jsonDeserializer.deserialize("", data); - - Object connectValue; - try { - connectValue = convertToConnectMethod.invoke( - null, - connectSchema, - jsonNode - ); - } catch (IllegalAccessException e) { - throw new SchemaSerializationException("Can not call JsonConverter#convertToConnect"); - } catch (InvocationTargetException e) { - throw new SchemaSerializationException(e.getCause()); - } - - Object avroValue = avroData.fromConnectData( - connectSchema, - connectValue - ); - - return writeAvroRecord((GenericRecord) avroValue); - } - - private GenericDatumWriter writer; - private BinaryEncoder encoder; - private ByteArrayOutputStream byteArrayOutputStream; - - synchronized void initializeAvroWriter(org.apache.pulsar.kafka.shade.avro.Schema schema) { - this.writer = new GenericDatumWriter<>(schema); - this.byteArrayOutputStream = new ByteArrayOutputStream(); - this.encoder = EncoderFactory.get().binaryEncoder(this.byteArrayOutputStream, this.encoder); - } - - synchronized byte[] writeAvroRecord(GenericRecord record) { - try { - this.writer.write(record, this.encoder); - this.encoder.flush(); - return this.byteArrayOutputStream.toByteArray(); - } catch (Exception e) { - throw new SchemaSerializationException(e); - } finally { - this.byteArrayOutputStream.reset(); - } - } - - @Override - public SchemaInfo getSchemaInfo() { - return schemaInfo; - } -} diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java new file mode 100644 index 0000000000000..ed2454fed679a --- /dev/null +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.kafka.connect.schema; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.util.HashMap; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.connect.json.JsonConverter; +import org.apache.kafka.connect.storage.Converter; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroReader; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; + +/** + * Wrapped schema for kafka connect schema, + */ +@Slf4j +public class KafkaSchemaWrappedSchema implements Schema { + + private SchemaInfo schemaInfo = null; + + public KafkaSchemaWrappedSchema(org.apache.pulsar.kafka.shade.avro.Schema schema, + Converter converter) { + if (converter instanceof JsonConverter) { + return; + } + Map props = new HashMap<>(); + props.put(GenericAvroReader.OFFSET_PROP, "5"); + + this.schemaInfo = SchemaInfo.builder() + .name("KafkaAvro") + .type(SchemaType.AVRO) + .schema(schema.toString().getBytes(UTF_8)) + .properties(props) + .build(); + } + + @Override + public byte[] encode(byte[] data) { + return data; + } + + @Override + public SchemaInfo getSchemaInfo() { + return schemaInfo; + } +} diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaTest.java deleted file mode 100644 index b221f3b08070a..0000000000000 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaTest.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.io.kafka.connect.schema; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.connect.json.JsonConverter; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.schema.GenericRecord; -import org.apache.pulsar.client.api.schema.SchemaDefinition; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; -import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter; -import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroData; -import org.apache.pulsar.kafka.shade.io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient; -import org.apache.pulsar.kafka.shade.io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; -import org.junit.Before; -import org.junit.Test; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; - -@Slf4j -public class KafkaSchemaTest { - - private final KafkaSchema kafkaSchema; - private final AvroData avroData; - private final Schema userJsonSchema = Schema.JSON( - SchemaDefinition.builder() - .withPojo(User.class) - .withAlwaysAllowNull(false) - .withSupportSchemaVersioning(true) - .build()); - - public KafkaSchemaTest() { - kafkaSchema = new KafkaSchema(); - avroData = new AvroData(1000); - } - - @Before - public void setup() { - } - - /** - * User class. - */ - @Data - @AllArgsConstructor - @NoArgsConstructor - public static class User { - String name; - int age; - } - - @Test - public void testJsonConverter() throws Exception { - String AVRO_USER_SCHEMA_DEF = "{" + - "\"type\":\"record\"," + - "\"name\":\"User\"," + - "\"namespace\":\"org.apache.pulsar.io.kafka.connect.schema.KafkaSchemaTest\"," + - "\"fields\":[" + - "{\"name\":\"name\",\"type\":\"string\",\"default\":null}," + - "{\"name\":\"age\",\"type\":\"int\"}" + - "]}"; - org.apache.pulsar.kafka.shade.avro.Schema AVRO_USER_SCHEMA = new org.apache.pulsar.kafka.shade.avro.Schema - .Parser().parse(AVRO_USER_SCHEMA_DEF); - kafkaSchema.setAvroSchema( - false, - avroData, - AVRO_USER_SCHEMA, - new JsonConverter() - ); - - User user = new User("user-1", 100); - byte[] data = userJsonSchema.encode(user); - byte[] encodedData = kafkaSchema.encode(data); - assertSame(data, encodedData); - } - - @Test - public void testAvroConverter() throws Exception { - String AVRO_USER_SCHEMA_DEF = "{" + - "\"type\":\"record\"," + - "\"name\":\"User\"," + - "\"namespace\":\"org.apache.pulsar.io.kafka.connect.schema.KafkaSchemaTest\"," + - "\"fields\":[" + - "{\"name\":\"name\",\"type\":\"string\"}," + - "{\"name\":\"age\",\"type\":\"int\"}" + - "]}"; - org.apache.pulsar.kafka.shade.avro.Schema AVRO_USER_SCHEMA = new org.apache.pulsar.kafka.shade.avro.Schema - .Parser().parse(AVRO_USER_SCHEMA_DEF); - Map config = new HashMap<>(); - AvroConverter converter = new AvroConverter(new MockSchemaRegistryClient()); - config.put( - AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, - "mock"); - converter.configure(config, false); - kafkaSchema.setAvroSchema( - false, - avroData, - AVRO_USER_SCHEMA, - converter - ); - - log.info("Initialized with avro schema {}", AVRO_USER_SCHEMA); - - User user = new User("user-1", 100); - byte[] jsonData = userJsonSchema.encode(user); - - byte[] avroData = kafkaSchema.encode(jsonData); - - - Schema userAvroSchema = Schema.generic( - SchemaInfo.builder() - .name("") - .properties(Collections.emptyMap()) - .type(SchemaType.AVRO) - .schema(userJsonSchema.getSchemaInfo().getSchema()) - .build() - ); - GenericRecord userRecord = userAvroSchema.decode(avroData); - assertEquals(user.getName(), userRecord.getField("name")); - assertEquals(user.getAge(), userRecord.getField("age")); - - } -} - diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 70e9f451ade40..62ffaa2604262 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -140,18 +140,24 @@ public void testRabbitMQSink() throws Exception { } @Test(groups = "source") - public void testDebeziumMySqlSource() throws Exception { - testDebeziumMySqlConnect(); + public void testDebeziumMySqlSourceJson() throws Exception { + testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter"); + } + + @Test(groups = "source") + public void testDebeziumMySqlSourceAvro() throws Exception { + testDebeziumMySqlConnect( + "org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter"); } @Test(groups = "source") public void testDebeziumPostgreSqlSource() throws Exception { - testDebeziumPostgreSqlConnect(); + testDebeziumPostgreSqlConnect("org.apache.kafka.connect.json.JsonConverter"); } @Test(groups = "source") public void testDebeziumMongoDbSource() throws Exception{ - testDebeziumMongoDbConnect(); + testDebeziumMongoDbConnect("org.apache.kafka.connect.json.JsonConverter"); } private void testSink(SinkTester tester, boolean builtin) throws Exception { @@ -2262,8 +2268,7 @@ public void testAvroSchemaFunction() throws Exception { getFunctionInfoNotFound(functionName); } - private void testDebeziumMySqlConnect() - throws Exception { + private void testDebeziumMySqlConnect(String converterClassName) throws Exception { final String tenant = TopicName.PUBLIC_TENANT; final String namespace = TopicName.DEFAULT_NAMESPACE; @@ -2301,14 +2306,14 @@ private void testDebeziumMySqlConnect() admin.topics().createNonPartitionedTopic(outputTopicName); @Cleanup - Consumer> consumer = client.newConsumer(KeyValueSchema.kvBytes()) + Consumer consumer = client.newConsumer(getSchema(converterClassName)) .topic(consumeTopicName) .subscriptionName("debezium-source-tester") .subscriptionType(SubscriptionType.Exclusive) .subscribe(); @Cleanup - DebeziumMySqlSourceTester sourceTester = new DebeziumMySqlSourceTester(pulsarCluster); + DebeziumMySqlSourceTester sourceTester = new DebeziumMySqlSourceTester(pulsarCluster, converterClassName); // setup debezium mysql server DebeziumMySQLContainer mySQLContainer = new DebeziumMySQLContainer(pulsarCluster.getClusterName()); @@ -2331,25 +2336,25 @@ private void testDebeziumMySqlConnect() waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); // validate the source result - sourceTester.validateSourceResult(consumer, 9, null); + sourceTester.validateSourceResult(consumer, 9, null, converterClassName); // prepare insert event sourceTester.prepareInsertEvent(); // validate the source insert event - sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT); + sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT, converterClassName); // prepare update event sourceTester.prepareUpdateEvent(); // validate the source update event - sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE); + sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE, converterClassName); // prepare delete event sourceTester.prepareDeleteEvent(); // validate the source delete event - sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE); + sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE, converterClassName); // delete the source deleteSource(tenant, namespace, sourceName); @@ -2358,7 +2363,7 @@ private void testDebeziumMySqlConnect() getSourceInfoNotFound(tenant, namespace, sourceName); } - private void testDebeziumPostgreSqlConnect() throws Exception { + private void testDebeziumPostgreSqlConnect(String converterClassName) throws Exception { final String tenant = TopicName.PUBLIC_TENANT; final String namespace = TopicName.DEFAULT_NAMESPACE; @@ -2396,7 +2401,7 @@ private void testDebeziumPostgreSqlConnect() throws Exception { admin.topics().createNonPartitionedTopic(outputTopicName); @Cleanup - Consumer> consumer = client.newConsumer(KeyValueSchema.kvBytes()) + Consumer consumer = client.newConsumer(getSchema(converterClassName)) .topic(consumeTopicName) .subscriptionName("debezium-source-tester") .subscriptionType(SubscriptionType.Exclusive) @@ -2426,25 +2431,25 @@ private void testDebeziumPostgreSqlConnect() throws Exception { waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); // validate the source result - sourceTester.validateSourceResult(consumer, 9, null); + sourceTester.validateSourceResult(consumer, 9, null, converterClassName); // prepare insert event sourceTester.prepareInsertEvent(); // validate the source insert event - sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT); + sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT, converterClassName); // prepare update event sourceTester.prepareUpdateEvent(); // validate the source update event - sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE); + sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE, converterClassName); // prepare delete event sourceTester.prepareDeleteEvent(); // validate the source delete event - sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE); + sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE, converterClassName); // delete the source deleteSource(tenant, namespace, sourceName); @@ -2453,7 +2458,7 @@ private void testDebeziumPostgreSqlConnect() throws Exception { getSourceInfoNotFound(tenant, namespace, sourceName); } - private void testDebeziumMongoDbConnect() throws Exception { + private void testDebeziumMongoDbConnect(String converterClassName) throws Exception { final String tenant = TopicName.PUBLIC_TENANT; final String namespace = TopicName.DEFAULT_NAMESPACE; @@ -2491,7 +2496,7 @@ private void testDebeziumMongoDbConnect() throws Exception { admin.topics().createNonPartitionedTopic(outputTopicName); @Cleanup - Consumer> consumer = client.newConsumer(KeyValueSchema.kvBytes()) + Consumer consumer = client.newConsumer(getSchema(converterClassName)) .topic(consumeTopicName) .subscriptionName("debezium-source-tester") .subscriptionType(SubscriptionType.Exclusive) @@ -2520,25 +2525,25 @@ private void testDebeziumMongoDbConnect() throws Exception { waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); // validate the source result - sourceTester.validateSourceResult(consumer, 9, null); + sourceTester.validateSourceResult(consumer, 9, null, converterClassName); // prepare insert event sourceTester.prepareInsertEvent(); // validate the source insert event - sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT); + sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT, converterClassName); // prepare update event sourceTester.prepareUpdateEvent(); // validate the source update event - sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE); + sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE, converterClassName); // prepare delete event sourceTester.prepareDeleteEvent(); // validate the source delete event - sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE); + sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE, converterClassName); // delete the source deleteSource(tenant, namespace, sourceName); @@ -2547,4 +2552,13 @@ private void testDebeziumMongoDbConnect() throws Exception { getSourceInfoNotFound(tenant, namespace, sourceName); } + private Schema getSchema(String converterClassName) { + if (converterClassName.endsWith("AvroConverter")) { + return KeyValueSchema.of(Schema.AUTO_CONSUME(), Schema.AUTO_CONSUME()); + } else { + return KeyValueSchema.kvBytes(); + } + + } + } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMySqlSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMySqlSourceTester.java index 3287e2b0750c2..901e66453ce11 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMySqlSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMySqlSourceTester.java @@ -48,7 +48,7 @@ public class DebeziumMySqlSourceTester extends SourceTester { protected final String sourceType; protected final Map sourceConfig; + public final static Set DEBEZIUM_FIELD_SET = new HashSet() {{ + add("before"); + add("after"); + add("source"); + add("op"); + add("ts_ms"); + }}; + protected SourceTester(String sourceType) { this.sourceType = sourceType; this.sourceConfig = Maps.newHashMap(); @@ -71,7 +83,16 @@ public Map sourceConfig() { public abstract Map produceSourceMessages(int numMessages) throws Exception; - public void validateSourceResult(Consumer> consumer, int number, String eventType) throws Exception { + public void validateSourceResult(Consumer consumer, int number, + String eventType, String converterClassName) throws Exception { + if (converterClassName.endsWith("AvroConverter")) { + validateSourceResultAvro(consumer, number, eventType); + } else { + validateSourceResultJson(consumer, number, eventType); + } + } + + public void validateSourceResultJson(Consumer> consumer, int number, String eventType) throws Exception { int recordsNumber = 0; Message> msg = consumer.receive(2, TimeUnit.SECONDS); while(msg != null) { @@ -91,9 +112,40 @@ public void validateSourceResult(Consumer> consumer, in Assert.assertEquals(recordsNumber, number); log.info("Stop {} server container. topic: {} has {} records.", getSourceType(), consumer.getTopic(), recordsNumber); } + + public void validateSourceResultAvro(Consumer> consumer, + int number, String eventType) throws Exception { + int recordsNumber = 0; + Message> msg = consumer.receive(2, TimeUnit.SECONDS); + while(msg != null) { + recordsNumber ++; + GenericRecord keyRecord = msg.getValue().getKey(); + Assert.assertNotNull(keyRecord.getFields()); + Assert.assertTrue(keyRecord.getFields().size() > 0); + + GenericRecord valueRecord = msg.getValue().getValue(); + Assert.assertNotNull(valueRecord.getFields()); + Assert.assertTrue(valueRecord.getFields().size() > 0); + for (Field field : valueRecord.getFields()) { + Assert.assertTrue(DEBEZIUM_FIELD_SET.contains(field.getName())); + } + + if (eventType != null) { + String op = valueRecord.getField("op").toString(); + Assert.assertEquals(this.eventContains(eventType), op); + } + consumer.acknowledge(msg); + msg = consumer.receive(1, TimeUnit.SECONDS); + } + + Assert.assertEquals(recordsNumber, number); + log.info("Stop {} server container. topic: {} has {} records.", getSourceType(), consumer.getTopic(), recordsNumber); + } + public String keyContains(){ return "dbserver1.inventory.products.Key"; } + public String valueContains(){ return "dbserver1.inventory.products.Value"; } From 6675996acc3e81d5a0685cadefea6cdb980ef32f Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Fri, 17 Apr 2020 11:41:43 +0800 Subject: [PATCH 07/36] fix --- .../io/kafka/connect/schema/KafkaSchemaWrappedSchema.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java index ed2454fed679a..f7a31e7332beb 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java @@ -63,4 +63,9 @@ public byte[] encode(byte[] data) { public SchemaInfo getSchemaInfo() { return schemaInfo; } + + @Override + public Schema clone() { + return null; + } } From 86d477925681edb76dd64ca75525ec8eea29c74c Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Fri, 17 Apr 2020 14:07:48 +0800 Subject: [PATCH 08/36] fix --- .../pulsar/io/kafka/connect/KafkaConnectSource.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index 68110f63700ea..9019048552d91 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -228,16 +228,20 @@ private class KafkaSourceRecord implements Record> { this.topicName = Optional.of(srcRecord.topic()); - keySchema = readerCache.getIfPresent(srcRecord.keySchema()); - valueSchema = readerCache.getIfPresent(srcRecord.valueSchema()); + if (srcRecord.keySchema() != null) { + keySchema = readerCache.getIfPresent(srcRecord.keySchema()); + } + if (srcRecord.valueSchema() != null) { + valueSchema = readerCache.getIfPresent(srcRecord.valueSchema()); + } - if (keySchema == null) { + if (srcRecord.keySchema() != null && keySchema == null) { keySchema = new KafkaSchemaWrappedSchema( avroData.fromConnectSchema(srcRecord.keySchema()), keyConverter); readerCache.put(srcRecord.keySchema(), keySchema); } - if (valueSchema == null) { + if (srcRecord.valueSchema() != null && valueSchema == null) { valueSchema = new KafkaSchemaWrappedSchema( avroData.fromConnectSchema(srcRecord.valueSchema()), valueConverter); readerCache.put(srcRecord.valueSchema(), valueSchema); From 0f041d7ec3a52478a3a62954e62e3d2e898898c6 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Tue, 21 Apr 2020 11:34:14 +0800 Subject: [PATCH 09/36] add some test log --- .../client/impl/TypedMessageBuilderImpl.java | 6 ++++++ .../pulsar/functions/instance/SinkRecord.java | 20 ++++++++++++++++++- .../pulsar/functions/sink/PulsarSink.java | 4 ++++ .../apache/pulsar/functions/LocalRunner.java | 3 ++- 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java index 8d7884954bd9a..bf9ec8a851ddb 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java @@ -30,6 +30,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClientException; @@ -43,6 +44,7 @@ import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString; +@Slf4j public class TypedMessageBuilderImpl implements TypedMessageBuilder { private static final long serialVersionUID = 0L; @@ -108,6 +110,10 @@ public CompletableFuture sendAsync() { @Override public TypedMessageBuilder key(String key) { if (schema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { + log.info("[key] KeyValueSchema className: {}, classLoader: {}", + KeyValueSchema.class.getName(), KeyValueSchema.class.getClassLoader()); + log.info("[key] schema className: {}, classLoader: {}", + schema.getClass().getName(), schema.getClass().getClassLoader()); KeyValueSchema kvSchema = (KeyValueSchema) schema; checkArgument(!(kvSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED), "This method is not allowed to set keys when in encoding type is SEPARATED"); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java index d45345d340e02..c9739a100c196 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java @@ -24,9 +24,13 @@ import lombok.AllArgsConstructor; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.Record; +@Slf4j @Data @AllArgsConstructor public class SinkRecord implements Record { @@ -85,6 +89,20 @@ public Optional getDestinationTopic() { @Override public Schema getSchema() { - return sourceRecord.getSchema(); + log.info("[SinkRecord] Schema classLoader: {}", Schema.class.getClassLoader()); + if (sourceRecord != null) { + SchemaInfo srcSchemaInfo = sourceRecord.getSchema().getSchemaInfo(); + SchemaInfo schemaInfo = SchemaInfo.builder() + .name(srcSchemaInfo.getName()) + .schema(srcSchemaInfo.getSchema()) + .type(SchemaType.valueOf(srcSchemaInfo.getType().getValue())) + .properties(srcSchemaInfo.getProperties()) + .build(); + Schema schema = (Schema) Schema.getSchema(schemaInfo); + log.info("[SinkRecord] schemaInfo: {}", schemaInfo); + return schema; + } else { + return null; + } } } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java index d2aaad0d5efb8..a6e5eabf847d9 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java @@ -293,6 +293,10 @@ public void open(Map config, SinkContext sinkContext) throws Exc @Override public void write(Record record) { TypedMessageBuilder msg = pulsarSinkProcessor.newMessage(record); + log.info("[write] KeyValueSchema className: {}, classLoader: {}", + KeyValueSchema.class.getName(), KeyValueSchema.class.getClassLoader()); + log.info("[write] schema className: {}, classLoader: {}", + record.getSchema().getClass().getName(), record.getSchema().getClass().getClassLoader()); if (record.getKey().isPresent() && !(record.getSchema() instanceof KeyValueSchema && ((KeyValueSchema) record.getSchema()).getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED)) { msg.key(record.getKey().get()); diff --git a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java index 244a757dbc480..9fb8de4668b35 100644 --- a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java +++ b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java @@ -249,6 +249,7 @@ public void start(boolean blocking) throws Exception { } String builtInSource = isBuiltInSource(userCodeFile); + log.info("builtInSource: {}", builtInSource); if (builtInSource != null) { sourceConfig.setArchive(builtInSource); } @@ -261,7 +262,7 @@ public void start(boolean blocking) throws Exception { } else { File file = new File(userCodeFile); if (!file.exists()) { - throw new RuntimeException("Source archive does not exist"); + throw new RuntimeException("Source archive (" + userCodeFile + ") does not exist"); } functionDetails = SourceConfigUtils.convert(sourceConfig, SourceConfigUtils.validate(sourceConfig, null, file)); } From 01eee75a928ca0181885649a74872e3ccdabbee9 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Tue, 21 Apr 2020 12:48:00 +0800 Subject: [PATCH 10/36] fix --- .../java/org/apache/pulsar/functions/instance/SinkRecord.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java index c9739a100c196..eac5e758a88c3 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java @@ -90,7 +90,7 @@ public Optional getDestinationTopic() { @Override public Schema getSchema() { log.info("[SinkRecord] Schema classLoader: {}", Schema.class.getClassLoader()); - if (sourceRecord != null) { + if (sourceRecord != null && sourceRecord.getSchema() != null) { SchemaInfo srcSchemaInfo = sourceRecord.getSchema().getSchemaInfo(); SchemaInfo schemaInfo = SchemaInfo.builder() .name(srcSchemaInfo.getName()) From 58bfc7de217b7b8ccb25466f8d8dce9bc34476f8 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Tue, 21 Apr 2020 12:56:48 +0800 Subject: [PATCH 11/36] add some test log --- .../pulsar/functions/instance/SinkRecord.java | 1 + .../containers/ChaosContainer.java | 21 +++++++++++++++++++ .../functions/PulsarFunctionsTest.java | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java index eac5e758a88c3..d7e3ebbbc4dd8 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java @@ -92,6 +92,7 @@ public Schema getSchema() { log.info("[SinkRecord] Schema classLoader: {}", Schema.class.getClassLoader()); if (sourceRecord != null && sourceRecord.getSchema() != null) { SchemaInfo srcSchemaInfo = sourceRecord.getSchema().getSchemaInfo(); + log.info("[SinkRecord] map classLoader: {}", srcSchemaInfo.getProperties().getClass().getClassLoader()); SchemaInfo schemaInfo = SchemaInfo.builder() .name(srcSchemaInfo.getName()) .schema(srcSchemaInfo.getSchema()) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java index 730584953d431..3a0568f296d06 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java @@ -139,4 +139,25 @@ public int hashCode() { clusterName); } + @Override + public void start() { + super.start(); + this.tailContainerLog(); + if (this.getContainerName().contains("functions-worker")) { + DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), + "tail", "-f", "/var/log/pulsar/functions_worker.log"); + DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), + "mkdir", "-p", + "/tmp/functions/public/default/test-source-connector-PROCESS-name-mysql"); + DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), + "touch", + "/tmp/functions/public/default/test-source-connector-PROCESS-name-mysql/" + + "test-source-connector-PROCESS-name-mysql-0.log"); + DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), + "tail", "-f", + "/tmp/functions/public/default/test-source-connector-PROCESS-name-mysql/" + + "test-source-connector-PROCESS-name-mysql-0.log"); + } + } + } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 62ffaa2604262..96ad825bdd68a 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -2275,7 +2275,7 @@ private void testDebeziumMySqlConnect(String converterClassName) throws Exceptio final String outputTopicName = "debe-output-topic-name"; final String consumeTopicName = "public/default/dbserver1.inventory.products"; final String sourceName = "test-source-connector-" - + functionRuntimeType + "-name-" + randomName(8); + + functionRuntimeType + "-name-mysql"; // This is the binlog count that contained in mysql container. final int numMessages = 47; From 78856e0ca0ffe9b8717241f49b67b725d60521b3 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Tue, 21 Apr 2020 14:08:08 +0800 Subject: [PATCH 12/36] fix --- .../org/apache/pulsar/functions/sink/PulsarSink.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java index a6e5eabf847d9..7b75232b7dac6 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java @@ -293,10 +293,14 @@ public void open(Map config, SinkContext sinkContext) throws Exc @Override public void write(Record record) { TypedMessageBuilder msg = pulsarSinkProcessor.newMessage(record); - log.info("[write] KeyValueSchema className: {}, classLoader: {}", - KeyValueSchema.class.getName(), KeyValueSchema.class.getClassLoader()); - log.info("[write] schema className: {}, classLoader: {}", - record.getSchema().getClass().getName(), record.getSchema().getClass().getClassLoader()); + + if (record != null && record.getSchema() != null) { + log.info("[write] KeyValueSchema className: {}, classLoader: {}", + KeyValueSchema.class.getName(), KeyValueSchema.class.getClassLoader()); + log.info("[write] schema className: {}, classLoader: {}", + record.getSchema().getClass().getName(), record.getSchema().getClass().getClassLoader()); + } + if (record.getKey().isPresent() && !(record.getSchema() instanceof KeyValueSchema && ((KeyValueSchema) record.getSchema()).getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED)) { msg.key(record.getKey().get()); From ded2cd0e07b5b9d40ff14e8c10e06b55bbf68206 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Tue, 21 Apr 2020 17:33:26 +0800 Subject: [PATCH 13/36] fix --- .../integration/functions/PulsarFunctionsTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 96ad825bdd68a..e30a79eb5400e 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -2305,13 +2305,6 @@ private void testDebeziumMySqlConnect(String converterClassName) throws Exceptio admin.topics().createNonPartitionedTopic(consumeTopicName); admin.topics().createNonPartitionedTopic(outputTopicName); - @Cleanup - Consumer consumer = client.newConsumer(getSchema(converterClassName)) - .topic(consumeTopicName) - .subscriptionName("debezium-source-tester") - .subscriptionType(SubscriptionType.Exclusive) - .subscribe(); - @Cleanup DebeziumMySqlSourceTester sourceTester = new DebeziumMySqlSourceTester(pulsarCluster, converterClassName); @@ -2335,6 +2328,13 @@ private void testDebeziumMySqlConnect(String converterClassName) throws Exceptio Failsafe.with(statusRetryPolicy).run(() -> waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); + @Cleanup + Consumer consumer = client.newConsumer(getSchema(converterClassName)) + .topic(consumeTopicName) + .subscriptionName("debezium-source-tester") + .subscriptionType(SubscriptionType.Exclusive) + .subscribe(); + // validate the source result sourceTester.validateSourceResult(consumer, 9, null, converterClassName); From b15839d8c28da79045fb0f856227bb36c9914e7d Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Wed, 22 Apr 2020 00:28:05 +0800 Subject: [PATCH 14/36] use the ClassLoader of the SourceContext to generate KeyValueSchema --- .../java/org/apache/pulsar/DebeziumTest.java | 83 +++++++++++++++++++ .../pulsar/functions/instance/SinkRecord.java | 33 +++++--- .../io/kafka/connect/KafkaConnectSource.java | 41 ++++++++- .../schema/KafkaSchemaWrappedSchema.java | 11 +-- .../functions/PulsarFunctionsTest.java | 20 +++-- 5 files changed, 156 insertions(+), 32 deletions(-) create mode 100644 pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java diff --git a/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java b/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java new file mode 100644 index 0000000000000..7113d12bff87a --- /dev/null +++ b/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java @@ -0,0 +1,83 @@ +package org.apache.pulsar; + +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.schema.Field; +import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.testng.annotations.Test; + +public class DebeziumTest { + + @Test + private void testJsonConverter() throws PulsarClientException { + PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); + + Schema> schema = + Schema.KeyValue(Schema.BYTES, Schema.BYTES); + + Consumer> consumer = pulsarClient.newConsumer(schema) + .topic("public/default/dbserver1.inventory.products") + .subscriptionName("journey-test") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + while (true) { + Message> message = consumer.receive(); + KeyValue keyValue = message.getValue(); + System.out.println("----------- get message -----------"); + System.out.println("key: " + new String(keyValue.getKey())); + System.out.println("value: " + new String(keyValue.getValue())); + } + } + + private void testAvroConverter() throws PulsarClientException { + PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); + + Schema> schema = + Schema.KeyValue(Schema.AUTO_CONSUME(), Schema.AUTO_CONSUME(), KeyValueEncodingType.SEPARATED); + + Consumer> consumer = pulsarClient.newConsumer(schema) + .topic("public/default/dbserver1.inventory.products") + .subscriptionName("journey-test") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + while (true) { + Message> message = consumer.receive(); + try { + message.getKeyBytes(); + message.getData(); + KeyValue result = message.getValue(); + System.out.println("------------- got message -------------"); + + System.out.println("key >>>>>>>>>>> "); + for (Field field : result.getKey().getFields()) { + Object obj = result.getKey().getField(field); + System.out.println(field.getName() + ":" + (obj == null ? "null" : obj.toString())); + } + + System.out.println("value >>>>>>>>>>> "); + for (Field field : result.getValue().getFields()) { + Object obj = result.getValue().getField(field); + System.out.println(field.getName() + ":" + (obj == null ? "null" : obj.toString())); + if (obj != null && !"null".equalsIgnoreCase(obj.toString()) && (field.getName().equals("source") || + field.getName().equals("before") || field.getName().equals("after"))) { + for (Field innerField : ((GenericRecord) obj).getFields()) { + Object innerObj = ((GenericRecord) obj).getField(innerField); + System.out.println(" " + innerField.getName() + ":" + (innerObj == null ? "null" : innerObj.toString())); + } + } + } + } catch (Exception e) { + e.printStackTrace(); +// consumer.acknowledge(message); + } + } + } +} diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java index d7e3ebbbc4dd8..211781b759c29 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java @@ -90,20 +90,27 @@ public Optional getDestinationTopic() { @Override public Schema getSchema() { log.info("[SinkRecord] Schema classLoader: {}", Schema.class.getClassLoader()); - if (sourceRecord != null && sourceRecord.getSchema() != null) { - SchemaInfo srcSchemaInfo = sourceRecord.getSchema().getSchemaInfo(); - log.info("[SinkRecord] map classLoader: {}", srcSchemaInfo.getProperties().getClass().getClassLoader()); - SchemaInfo schemaInfo = SchemaInfo.builder() - .name(srcSchemaInfo.getName()) - .schema(srcSchemaInfo.getSchema()) - .type(SchemaType.valueOf(srcSchemaInfo.getType().getValue())) - .properties(srcSchemaInfo.getProperties()) - .build(); - Schema schema = (Schema) Schema.getSchema(schemaInfo); - log.info("[SinkRecord] schemaInfo: {}", schemaInfo); - return schema; - } else { + if (sourceRecord == null || sourceRecord.getSchema() == null) { return null; } + log.info("[SinkRecord] sourceRecord schema: {}, classLoader: {}", + sourceRecord.getSchema().getSchemaInfo().toString(), Schema.class.getClassLoader()); + return sourceRecord.getSchema(); +// log.info("[SinkRecord] Schema classLoader: {}", Schema.class.getClassLoader()); +// if (sourceRecord != null && sourceRecord.getSchema() != null) { +// SchemaInfo srcSchemaInfo = sourceRecord.getSchema().getSchemaInfo(); +// log.info("[SinkRecord] map classLoader: {}", srcSchemaInfo.getProperties().getClass().getClassLoader()); +// SchemaInfo schemaInfo = SchemaInfo.builder() +// .name(srcSchemaInfo.getName()) +// .schema(srcSchemaInfo.getSchema()) +// .type(SchemaType.valueOf(srcSchemaInfo.getType().getValue())) +// .properties(srcSchemaInfo.getProperties()) +// .build(); +// Schema schema = (Schema) Schema.getSchema(schemaInfo); +// log.info("[SinkRecord] schemaInfo: {}", schemaInfo); +// return schema; +// } else { +// return null; +// } } } diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index 9019048552d91..cad7a74fc1030 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -20,6 +20,7 @@ import static org.apache.pulsar.io.kafka.connect.PulsarKafkaWorkerConfig.TOPIC_NAMESPACE_CONFIG; +import java.lang.reflect.Method; import java.util.Base64; import java.util.Collections; import java.util.HashMap; @@ -37,6 +38,8 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import org.apache.kafka.connect.json.JsonConverter; +import org.apache.kafka.connect.json.JsonConverterConfig; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter; import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroData; @@ -85,12 +88,19 @@ public class KafkaConnectSource implements Source> { // number of outstandingRecords that have been polled but not been acked private AtomicInteger outstandingRecords = new AtomicInteger(0); + private boolean jsonWithEnvelope = false; + static private final String JSON_WITH_ENVELOPE_CONFIG = "json-with-envelope"; + private final Cache readerCache = CacheBuilder.newBuilder().maximumSize(10000) .expireAfterAccess(30, TimeUnit.MINUTES).build(); + private Method keyValueSchemaOfMethod; + @Override public void open(Map config, SourceContext sourceContext) throws Exception { + initKeyValueSchemaOfMethod(sourceContext.getClass().getClassLoader()); + Map stringConfig = new HashMap<>(); config.forEach((key, value) -> { if (value instanceof String) { @@ -98,6 +108,13 @@ public void open(Map config, SourceContext sourceContext) throws } }); + if (config.get(JSON_WITH_ENVELOPE_CONFIG) != null) { + jsonWithEnvelope = Boolean.parseBoolean(config.get(JSON_WITH_ENVELOPE_CONFIG).toString()); + config.put(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, false); + } else { + config.put(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, false); + } + // get the source class name from config and create source task from reflection sourceTask = ((Class)Class.forName(stringConfig.get(TaskConfig.TASK_CLASS_CONFIG))) .asSubclass(SourceTask.class) @@ -262,10 +279,17 @@ public Schema getSchema() { // as the key.converter and value.converter, make the `KeyValueSchema` encodingType // use the `KeyValueEncodingType.SEPARATED`, then the pulsar client could get the original // byte array which are converted by the AvroConverter, or consume the GenericRecord object. - if (keyConverter instanceof AvroConverter && valueConverter instanceof AvroConverter) { - return KeyValueSchema.of(keySchema, valueSchema, KeyValueEncodingType.SEPARATED); - } else { - return KeyValueSchema.of(Schema.BYTES, Schema.BYTES); + try { + if (jsonWithEnvelope) { + return (Schema) keyValueSchemaOfMethod.invoke( + Schema.BYTES, Schema.BYTES, KeyValueEncodingType.INLINE); + } else { + return (Schema) keyValueSchemaOfMethod.invoke( + keySchema, valueSchema, KeyValueEncodingType.SEPARATED); + } + } catch (Exception e) { + log.error("failed to invoke the keyValueSchemaOfMethod."); + return null; } } @@ -338,4 +362,13 @@ public void fail() { } } } + + private void initKeyValueSchemaOfMethod(ClassLoader classLoader) throws ClassNotFoundException, + NoSuchMethodException { + Class keyValueSchemaClazz = + (Class) classLoader.loadClass(KeyValueSchema.class.getName()); + keyValueSchemaOfMethod = keyValueSchemaClazz.getDeclaredMethod( + "of", Schema.class, Schema.class, KeyValueEncodingType.class); + keyValueSchemaOfMethod.setAccessible(true); + } } diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java index f7a31e7332beb..6b9158aba4aad 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java @@ -40,15 +40,12 @@ public class KafkaSchemaWrappedSchema implements Schema { public KafkaSchemaWrappedSchema(org.apache.pulsar.kafka.shade.avro.Schema schema, Converter converter) { - if (converter instanceof JsonConverter) { - return; - } Map props = new HashMap<>(); - props.put(GenericAvroReader.OFFSET_PROP, "5"); - + boolean isJsonConverter = converter instanceof JsonConverter; + props.put(GenericAvroReader.OFFSET_PROP, isJsonConverter ? "0" : "5"); this.schemaInfo = SchemaInfo.builder() - .name("KafkaAvro") - .type(SchemaType.AVRO) + .name(isJsonConverter? "KafKaJson" : "KafkaAvro") + .type(isJsonConverter ? SchemaType.JSON : SchemaType.AVRO) .schema(schema.toString().getBytes(UTF_8)) .properties(props) .build(); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index e30a79eb5400e..cd8bed0787aa1 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -2302,7 +2302,18 @@ private void testDebeziumMySqlConnect(String converterClassName) throws Exceptio log.info("Topic: {} does not exist, we can continue the following tests. Exceptions message: {}", consumeTopicName, e.getMessage()); } - admin.topics().createNonPartitionedTopic(consumeTopicName); + + SchemaInfo lastSchemaInfo = admin.schemas().getSchemaInfo(consumeTopicName); + log.info("lastSchemaInfo: {}", lastSchemaInfo == null ? "null" : lastSchemaInfo.toString()); + + @Cleanup + Consumer consumer = client.newConsumer(getSchema(converterClassName)) + .topic(consumeTopicName) + .subscriptionName("debezium-source-tester") + .subscriptionType(SubscriptionType.Exclusive) + .subscribe(); + +// admin.topics().createNonPartitionedTopic(consumeTopicName); admin.topics().createNonPartitionedTopic(outputTopicName); @Cleanup @@ -2328,13 +2339,6 @@ private void testDebeziumMySqlConnect(String converterClassName) throws Exceptio Failsafe.with(statusRetryPolicy).run(() -> waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); - @Cleanup - Consumer consumer = client.newConsumer(getSchema(converterClassName)) - .topic(consumeTopicName) - .subscriptionName("debezium-source-tester") - .subscriptionType(SubscriptionType.Exclusive) - .subscribe(); - // validate the source result sourceTester.validateSourceResult(consumer, 9, null, converterClassName); From dcf920230f9297bdc250371076f43d609ffd6768 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Wed, 22 Apr 2020 12:35:26 +0800 Subject: [PATCH 15/36] some test --- .../client/api/schema/SchemaInfoProvider.java | 3 +- .../pulsar/common/schema/SchemaInfo.java | 3 +- .../client/impl/TypedMessageBuilderImpl.java | 16 ++++--- .../client/impl/schema/KeyValueSchema.java | 3 +- .../java/org/apache/pulsar/DebeziumTest.java | 26 +++++++++- .../pulsar/functions/instance/SinkRecord.java | 28 ++++++++++- .../io/kafka/connect/KafkaConnectSource.java | 48 +++++++++++++------ .../schema/KafkaSchemaWrappedSchema.java | 5 +- .../pulsar/io/kafka/connect/SerTest.java | 42 ++++++++++++++++ 9 files changed, 147 insertions(+), 27 deletions(-) create mode 100644 pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SerTest.java diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaInfoProvider.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaInfoProvider.java index c4ba4813ef98d..6de3afb15738e 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaInfoProvider.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaInfoProvider.java @@ -18,13 +18,14 @@ */ package org.apache.pulsar.client.api.schema; +import java.io.Serializable; import java.util.concurrent.CompletableFuture; import org.apache.pulsar.common.schema.SchemaInfo; /** * Schema Provider. */ -public interface SchemaInfoProvider { +public interface SchemaInfoProvider extends Serializable { /** * Retrieve the schema info of a given schemaVersion. diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java b/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java index acbe34e58d431..846047ec16941 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java @@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import java.io.Serializable; import java.util.Base64; import java.util.Collections; import java.util.Map; @@ -40,7 +41,7 @@ @NoArgsConstructor @Accessors(chain = true) @Builder -public class SchemaInfo { +public class SchemaInfo implements Serializable { @EqualsAndHashCode.Exclude private String name; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java index bf9ec8a851ddb..2c9039815498c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java @@ -114,8 +114,12 @@ public TypedMessageBuilder key(String key) { KeyValueSchema.class.getName(), KeyValueSchema.class.getClassLoader()); log.info("[key] schema className: {}, classLoader: {}", schema.getClass().getName(), schema.getClass().getClassLoader()); - KeyValueSchema kvSchema = (KeyValueSchema) schema; - checkArgument(!(kvSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED), + + log.info("[key] KeyValueSchema encodingType, classLoader: {}", KeyValueEncodingType.SEPARATED); + log.info("[key] schema encodingType, classLoader: {}", + ((KeyValueSchema) schema).getKeyValueEncodingType().getClass().getClassLoader()); +// KeyValueSchema kvSchema = (KeyValueSchema) schema; + checkArgument(!(((KeyValueSchema) schema).getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED), "This method is not allowed to set keys when in encoding type is SEPARATED"); } msgMetadataBuilder.setPartitionKey(key); @@ -146,15 +150,15 @@ public TypedMessageBuilder value(T value) { checkArgument(value != null, "Need Non-Null content value"); if (schema.getSchemaInfo() != null && schema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { - KeyValueSchema kvSchema = (KeyValueSchema) schema; +// KeyValueSchema kvSchema = (KeyValueSchema) schema; org.apache.pulsar.common.schema.KeyValue kv = (org.apache.pulsar.common.schema.KeyValue) value; - if (kvSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) { + if (((KeyValueSchema) schema).getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) { // set key as the message key msgMetadataBuilder.setPartitionKey( - Base64.getEncoder().encodeToString(kvSchema.getKeySchema().encode(kv.getKey()))); + Base64.getEncoder().encodeToString(((KeyValueSchema) schema).getKeySchema().encode(kv.getKey()))); msgMetadataBuilder.setPartitionKeyB64Encoded(true); // set value as the payload - this.content = ByteBuffer.wrap(kvSchema.getValueSchema().encode(kv.getValue())); + this.content = ByteBuffer.wrap(((KeyValueSchema) schema).getValueSchema().encode(kv.getValue())); return this; } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java index b81a94706bbc8..fa5ebfa8de5f4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkArgument; +import java.io.Serializable; import java.util.concurrent.CompletableFuture; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -35,7 +36,7 @@ * [Key, Value] pair schema definition */ @Slf4j -public class KeyValueSchema implements Schema> { +public class KeyValueSchema implements Schema>, Serializable { @Getter private final Schema keySchema; diff --git a/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java b/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java index 7113d12bff87a..b5d08e48d8a7d 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java @@ -8,6 +8,7 @@ import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.schema.Field; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.impl.schema.KeyValueSchema; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.testng.annotations.Test; @@ -15,7 +16,7 @@ public class DebeziumTest { @Test - private void testJsonConverter() throws PulsarClientException { + private void testJsonConverterBytes() throws PulsarClientException { PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); Schema> schema = @@ -36,6 +37,29 @@ private void testJsonConverter() throws PulsarClientException { } } + @Test + private void testJsonConverter() throws PulsarClientException { + PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); + + Schema> schema = + Schema.KeyValue(Schema.AUTO_CONSUME(), Schema.AUTO_CONSUME(), KeyValueEncodingType.SEPARATED); + + Consumer> consumer = pulsarClient.newConsumer(schema) + .topic("public/default/dbserver1.inventory.products") + .subscriptionName("journey-test") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + while (true) { + Message> message = consumer.receive(); + KeyValue keyValue = message.getValue(); + System.out.println("----------- get message -----------"); + System.out.println("key: " + new String(message.getKeyBytes())); + System.out.println("value: " + new String(message.getData())); + } + } + + @Test private void testAvroConverter() throws PulsarClientException { PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java index 211781b759c29..8ee8cb1e013cf 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java @@ -18,6 +18,12 @@ */ package org.apache.pulsar.functions.instance; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.util.Map; import java.util.Optional; @@ -89,13 +95,33 @@ public Optional getDestinationTopic() { @Override public Schema getSchema() { - log.info("[SinkRecord] Schema classLoader: {}", Schema.class.getClassLoader()); if (sourceRecord == null || sourceRecord.getSchema() == null) { return null; } + + log.info("[SinkRecord] Schema classLoader: {}", Schema.class.getClassLoader()); log.info("[SinkRecord] sourceRecord schema: {}, classLoader: {}", sourceRecord.getSchema().getSchemaInfo().toString(), Schema.class.getClassLoader()); + return sourceRecord.getSchema(); +// ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); +// try { +// ObjectOutputStream oos = new ObjectOutputStream(byteArrayOutputStream); +// oos.writeObject(sourceRecord.getSchema()); +// oos.flush(); +// oos.close(); +// +// ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArrayOutputStream.toByteArray()); +// ObjectInputStream ois = new ObjectInputStream(byteArrayInputStream); +// Schema schema = (Schema) ois.readObject(); +// log.info("deserializable schema: {}, classLoader: {}", +// schema.getClass().getName(), schema.getClass().getClassLoader()); +// return schema; +// } catch (IOException | ClassNotFoundException e) { +// e.printStackTrace(); +// return null; +// } + // log.info("[SinkRecord] Schema classLoader: {}", Schema.class.getClassLoader()); // if (sourceRecord != null && sourceRecord.getSchema() != null) { // SchemaInfo srcSchemaInfo = sourceRecord.getSchema().getSchemaInfo(); diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index cad7a74fc1030..fed2522dd5891 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -99,7 +99,8 @@ public class KafkaConnectSource implements Source> { @Override public void open(Map config, SourceContext sourceContext) throws Exception { - initKeyValueSchemaOfMethod(sourceContext.getClass().getClassLoader()); + log.info("sourceContext classLoader: {}", sourceContext.getClass().getClassLoader()); +// initKeyValueSchemaOfMethod(sourceContext.getClass().getClassLoader()); Map stringConfig = new HashMap<>(); config.forEach((key, value) -> { @@ -110,10 +111,11 @@ public void open(Map config, SourceContext sourceContext) throws if (config.get(JSON_WITH_ENVELOPE_CONFIG) != null) { jsonWithEnvelope = Boolean.parseBoolean(config.get(JSON_WITH_ENVELOPE_CONFIG).toString()); - config.put(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, false); + config.put(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, jsonWithEnvelope); } else { config.put(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, false); } + log.info("jsonWithEnvelope: {}", jsonWithEnvelope); // get the source class name from config and create source task from reflection sourceTask = ((Class)Class.forName(stringConfig.get(TaskConfig.TASK_CLASS_CONFIG))) @@ -200,7 +202,9 @@ public synchronized Record> read() throws Exception { @Override public void close() { - sourceTask.stop(); + if (sourceTask != null) { + sourceTask.stop(); + } } private synchronized Record> processSourceRecord(final SourceRecord srcRecord) { @@ -279,18 +283,28 @@ public Schema getSchema() { // as the key.converter and value.converter, make the `KeyValueSchema` encodingType // use the `KeyValueEncodingType.SEPARATED`, then the pulsar client could get the original // byte array which are converted by the AvroConverter, or consume the GenericRecord object. - try { - if (jsonWithEnvelope) { - return (Schema) keyValueSchemaOfMethod.invoke( - Schema.BYTES, Schema.BYTES, KeyValueEncodingType.INLINE); - } else { - return (Schema) keyValueSchemaOfMethod.invoke( - keySchema, valueSchema, KeyValueEncodingType.SEPARATED); - } - } catch (Exception e) { - log.error("failed to invoke the keyValueSchemaOfMethod."); - return null; + + if (jsonWithEnvelope) { + return KeyValueSchema.of(Schema.BYTES, Schema.BYTES, KeyValueEncodingType.SEPARATED); + } else { + return KeyValueSchema.of(keySchema, valueSchema, KeyValueEncodingType.SEPARATED); } + +// try { +// log.info("key classLoader: {}", keySchema.getClass().getClassLoader()); +// log.info("value classLoader: {}", valueSchema.getClass().getClassLoader()); +// if (jsonWithEnvelope) { +// return (Schema) keyValueSchemaOfMethod.invoke( +// Schema.BYTES, Schema.BYTES, KeyValueEncodingType.INLINE); +// } else { +// return (Schema) keyValueSchemaOfMethod.invoke( +// keySchema, valueSchema, KeyValueEncodingType.SEPARATED); +// } +// } catch (Exception e) { +// e.printStackTrace(); +// log.error("failed to invoke the keyValueSchemaOfMethod."); +// return null; +// } } @Override @@ -367,8 +381,14 @@ private void initKeyValueSchemaOfMethod(ClassLoader classLoader) throws ClassNot NoSuchMethodException { Class keyValueSchemaClazz = (Class) classLoader.loadClass(KeyValueSchema.class.getName()); + log.info("keyValueSchemaClazz: {}, classLoader: {}", keyValueSchemaClazz.getName(), keyValueSchemaClazz.getClassLoader()); keyValueSchemaOfMethod = keyValueSchemaClazz.getDeclaredMethod( "of", Schema.class, Schema.class, KeyValueEncodingType.class); keyValueSchemaOfMethod.setAccessible(true); + log.info("keyValueSchemaOfMethod: {}", keyValueSchemaOfMethod.toString()); + Class[] clazzArr = keyValueSchemaOfMethod.getParameterTypes(); + for (Class paramClass : clazzArr) { + log.info("paramClass: {}", paramClass.getName()); + } } } diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java index 6b9158aba4aad..ba2aea30b1401 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java @@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import java.io.Serializable; import java.util.HashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -31,10 +32,10 @@ import org.apache.pulsar.common.schema.SchemaType; /** - * Wrapped schema for kafka connect schema, + * Wrapped schema for kafka connect schema. */ @Slf4j -public class KafkaSchemaWrappedSchema implements Schema { +public class KafkaSchemaWrappedSchema implements Schema, Serializable { private SchemaInfo schemaInfo = null; diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SerTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SerTest.java new file mode 100644 index 0000000000000..e4f70fd3fe59e --- /dev/null +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SerTest.java @@ -0,0 +1,42 @@ +package org.apache.pulsar.io.kafka.connect; + +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.schema.KeyValueSchema; +import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.apache.pulsar.io.kafka.connect.schema.KafkaSchemaWrappedSchema; +import org.testng.annotations.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +@Slf4j +public class SerTest { + + @Test + public void test() { + + Schema schema = KeyValueSchema.of(new KafkaSchemaWrappedSchema(null, null), + new KafkaSchemaWrappedSchema(null, null), KeyValueEncodingType.SEPARATED); + + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try { + ObjectOutputStream oos = new ObjectOutputStream(byteArrayOutputStream); + oos.writeObject(schema); + oos.flush(); + oos.close(); + + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArrayOutputStream.toByteArray()); + ObjectInputStream ois = new ObjectInputStream(byteArrayInputStream); + Schema schema2 = (Schema) ois.readObject(); + log.info("deserializable schema: {}, classLoader: {}", + schema.getClass().getName(), schema.getClass().getClassLoader()); + } catch (IOException | ClassNotFoundException e) { + e.printStackTrace(); + } + } + +} From 2a9de584756f663bbda00f7c5e158c7419dcca01 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Wed, 22 Apr 2020 13:59:45 +0800 Subject: [PATCH 16/36] some test --- .../src/test/java/org/apache/pulsar/DebeziumTest.java | 8 ++++---- .../pulsar/io/kafka/connect/KafkaConnectSource.java | 2 +- .../tests/integration/functions/PulsarFunctionsTest.java | 8 ++++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java b/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java index b5d08e48d8a7d..2ce78d56287a9 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java @@ -15,12 +15,12 @@ public class DebeziumTest { - @Test +// @Test private void testJsonConverterBytes() throws PulsarClientException { PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); Schema> schema = - Schema.KeyValue(Schema.BYTES, Schema.BYTES); + Schema.KeyValue(Schema.BYTES, Schema.BYTES, KeyValueEncodingType.SEPARATED); Consumer> consumer = pulsarClient.newConsumer(schema) .topic("public/default/dbserver1.inventory.products") @@ -37,7 +37,7 @@ private void testJsonConverterBytes() throws PulsarClientException { } } - @Test +// @Test private void testJsonConverter() throws PulsarClientException { PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); @@ -59,7 +59,7 @@ private void testJsonConverter() throws PulsarClientException { } } - @Test +// @Test private void testAvroConverter() throws PulsarClientException { PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index fed2522dd5891..77df975489a52 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -285,7 +285,7 @@ public Schema getSchema() { // byte array which are converted by the AvroConverter, or consume the GenericRecord object. if (jsonWithEnvelope) { - return KeyValueSchema.of(Schema.BYTES, Schema.BYTES, KeyValueEncodingType.SEPARATED); + return KeyValueSchema.kvBytes(); } else { return KeyValueSchema.of(keySchema, valueSchema, KeyValueEncodingType.SEPARATED); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index cd8bed0787aa1..5470ba6856b4e 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -2303,8 +2303,12 @@ private void testDebeziumMySqlConnect(String converterClassName) throws Exceptio consumeTopicName, e.getMessage()); } - SchemaInfo lastSchemaInfo = admin.schemas().getSchemaInfo(consumeTopicName); - log.info("lastSchemaInfo: {}", lastSchemaInfo == null ? "null" : lastSchemaInfo.toString()); + try { + SchemaInfo lastSchemaInfo = admin.schemas().getSchemaInfo(consumeTopicName); + log.info("lastSchemaInfo: {}", lastSchemaInfo == null ? "null" : lastSchemaInfo.toString()); + } catch (Exception e) { + log.warn("failed to get schemaInfo for topic: {}", consumeTopicName); + } @Cleanup Consumer consumer = client.newConsumer(getSchema(converterClassName)) From 8a77ba888af2475513dbc0d01ee1f15981619608 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Wed, 22 Apr 2020 16:02:47 +0800 Subject: [PATCH 17/36] fix --- .../client/api/schema/SchemaInfoProvider.java | 3 +- .../pulsar/common/schema/SchemaInfo.java | 3 +- .../client/impl/TypedMessageBuilderImpl.java | 10 +++--- .../client/impl/schema/KeyValueSchema.java | 3 +- .../apache/pulsar/functions/api/KVRecord.java | 11 +++++++ .../pulsar/functions/instance/SinkRecord.java | 21 +++++++++--- .../io/kafka/connect/KafkaConnectSource.java | 32 +++++++++++++++---- 7 files changed, 62 insertions(+), 21 deletions(-) create mode 100644 pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/KVRecord.java diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaInfoProvider.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaInfoProvider.java index 6de3afb15738e..c4ba4813ef98d 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaInfoProvider.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/schema/SchemaInfoProvider.java @@ -18,14 +18,13 @@ */ package org.apache.pulsar.client.api.schema; -import java.io.Serializable; import java.util.concurrent.CompletableFuture; import org.apache.pulsar.common.schema.SchemaInfo; /** * Schema Provider. */ -public interface SchemaInfoProvider extends Serializable { +public interface SchemaInfoProvider { /** * Retrieve the schema info of a given schemaVersion. diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java b/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java index 846047ec16941..acbe34e58d431 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/common/schema/SchemaInfo.java @@ -20,7 +20,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import java.io.Serializable; import java.util.Base64; import java.util.Collections; import java.util.Map; @@ -41,7 +40,7 @@ @NoArgsConstructor @Accessors(chain = true) @Builder -public class SchemaInfo implements Serializable { +public class SchemaInfo { @EqualsAndHashCode.Exclude private String name; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java index 2c9039815498c..e6db901120009 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java @@ -118,8 +118,8 @@ public TypedMessageBuilder key(String key) { log.info("[key] KeyValueSchema encodingType, classLoader: {}", KeyValueEncodingType.SEPARATED); log.info("[key] schema encodingType, classLoader: {}", ((KeyValueSchema) schema).getKeyValueEncodingType().getClass().getClassLoader()); -// KeyValueSchema kvSchema = (KeyValueSchema) schema; - checkArgument(!(((KeyValueSchema) schema).getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED), + KeyValueSchema kvSchema = (KeyValueSchema) schema; + checkArgument(!(kvSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED), "This method is not allowed to set keys when in encoding type is SEPARATED"); } msgMetadataBuilder.setPartitionKey(key); @@ -150,15 +150,15 @@ public TypedMessageBuilder value(T value) { checkArgument(value != null, "Need Non-Null content value"); if (schema.getSchemaInfo() != null && schema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { -// KeyValueSchema kvSchema = (KeyValueSchema) schema; + KeyValueSchema kvSchema = (KeyValueSchema) schema; org.apache.pulsar.common.schema.KeyValue kv = (org.apache.pulsar.common.schema.KeyValue) value; if (((KeyValueSchema) schema).getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) { // set key as the message key msgMetadataBuilder.setPartitionKey( - Base64.getEncoder().encodeToString(((KeyValueSchema) schema).getKeySchema().encode(kv.getKey()))); + Base64.getEncoder().encodeToString(kvSchema.getKeySchema().encode(kv.getKey()))); msgMetadataBuilder.setPartitionKeyB64Encoded(true); // set value as the payload - this.content = ByteBuffer.wrap(((KeyValueSchema) schema).getValueSchema().encode(kv.getValue())); + this.content = ByteBuffer.wrap((kvSchema.getValueSchema().encode(kv.getValue()))); return this; } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java index fa5ebfa8de5f4..b81a94706bbc8 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/KeyValueSchema.java @@ -20,7 +20,6 @@ import static com.google.common.base.Preconditions.checkArgument; -import java.io.Serializable; import java.util.concurrent.CompletableFuture; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -36,7 +35,7 @@ * [Key, Value] pair schema definition */ @Slf4j -public class KeyValueSchema implements Schema>, Serializable { +public class KeyValueSchema implements Schema> { @Getter private final Schema keySchema; diff --git a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/KVRecord.java b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/KVRecord.java new file mode 100644 index 0000000000000..8767391279c3e --- /dev/null +++ b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/KVRecord.java @@ -0,0 +1,11 @@ +package org.apache.pulsar.functions.api; + +import org.apache.pulsar.client.api.Schema; + +public interface KVRecord extends Record { + + Schema getKeySchema(); + + Schema getValueSchema(); + +} diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java index 8ee8cb1e013cf..c56604f1963ad 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java @@ -32,8 +32,10 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.schema.KeyValueSchema; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; +import org.apache.pulsar.functions.api.KVRecord; import org.apache.pulsar.functions.api.Record; @Slf4j @@ -95,15 +97,26 @@ public Optional getDestinationTopic() { @Override public Schema getSchema() { - if (sourceRecord == null || sourceRecord.getSchema() == null) { + if (sourceRecord == null) { return null; } + if (sourceRecord.getSchema() != null) { + return sourceRecord.getSchema(); + } + log.info("[SinkRecord] Schema classLoader: {}", Schema.class.getClassLoader()); - log.info("[SinkRecord] sourceRecord schema: {}, classLoader: {}", - sourceRecord.getSchema().getSchemaInfo().toString(), Schema.class.getClassLoader()); - return sourceRecord.getSchema(); + if (sourceRecord instanceof KVRecord) { + log.info("[SinkRecord] sourceRecord keySchema classLoader: {}, valueSchema classLoader: {}", + sourceRecord.getSchema().getSchemaInfo().toString(), Schema.class.getClassLoader()); + Schema keySchema= ((KVRecord) sourceRecord).getKeySchema(); + Schema valueSchema = ((KVRecord) sourceRecord).getValueSchema(); + return KeyValueSchema.of(keySchema, valueSchema); + } + + return null; + // ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); // try { // ObjectOutputStream oos = new ObjectOutputStream(byteArrayOutputStream); diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index 77df975489a52..54b98e1106ec0 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -41,6 +41,7 @@ import org.apache.kafka.connect.json.JsonConverter; import org.apache.kafka.connect.json.JsonConverterConfig; import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.apache.pulsar.functions.api.KVRecord; import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter; import org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroData; import lombok.Getter; @@ -217,7 +218,7 @@ private synchronized Record> processSourceRecord(final private static Optional RECORD_SEQUENCE = Optional.empty(); private static long FLUSH_TIMEOUT_MS = 2000; - private class KafkaSourceRecord implements Record> { + private class KafkaSourceRecord implements KVRecord { @Getter Optional key; @Getter @@ -277,6 +278,24 @@ private class KafkaSourceRecord implements Record> { this.destinationTopic = Optional.of(topicNamespace + "/" + srcRecord.topic()); } + @Override + public Schema getKeySchema() { + if (jsonWithEnvelope) { + return Schema.BYTES; + } else { + return keySchema; + } + } + + @Override + public Schema getValueSchema() { + if (jsonWithEnvelope) { + return Schema.BYTES; + } else { + return valueSchema; + } + } + @Override public Schema getSchema() { // When use `org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter` @@ -284,11 +303,12 @@ public Schema getSchema() { // use the `KeyValueEncodingType.SEPARATED`, then the pulsar client could get the original // byte array which are converted by the AvroConverter, or consume the GenericRecord object. - if (jsonWithEnvelope) { - return KeyValueSchema.kvBytes(); - } else { - return KeyValueSchema.of(keySchema, valueSchema, KeyValueEncodingType.SEPARATED); - } + return null; +// if (jsonWithEnvelope) { +// return KeyValueSchema.kvBytes(); +// } else { +// return KeyValueSchema.of(keySchema, valueSchema, KeyValueEncodingType.SEPARATED); +// } // try { // log.info("key classLoader: {}", keySchema.getClass().getClassLoader()); From 56b4d274d4665caef4c960a937c74ab655b59ad3 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Wed, 22 Apr 2020 16:50:22 +0800 Subject: [PATCH 18/36] fix --- .../java/org/apache/pulsar/DebeziumTest.java | 18 ++++++++++++++++ .../apache/pulsar/functions/api/KVRecord.java | 21 +++++++++++++++++++ .../pulsar/functions/instance/SinkRecord.java | 3 ++- .../pulsar/io/kafka/connect/SerTest.java | 18 ++++++++++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java b/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java index 2ce78d56287a9..d6d8c86cbcc8d 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.pulsar; import org.apache.pulsar.client.api.Consumer; diff --git a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/KVRecord.java b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/KVRecord.java index 8767391279c3e..a3a378cdabf86 100644 --- a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/KVRecord.java +++ b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/KVRecord.java @@ -1,7 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.pulsar.functions.api; import org.apache.pulsar.client.api.Schema; +/** + * key value schema record. + */ public interface KVRecord extends Record { Schema getKeySchema(); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java index c56604f1963ad..0de29257a7c93 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java @@ -109,7 +109,8 @@ public Schema getSchema() { if (sourceRecord instanceof KVRecord) { log.info("[SinkRecord] sourceRecord keySchema classLoader: {}, valueSchema classLoader: {}", - sourceRecord.getSchema().getSchemaInfo().toString(), Schema.class.getClassLoader()); + ((KVRecord) sourceRecord).getKeySchema().getClass().getClassLoader(), + ((KVRecord) sourceRecord).getValueSchema().getClass().getClassLoader()); Schema keySchema= ((KVRecord) sourceRecord).getKeySchema(); Schema valueSchema = ((KVRecord) sourceRecord).getValueSchema(); return KeyValueSchema.of(keySchema, valueSchema); diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SerTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SerTest.java index e4f70fd3fe59e..03d2142007862 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SerTest.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SerTest.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.pulsar.io.kafka.connect; import lombok.extern.slf4j.Slf4j; From 54d636e2605cb5d7ae0697b234bd0512f0a2c08d Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Wed, 22 Apr 2020 18:41:34 +0800 Subject: [PATCH 19/36] add test log --- .../java/org/apache/pulsar/DebeziumTest.java | 2 +- .../apache/pulsar/functions/api/KVRecord.java | 3 +++ .../pulsar/functions/instance/SinkRecord.java | 23 ++++++++----------- .../pulsar/functions/sink/PulsarSink.java | 1 + .../io/kafka/connect/KafkaConnectSource.java | 9 ++++++++ .../containers/ChaosContainer.java | 15 ++++++++++++ .../functions/PulsarFunctionsTest.java | 22 ++++++++++-------- 7 files changed, 51 insertions(+), 24 deletions(-) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java b/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java index d6d8c86cbcc8d..e4c382899cdbc 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java @@ -38,7 +38,7 @@ private void testJsonConverterBytes() throws PulsarClientException { PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); Schema> schema = - Schema.KeyValue(Schema.BYTES, Schema.BYTES, KeyValueEncodingType.SEPARATED); + Schema.KeyValue(Schema.BYTES, Schema.BYTES, KeyValueEncodingType.INLINE); Consumer> consumer = pulsarClient.newConsumer(schema) .topic("public/default/dbserver1.inventory.products") diff --git a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/KVRecord.java b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/KVRecord.java index a3a378cdabf86..812e8c698ec26 100644 --- a/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/KVRecord.java +++ b/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/KVRecord.java @@ -19,6 +19,7 @@ package org.apache.pulsar.functions.api; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.schema.KeyValueEncodingType; /** * key value schema record. @@ -29,4 +30,6 @@ public interface KVRecord extends Record { Schema getValueSchema(); + KeyValueEncodingType getKeyValueEncodingType(); + } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java index 0de29257a7c93..5504337007c8a 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java @@ -18,12 +18,6 @@ */ package org.apache.pulsar.functions.instance; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; import java.util.Map; import java.util.Optional; @@ -33,8 +27,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.schema.KeyValueSchema; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.functions.api.KVRecord; import org.apache.pulsar.functions.api.Record; @@ -106,14 +98,17 @@ public Schema getSchema() { } log.info("[SinkRecord] Schema classLoader: {}", Schema.class.getClassLoader()); + log.info("[SinkRecord] sourceRecord classLoader: {}", sourceRecord.getClass().getClassLoader()); + log.info("[SinkRecord] KVRecord classLoader: {}", KVRecord.class.getClassLoader()); if (sourceRecord instanceof KVRecord) { - log.info("[SinkRecord] sourceRecord keySchema classLoader: {}, valueSchema classLoader: {}", - ((KVRecord) sourceRecord).getKeySchema().getClass().getClassLoader(), - ((KVRecord) sourceRecord).getValueSchema().getClass().getClassLoader()); - Schema keySchema= ((KVRecord) sourceRecord).getKeySchema(); - Schema valueSchema = ((KVRecord) sourceRecord).getValueSchema(); - return KeyValueSchema.of(keySchema, valueSchema); + KVRecord kvRecord = (KVRecord) sourceRecord; + log.info("[SinkRecord] keySchema classLoader: {}, schemaInfo: {}", + kvRecord.getKeySchema().getClass().getClassLoader(), kvRecord.getKeySchema().getSchemaInfo().toString()); + log.info("[SinkRecord] valueSchema classLoader: {}, schemaInfo: {}", + kvRecord.getValueSchema().getClass().getClassLoader(), kvRecord.getValueSchema().getSchemaInfo().toString()); + return KeyValueSchema.of(kvRecord.getKeySchema(), kvRecord.getValueSchema(), + kvRecord.getKeyValueEncodingType()); } return null; diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java index 7b75232b7dac6..6ffebbac13bb5 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java @@ -87,6 +87,7 @@ protected PulsarSinkProcessorBase(Schema schema) { public Producer createProducer(PulsarClient client, String topic, String producerName, Schema schema) throws PulsarClientException { + log.info("[createProducer] schema: {}", schema == null ? "null" : schema.getSchemaInfo()); ProducerBuilder builder = client.newProducer(schema) .blockIfQueueFull(true) .enableBatching(true) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index 54b98e1106ec0..e9ea273366a0a 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -296,6 +296,15 @@ public Schema getValueSchema() { } } + @Override + public KeyValueEncodingType getKeyValueEncodingType() { + if (jsonWithEnvelope) { + return KeyValueEncodingType.INLINE; + } else { + return KeyValueEncodingType.SEPARATED; + } + } + @Override public Schema getSchema() { // When use `org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter` diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java index 3a0568f296d06..88ff072496ca0 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java @@ -146,6 +146,7 @@ public void start() { if (this.getContainerName().contains("functions-worker")) { DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), "tail", "-f", "/var/log/pulsar/functions_worker.log"); + DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), "mkdir", "-p", "/tmp/functions/public/default/test-source-connector-PROCESS-name-mysql"); @@ -157,6 +158,20 @@ public void start() { "tail", "-f", "/tmp/functions/public/default/test-source-connector-PROCESS-name-mysql/" + "test-source-connector-PROCESS-name-mysql-0.log"); + + // postgresql + DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), + "mkdir", "-p", + "/tmp/functions/public/default/test-source-connector-PROCESS-name-postgresql"); + DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), + "touch", + "/tmp/functions/public/default/test-source-connector-PROCESS-name-postgresql/" + + "test-source-connector-PROCESS-name-postgresql-0.log"); + DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), + "tail", "-f", + "/tmp/functions/public/default/test-source-connector-PROCESS-name-postgresql/" + + "test-source-connector-PROCESS-name-postgresql-0.log"); + } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 5470ba6856b4e..126948f148b6f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -33,6 +33,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.schema.AvroSchema; @@ -2307,16 +2308,10 @@ private void testDebeziumMySqlConnect(String converterClassName) throws Exceptio SchemaInfo lastSchemaInfo = admin.schemas().getSchemaInfo(consumeTopicName); log.info("lastSchemaInfo: {}", lastSchemaInfo == null ? "null" : lastSchemaInfo.toString()); } catch (Exception e) { - log.warn("failed to get schemaInfo for topic: {}", consumeTopicName); + log.warn("failed to get schemaInfo for topic: {}, exceptions message: {}", + consumeTopicName, e.getMessage()); } - @Cleanup - Consumer consumer = client.newConsumer(getSchema(converterClassName)) - .topic(consumeTopicName) - .subscriptionName("debezium-source-tester") - .subscriptionType(SubscriptionType.Exclusive) - .subscribe(); - // admin.topics().createNonPartitionedTopic(consumeTopicName); admin.topics().createNonPartitionedTopic(outputTopicName); @@ -2343,6 +2338,14 @@ private void testDebeziumMySqlConnect(String converterClassName) throws Exceptio Failsafe.with(statusRetryPolicy).run(() -> waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); + @Cleanup + Consumer consumer = client.newConsumer(getSchema(converterClassName)) + .topic(consumeTopicName) + .subscriptionName("debezium-source-tester") + .subscriptionType(SubscriptionType.Exclusive) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + // validate the source result sourceTester.validateSourceResult(consumer, 9, null, converterClassName); @@ -2378,7 +2381,8 @@ private void testDebeziumPostgreSqlConnect(String converterClassName) throws Ex final String outputTopicName = "debe-output-topic-name"; final String consumeTopicName = "public/default/dbserver1.inventory.products"; final String sourceName = "test-source-connector-" - + functionRuntimeType + "-name-" + randomName(8); +// + functionRuntimeType + "-name-" + randomName(8); + + functionRuntimeType + "-name-postgresql"; // This is the binlog count that contained in postgresql container. final int numMessages = 26; From 980ecb8ed3d072c4b89cd5b0ae5e6cfa8d97bbd8 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Thu, 23 Apr 2020 00:21:35 +0800 Subject: [PATCH 20/36] fix test --- .../functions/PulsarFunctionsTest.java | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 126948f148b6f..cd2e4c6874009 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -45,6 +45,7 @@ import org.apache.pulsar.common.policies.data.SourceStatus; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.functions.api.examples.AutoSchemaFunction; import org.apache.pulsar.functions.api.examples.AvroSchemaTestFunction; @@ -142,23 +143,23 @@ public void testRabbitMQSink() throws Exception { @Test(groups = "source") public void testDebeziumMySqlSourceJson() throws Exception { - testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter"); + testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter", true); } @Test(groups = "source") public void testDebeziumMySqlSourceAvro() throws Exception { testDebeziumMySqlConnect( - "org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter"); + "org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter", false); } @Test(groups = "source") public void testDebeziumPostgreSqlSource() throws Exception { - testDebeziumPostgreSqlConnect("org.apache.kafka.connect.json.JsonConverter"); + testDebeziumPostgreSqlConnect("org.apache.kafka.connect.json.JsonConverter", true); } @Test(groups = "source") public void testDebeziumMongoDbSource() throws Exception{ - testDebeziumMongoDbConnect("org.apache.kafka.connect.json.JsonConverter"); + testDebeziumMongoDbConnect("org.apache.kafka.connect.json.JsonConverter", true); } private void testSink(SinkTester tester, boolean builtin) throws Exception { @@ -2269,7 +2270,7 @@ public void testAvroSchemaFunction() throws Exception { getFunctionInfoNotFound(functionName); } - private void testDebeziumMySqlConnect(String converterClassName) throws Exception { + private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { final String tenant = TopicName.PUBLIC_TENANT; final String namespace = TopicName.DEFAULT_NAMESPACE; @@ -2317,6 +2318,7 @@ private void testDebeziumMySqlConnect(String converterClassName) throws Exceptio @Cleanup DebeziumMySqlSourceTester sourceTester = new DebeziumMySqlSourceTester(pulsarCluster, converterClassName); + sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); // setup debezium mysql server DebeziumMySQLContainer mySQLContainer = new DebeziumMySQLContainer(pulsarCluster.getClusterName()); @@ -2339,7 +2341,7 @@ private void testDebeziumMySqlConnect(String converterClassName) throws Exceptio waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); @Cleanup - Consumer consumer = client.newConsumer(getSchema(converterClassName)) + Consumer consumer = client.newConsumer(getSchema(jsonWithEnvelope)) .topic(consumeTopicName) .subscriptionName("debezium-source-tester") .subscriptionType(SubscriptionType.Exclusive) @@ -2374,7 +2376,7 @@ private void testDebeziumMySqlConnect(String converterClassName) throws Exceptio getSourceInfoNotFound(tenant, namespace, sourceName); } - private void testDebeziumPostgreSqlConnect(String converterClassName) throws Exception { + private void testDebeziumPostgreSqlConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { final String tenant = TopicName.PUBLIC_TENANT; final String namespace = TopicName.DEFAULT_NAMESPACE; @@ -2413,7 +2415,7 @@ private void testDebeziumPostgreSqlConnect(String converterClassName) throws Ex admin.topics().createNonPartitionedTopic(outputTopicName); @Cleanup - Consumer consumer = client.newConsumer(getSchema(converterClassName)) + Consumer consumer = client.newConsumer(getSchema(jsonWithEnvelope)) .topic(consumeTopicName) .subscriptionName("debezium-source-tester") .subscriptionType(SubscriptionType.Exclusive) @@ -2421,6 +2423,7 @@ private void testDebeziumPostgreSqlConnect(String converterClassName) throws Ex @Cleanup DebeziumPostgreSqlSourceTester sourceTester = new DebeziumPostgreSqlSourceTester(pulsarCluster); + sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); // setup debezium postgresql server DebeziumPostgreSqlContainer postgreSqlContainer = new DebeziumPostgreSqlContainer(pulsarCluster.getClusterName()); @@ -2470,7 +2473,7 @@ private void testDebeziumPostgreSqlConnect(String converterClassName) throws Ex getSourceInfoNotFound(tenant, namespace, sourceName); } - private void testDebeziumMongoDbConnect(String converterClassName) throws Exception { + private void testDebeziumMongoDbConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { final String tenant = TopicName.PUBLIC_TENANT; final String namespace = TopicName.DEFAULT_NAMESPACE; @@ -2508,7 +2511,7 @@ private void testDebeziumMongoDbConnect(String converterClassName) throws Excep admin.topics().createNonPartitionedTopic(outputTopicName); @Cleanup - Consumer consumer = client.newConsumer(getSchema(converterClassName)) + Consumer consumer = client.newConsumer(getSchema(jsonWithEnvelope)) .topic(consumeTopicName) .subscriptionName("debezium-source-tester") .subscriptionType(SubscriptionType.Exclusive) @@ -2516,6 +2519,7 @@ private void testDebeziumMongoDbConnect(String converterClassName) throws Excep @Cleanup DebeziumMongoDbSourceTester sourceTester = new DebeziumMongoDbSourceTester(pulsarCluster); + sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); // setup debezium mongodb server DebeziumMongoDbContainer mongoDbContainer = new DebeziumMongoDbContainer(pulsarCluster.getClusterName()); @@ -2564,13 +2568,12 @@ private void testDebeziumMongoDbConnect(String converterClassName) throws Excep getSourceInfoNotFound(tenant, namespace, sourceName); } - private Schema getSchema(String converterClassName) { - if (converterClassName.endsWith("AvroConverter")) { - return KeyValueSchema.of(Schema.AUTO_CONSUME(), Schema.AUTO_CONSUME()); - } else { + private Schema getSchema(boolean jsonWithEnvelope) { + if (jsonWithEnvelope) { return KeyValueSchema.kvBytes(); + } else { + return KeyValueSchema.of(Schema.AUTO_CONSUME(), Schema.AUTO_CONSUME(), KeyValueEncodingType.SEPARATED); } - } } From c09e9b426690601f5884ffa7e4c9620af5986ca3 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Thu, 23 Apr 2020 02:19:12 +0800 Subject: [PATCH 21/36] add test log --- .../integration/containers/ChaosContainer.java | 16 ++++++---------- .../functions/PulsarFunctionsTest.java | 10 ++++++---- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java index 88ff072496ca0..874bd0b1c8616 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java @@ -149,28 +149,24 @@ public void start() { DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), "mkdir", "-p", - "/tmp/functions/public/default/test-source-connector-PROCESS-name-mysql"); + "/tmp/functions/public/default/test-source-debezium-mysql"); DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), "touch", - "/tmp/functions/public/default/test-source-connector-PROCESS-name-mysql/" + - "test-source-connector-PROCESS-name-mysql-0.log"); + "/tmp/functions/public/default/test-source-debezium-mysql/test-source-debezium-mysql-0.log"); DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), "tail", "-f", - "/tmp/functions/public/default/test-source-connector-PROCESS-name-mysql/" + - "test-source-connector-PROCESS-name-mysql-0.log"); + "/tmp/functions/public/default/test-source-debezium-mysql/test-source-debezium-mysql-0.log"); // postgresql DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), "mkdir", "-p", - "/tmp/functions/public/default/test-source-connector-PROCESS-name-postgresql"); + "/tmp/functions/public/default/test-source-debezium-postgersql"); DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), "touch", - "/tmp/functions/public/default/test-source-connector-PROCESS-name-postgresql/" + - "test-source-connector-PROCESS-name-postgresql-0.log"); + "/tmp/functions/public/default/test-source-debezium-postgersql/test-source-debezium-postgersql-0.log"); DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), "tail", "-f", - "/tmp/functions/public/default/test-source-connector-PROCESS-name-postgresql/" + - "test-source-connector-PROCESS-name-postgresql-0.log"); + "/tmp/functions/public/default/test-source-debezium-postgersql/test-source-debezium-postgersql-0.log"); } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index cd2e4c6874009..722db4e73da2a 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -2276,8 +2276,9 @@ private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWit final String namespace = TopicName.DEFAULT_NAMESPACE; final String outputTopicName = "debe-output-topic-name"; final String consumeTopicName = "public/default/dbserver1.inventory.products"; - final String sourceName = "test-source-connector-" - + functionRuntimeType + "-name-mysql"; +// final String sourceName = "test-source-connector-" +// + functionRuntimeType + "-name-mysql"; + final String sourceName = "test-source-debezium-mysql"; // This is the binlog count that contained in mysql container. final int numMessages = 47; @@ -2382,9 +2383,10 @@ private void testDebeziumPostgreSqlConnect(String converterClassName, boolean j final String namespace = TopicName.DEFAULT_NAMESPACE; final String outputTopicName = "debe-output-topic-name"; final String consumeTopicName = "public/default/dbserver1.inventory.products"; - final String sourceName = "test-source-connector-" +// final String sourceName = "test-source-connector-" // + functionRuntimeType + "-name-" + randomName(8); - + functionRuntimeType + "-name-postgresql"; + final String sourceName = "test-source-debezium-postgersql"; + // This is the binlog count that contained in postgresql container. final int numMessages = 26; From 406d3af20e65d660f7518b9f99c3dc02afc6e90a Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Thu, 23 Apr 2020 10:43:40 +0800 Subject: [PATCH 22/36] add test log --- .../pulsar/client/impl/TypedMessageBuilderImpl.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java index e6db901120009..38ac60e9a0b9d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java @@ -119,11 +119,14 @@ public TypedMessageBuilder key(String key) { log.info("[key] schema encodingType, classLoader: {}", ((KeyValueSchema) schema).getKeyValueEncodingType().getClass().getClassLoader()); KeyValueSchema kvSchema = (KeyValueSchema) schema; + log.info("[value] kvSchema class: {}, schemaInfo: {}", + kvSchema.getClass().getName(), kvSchema.getSchemaInfo().toString()); checkArgument(!(kvSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED), "This method is not allowed to set keys when in encoding type is SEPARATED"); } msgMetadataBuilder.setPartitionKey(key); msgMetadataBuilder.setPartitionKeyB64Encoded(false); + log.info("[key] success encode"); return this; } @@ -150,7 +153,12 @@ public TypedMessageBuilder value(T value) { checkArgument(value != null, "Need Non-Null content value"); if (schema.getSchemaInfo() != null && schema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { + log.info("[value] instanceof: {}", schema instanceof KeyValueSchema); + log.info("[value] KeyValueScehma classLoader: {}", KeyValueSchema.class.getClassLoader()); + log.info("[value] schema: {}", schema.getClass().getClassLoader()); KeyValueSchema kvSchema = (KeyValueSchema) schema; + log.info("[value] kvSchema class: {}, schemaInfo: {}", + kvSchema.getClass().getName(), kvSchema.getSchemaInfo().toString()); org.apache.pulsar.common.schema.KeyValue kv = (org.apache.pulsar.common.schema.KeyValue) value; if (((KeyValueSchema) schema).getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) { // set key as the message key @@ -159,10 +167,12 @@ public TypedMessageBuilder value(T value) { msgMetadataBuilder.setPartitionKeyB64Encoded(true); // set value as the payload this.content = ByteBuffer.wrap((kvSchema.getValueSchema().encode(kv.getValue()))); + log.info("[value] success encode1"); return this; } } this.content = ByteBuffer.wrap(schema.encode(value)); + log.info("[value] success encode2"); return this; } From 0057cbb3cebdc8dc5978b3737ee76fb990fd7670 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Thu, 23 Apr 2020 22:16:19 +0800 Subject: [PATCH 23/36] add test log --- .../pulsar/io/kafka/connect/KafkaConnectSource.java | 3 ++- .../tests/integration/containers/ChaosContainer.java | 8 +++++++- .../tests/integration/functions/PulsarFunctionsTest.java | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index e9ea273366a0a..52ab4253e9ed0 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -401,7 +401,8 @@ public void ack() { @Override public void fail() { if (flushFuture != null) { - flushFuture.completeExceptionally(new Exception("Sink Error")); + log.info("[fail] isCompletedExceptionally: {}", flushFuture.isCompletedExceptionally()); +// flushFuture.completeExceptionally(new Exception("Sink Error")); } } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java index 874bd0b1c8616..117e8bf04baf6 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java @@ -143,7 +143,13 @@ public int hashCode() { public void start() { super.start(); this.tailContainerLog(); - if (this.getContainerName().contains("functions-worker")) { + if (this.getContainerName().contains("pulsar-broker")) { + DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), + "tail", "-f", "/var/log/pulsar/broker.log"); + } else if (this.getContainerName().contains("bookie")) { + DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), + "tail", "-f", "/var/log/pulsar/bookie.log"); + } else if (this.getContainerName().contains("functions-worker")) { DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), "tail", "-f", "/var/log/pulsar/functions_worker.log"); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 722db4e73da2a..ea8224cefaa85 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -2348,6 +2348,7 @@ private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWit .subscriptionType(SubscriptionType.Exclusive) .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscribe(); + log.info("[debezium mysql test] create consumer finish. converterName: {}", converterClassName); // validate the source result sourceTester.validateSourceResult(consumer, 9, null, converterClassName); From 1e19a9aa78a54e87ef7a6ecc52d5f07b9633b9a3 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Thu, 23 Apr 2020 22:38:08 +0800 Subject: [PATCH 24/36] fix docs --- site2/docs/io-debezium-source.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/site2/docs/io-debezium-source.md b/site2/docs/io-debezium-source.md index bfa282ca1dbc3..360a5bf81496f 100644 --- a/site2/docs/io-debezium-source.md +++ b/site2/docs/io-debezium-source.md @@ -28,6 +28,24 @@ The configuration of Debezium source connector has the following properties. | `database.history.pulsar.service.url` | true | null | Pulsar cluster service URL for history topic. | | `pulsar.service.url` | true | null | Pulsar cluster service URL. | | `offset.storage.topic` | true | null | Record the last committed offsets that the connector successfully completes. | +| `json-with-envelope` | false | false | Present the message only consist of payload. + +### Converter Options + +1. org.apache.kafka.connect.json.JsonConverter + +This config `json-with-envelope` is valid only for the JsonConverter. It's default value is false, the consumer use the schema ` +Schema.KeyValue(Schema.AUTO_CONSUME(), Schema.AUTO_CONSUME(), KeyValueEncodingType.SEPARATED)`, +and the message only consist of payload. + +If the config `json-with-envelope` value is true, the consumer use the schema +`Schema.KeyValue(Schema.BYTES, Schema.BYTES`, the message consist of schema and payload. + +2. org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter + +If users select the AvroConverter, then the pulsar consumer should use the schema `Schema.KeyValue(Schema.AUTO_CONSUME(), +Schema.AUTO_CONSUME(), KeyValueEncodingType.SEPARATED)`, and the message consist of payload. + ### MongoDB Configuration | Name | Required | Default | Description | |------|----------|---------|-------------| From 39f9f7d07c6ac10829ce12233d1e3d00e44d69f1 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Sun, 26 Apr 2020 01:01:07 +0800 Subject: [PATCH 25/36] fix test --- .../tests/integration/containers/ChaosContainer.java | 12 ------------ .../integration/functions/PulsarFunctionsTest.java | 4 ++++ .../pulsar/tests/integration/io/SourceTester.java | 12 ++++++------ 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java index 117e8bf04baf6..3fedea2d417d2 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java @@ -162,18 +162,6 @@ public void start() { DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), "tail", "-f", "/tmp/functions/public/default/test-source-debezium-mysql/test-source-debezium-mysql-0.log"); - - // postgresql - DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), - "mkdir", "-p", - "/tmp/functions/public/default/test-source-debezium-postgersql"); - DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), - "touch", - "/tmp/functions/public/default/test-source-debezium-postgersql/test-source-debezium-postgersql-0.log"); - DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), - "tail", "-f", - "/tmp/functions/public/default/test-source-debezium-postgersql/test-source-debezium-postgersql-0.log"); - } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index ea8224cefaa85..d9c8a6f727fbf 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -143,22 +143,26 @@ public void testRabbitMQSink() throws Exception { @Test(groups = "source") public void testDebeziumMySqlSourceJson() throws Exception { + Thread.sleep(1000 * 30); testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter", true); } @Test(groups = "source") public void testDebeziumMySqlSourceAvro() throws Exception { + Thread.sleep(1000 * 30); testDebeziumMySqlConnect( "org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter", false); } @Test(groups = "source") public void testDebeziumPostgreSqlSource() throws Exception { + Thread.sleep(1000 * 30); testDebeziumPostgreSqlConnect("org.apache.kafka.connect.json.JsonConverter", true); } @Test(groups = "source") public void testDebeziumMongoDbSource() throws Exception{ + Thread.sleep(1000 * 30); testDebeziumMongoDbConnect("org.apache.kafka.connect.json.JsonConverter", true); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SourceTester.java index 997c9385a5ee6..27037ab302082 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SourceTester.java @@ -103,7 +103,7 @@ public void validateSourceResultJson(Consumer> consumer Assert.assertTrue(key.contains(this.keyContains())); Assert.assertTrue(value.contains(this.valueContains())); if (eventType != null) { - Assert.assertTrue(value.contains(this.eventContains(eventType))); + Assert.assertTrue(value.contains(this.eventContains(eventType, true))); } consumer.acknowledge(msg); msg = consumer.receive(1, TimeUnit.SECONDS); @@ -132,7 +132,7 @@ public void validateSourceResultAvro(Consumer Date: Mon, 27 Apr 2020 11:42:28 +0800 Subject: [PATCH 26/36] fix test --- .../io/kafka/connect/KafkaConnectSource.java | 56 ++----------------- 1 file changed, 4 insertions(+), 52 deletions(-) diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index 52ab4253e9ed0..fbc2603726627 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -20,7 +20,6 @@ import static org.apache.pulsar.io.kafka.connect.PulsarKafkaWorkerConfig.TOPIC_NAMESPACE_CONFIG; -import java.lang.reflect.Method; import java.util.Base64; import java.util.Collections; import java.util.HashMap; @@ -38,7 +37,6 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import org.apache.kafka.connect.json.JsonConverter; import org.apache.kafka.connect.json.JsonConverterConfig; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.functions.api.KVRecord; @@ -56,7 +54,6 @@ import org.apache.kafka.connect.storage.OffsetStorageReaderImpl; import org.apache.kafka.connect.storage.OffsetStorageWriter; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.schema.KeyValueSchema; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.io.core.Source; @@ -96,13 +93,8 @@ public class KafkaConnectSource implements Source> { CacheBuilder.newBuilder().maximumSize(10000) .expireAfterAccess(30, TimeUnit.MINUTES).build(); - private Method keyValueSchemaOfMethod; - @Override public void open(Map config, SourceContext sourceContext) throws Exception { - log.info("sourceContext classLoader: {}", sourceContext.getClass().getClassLoader()); -// initKeyValueSchemaOfMethod(sourceContext.getClass().getClassLoader()); - Map stringConfig = new HashMap<>(); config.forEach((key, value) -> { if (value instanceof String) { @@ -188,6 +180,7 @@ public synchronized Record> read() throws Exception { Record> processRecord = processSourceRecord(currentBatch.next()); if (processRecord.getValue().getValue() == null) { outstandingRecords.decrementAndGet(); + continue; } else { return processRecord; } @@ -240,8 +233,7 @@ private class KafkaSourceRecord implements KVRecord { AvroData avroData = new AvroData(1000); byte[] keyBytes = keyConverter.fromConnectData( srcRecord.topic(), srcRecord.keySchema(), srcRecord.key()); - this.key = keyBytes != null ? Optional.of( - Base64.getEncoder().encodeToString(keyBytes)) : Optional.empty(); + this.key = keyBytes != null ? Optional.of(Base64.getEncoder().encodeToString(keyBytes)) : Optional.empty(); byte[] valueBytes = valueConverter.fromConnectData( srcRecord.topic(), srcRecord.valueSchema(), srcRecord.value()); @@ -280,7 +272,7 @@ private class KafkaSourceRecord implements KVRecord { @Override public Schema getKeySchema() { - if (jsonWithEnvelope) { + if (jsonWithEnvelope || keySchema == null) { return Schema.BYTES; } else { return keySchema; @@ -289,7 +281,7 @@ public Schema getKeySchema() { @Override public Schema getValueSchema() { - if (jsonWithEnvelope) { + if (jsonWithEnvelope || valueSchema == null) { return Schema.BYTES; } else { return valueSchema; @@ -307,33 +299,7 @@ public KeyValueEncodingType getKeyValueEncodingType() { @Override public Schema getSchema() { - // When use `org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter` - // as the key.converter and value.converter, make the `KeyValueSchema` encodingType - // use the `KeyValueEncodingType.SEPARATED`, then the pulsar client could get the original - // byte array which are converted by the AvroConverter, or consume the GenericRecord object. - return null; -// if (jsonWithEnvelope) { -// return KeyValueSchema.kvBytes(); -// } else { -// return KeyValueSchema.of(keySchema, valueSchema, KeyValueEncodingType.SEPARATED); -// } - -// try { -// log.info("key classLoader: {}", keySchema.getClass().getClassLoader()); -// log.info("value classLoader: {}", valueSchema.getClass().getClassLoader()); -// if (jsonWithEnvelope) { -// return (Schema) keyValueSchemaOfMethod.invoke( -// Schema.BYTES, Schema.BYTES, KeyValueEncodingType.INLINE); -// } else { -// return (Schema) keyValueSchemaOfMethod.invoke( -// keySchema, valueSchema, KeyValueEncodingType.SEPARATED); -// } -// } catch (Exception e) { -// e.printStackTrace(); -// log.error("failed to invoke the keyValueSchemaOfMethod."); -// return null; -// } } @Override @@ -407,18 +373,4 @@ public void fail() { } } - private void initKeyValueSchemaOfMethod(ClassLoader classLoader) throws ClassNotFoundException, - NoSuchMethodException { - Class keyValueSchemaClazz = - (Class) classLoader.loadClass(KeyValueSchema.class.getName()); - log.info("keyValueSchemaClazz: {}, classLoader: {}", keyValueSchemaClazz.getName(), keyValueSchemaClazz.getClassLoader()); - keyValueSchemaOfMethod = keyValueSchemaClazz.getDeclaredMethod( - "of", Schema.class, Schema.class, KeyValueEncodingType.class); - keyValueSchemaOfMethod.setAccessible(true); - log.info("keyValueSchemaOfMethod: {}", keyValueSchemaOfMethod.toString()); - Class[] clazzArr = keyValueSchemaOfMethod.getParameterTypes(); - for (Class paramClass : clazzArr) { - log.info("paramClass: {}", paramClass.getName()); - } - } } From 4e077b5682970e9b04aa9b9ea88af6588743219e Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Mon, 27 Apr 2020 11:43:35 +0800 Subject: [PATCH 27/36] fix test --- .../pulsar/io/kafka/connect/SerTest.java | 60 ------------------- 1 file changed, 60 deletions(-) delete mode 100644 pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SerTest.java diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SerTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SerTest.java deleted file mode 100644 index 03d2142007862..0000000000000 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/SerTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.io.kafka.connect; - -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.schema.KeyValueSchema; -import org.apache.pulsar.common.schema.KeyValueEncodingType; -import org.apache.pulsar.io.kafka.connect.schema.KafkaSchemaWrappedSchema; -import org.testng.annotations.Test; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; - -@Slf4j -public class SerTest { - - @Test - public void test() { - - Schema schema = KeyValueSchema.of(new KafkaSchemaWrappedSchema(null, null), - new KafkaSchemaWrappedSchema(null, null), KeyValueEncodingType.SEPARATED); - - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - try { - ObjectOutputStream oos = new ObjectOutputStream(byteArrayOutputStream); - oos.writeObject(schema); - oos.flush(); - oos.close(); - - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArrayOutputStream.toByteArray()); - ObjectInputStream ois = new ObjectInputStream(byteArrayInputStream); - Schema schema2 = (Schema) ois.readObject(); - log.info("deserializable schema: {}, classLoader: {}", - schema.getClass().getName(), schema.getClass().getClassLoader()); - } catch (IOException | ClassNotFoundException e) { - e.printStackTrace(); - } - } - -} From 78ef864e92fc7c7d10799af489ad30897d6550d2 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Mon, 27 Apr 2020 14:23:04 +0800 Subject: [PATCH 28/36] fix test --- .../client/impl/TypedMessageBuilderImpl.java | 18 ------ .../pulsar/functions/instance/SinkRecord.java | 56 ++++--------------- .../pulsar/functions/sink/PulsarSink.java | 19 +++---- 3 files changed, 21 insertions(+), 72 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java index 38ac60e9a0b9d..460e03827f723 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java @@ -110,23 +110,12 @@ public CompletableFuture sendAsync() { @Override public TypedMessageBuilder key(String key) { if (schema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { - log.info("[key] KeyValueSchema className: {}, classLoader: {}", - KeyValueSchema.class.getName(), KeyValueSchema.class.getClassLoader()); - log.info("[key] schema className: {}, classLoader: {}", - schema.getClass().getName(), schema.getClass().getClassLoader()); - - log.info("[key] KeyValueSchema encodingType, classLoader: {}", KeyValueEncodingType.SEPARATED); - log.info("[key] schema encodingType, classLoader: {}", - ((KeyValueSchema) schema).getKeyValueEncodingType().getClass().getClassLoader()); KeyValueSchema kvSchema = (KeyValueSchema) schema; - log.info("[value] kvSchema class: {}, schemaInfo: {}", - kvSchema.getClass().getName(), kvSchema.getSchemaInfo().toString()); checkArgument(!(kvSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED), "This method is not allowed to set keys when in encoding type is SEPARATED"); } msgMetadataBuilder.setPartitionKey(key); msgMetadataBuilder.setPartitionKeyB64Encoded(false); - log.info("[key] success encode"); return this; } @@ -153,12 +142,7 @@ public TypedMessageBuilder value(T value) { checkArgument(value != null, "Need Non-Null content value"); if (schema.getSchemaInfo() != null && schema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { - log.info("[value] instanceof: {}", schema instanceof KeyValueSchema); - log.info("[value] KeyValueScehma classLoader: {}", KeyValueSchema.class.getClassLoader()); - log.info("[value] schema: {}", schema.getClass().getClassLoader()); KeyValueSchema kvSchema = (KeyValueSchema) schema; - log.info("[value] kvSchema class: {}, schemaInfo: {}", - kvSchema.getClass().getName(), kvSchema.getSchemaInfo().toString()); org.apache.pulsar.common.schema.KeyValue kv = (org.apache.pulsar.common.schema.KeyValue) value; if (((KeyValueSchema) schema).getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) { // set key as the message key @@ -167,12 +151,10 @@ public TypedMessageBuilder value(T value) { msgMetadataBuilder.setPartitionKeyB64Encoded(true); // set value as the payload this.content = ByteBuffer.wrap((kvSchema.getValueSchema().encode(kv.getValue()))); - log.info("[value] success encode1"); return this; } } this.content = ByteBuffer.wrap(schema.encode(value)); - log.info("[value] success encode2"); return this; } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java index 5504337007c8a..21fb94351c299 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java @@ -90,62 +90,30 @@ public Optional getDestinationTopic() { @Override public Schema getSchema() { if (sourceRecord == null) { + log.info("[SinkRecord] topic: {}, schema is null", sourceRecord.getDestinationTopic().isPresent() + ? sourceRecord.getDestinationTopic().get() : "null"); return null; } if (sourceRecord.getSchema() != null) { + log.info("[SinkRecord] topic: {}, Schema: {}", + sourceRecord.getDestinationTopic().isPresent() ? sourceRecord.getDestinationTopic().get() : "null", + sourceRecord.getSchema().getClass().getName()); return sourceRecord.getSchema(); } - log.info("[SinkRecord] Schema classLoader: {}", Schema.class.getClassLoader()); - log.info("[SinkRecord] sourceRecord classLoader: {}", sourceRecord.getClass().getClassLoader()); - log.info("[SinkRecord] KVRecord classLoader: {}", KVRecord.class.getClassLoader()); - if (sourceRecord instanceof KVRecord) { KVRecord kvRecord = (KVRecord) sourceRecord; - log.info("[SinkRecord] keySchema classLoader: {}, schemaInfo: {}", - kvRecord.getKeySchema().getClass().getClassLoader(), kvRecord.getKeySchema().getSchemaInfo().toString()); - log.info("[SinkRecord] valueSchema classLoader: {}, schemaInfo: {}", - kvRecord.getValueSchema().getClass().getClassLoader(), kvRecord.getValueSchema().getSchemaInfo().toString()); - return KeyValueSchema.of(kvRecord.getKeySchema(), kvRecord.getValueSchema(), + Schema schema = KeyValueSchema.of(kvRecord.getKeySchema(), kvRecord.getValueSchema(), kvRecord.getKeyValueEncodingType()); + log.info("[SinkRecord] topic: {}, Schema: {}", + sourceRecord.getDestinationTopic().isPresent() ? sourceRecord.getDestinationTopic().get() : "null", + schema.getClass().getName()); } + log.info("[SinkRecord] topic: {}, schema is null", sourceRecord.getDestinationTopic().isPresent() + ? sourceRecord.getDestinationTopic().get() : "null"); return null; - -// ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); -// try { -// ObjectOutputStream oos = new ObjectOutputStream(byteArrayOutputStream); -// oos.writeObject(sourceRecord.getSchema()); -// oos.flush(); -// oos.close(); -// -// ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArrayOutputStream.toByteArray()); -// ObjectInputStream ois = new ObjectInputStream(byteArrayInputStream); -// Schema schema = (Schema) ois.readObject(); -// log.info("deserializable schema: {}, classLoader: {}", -// schema.getClass().getName(), schema.getClass().getClassLoader()); -// return schema; -// } catch (IOException | ClassNotFoundException e) { -// e.printStackTrace(); -// return null; -// } - -// log.info("[SinkRecord] Schema classLoader: {}", Schema.class.getClassLoader()); -// if (sourceRecord != null && sourceRecord.getSchema() != null) { -// SchemaInfo srcSchemaInfo = sourceRecord.getSchema().getSchemaInfo(); -// log.info("[SinkRecord] map classLoader: {}", srcSchemaInfo.getProperties().getClass().getClassLoader()); -// SchemaInfo schemaInfo = SchemaInfo.builder() -// .name(srcSchemaInfo.getName()) -// .schema(srcSchemaInfo.getSchema()) -// .type(SchemaType.valueOf(srcSchemaInfo.getType().getValue())) -// .properties(srcSchemaInfo.getProperties()) -// .build(); -// Schema schema = (Schema) Schema.getSchema(schemaInfo); -// log.info("[SinkRecord] schemaInfo: {}", schemaInfo); -// return schema; -// } else { -// return null; -// } } + } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java index 6ffebbac13bb5..51c3b9d10e384 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java @@ -87,7 +87,7 @@ protected PulsarSinkProcessorBase(Schema schema) { public Producer createProducer(PulsarClient client, String topic, String producerName, Schema schema) throws PulsarClientException { - log.info("[createProducer] schema: {}", schema == null ? "null" : schema.getSchemaInfo()); + log.info("[createProducer] topic: {}, schema: {}", topic, schema == null ? "null" : schema.getSchemaInfo()); ProducerBuilder builder = client.newProducer(schema) .blockIfQueueFull(true) .enableBatching(true) @@ -107,18 +107,18 @@ public Producer createProducer(PulsarClient client, String topic, String prod return builder.properties(properties).create(); } - protected Producer getProducer(String destinationTopic) { - return getProducer(destinationTopic, null, destinationTopic); + protected Producer getProducer(String destinationTopic, Schema schema) { + return getProducer(destinationTopic, null, destinationTopic, schema); } - protected Producer getProducer(String producerId, String producerName, String topicName) { + protected Producer getProducer(String producerId, String producerName, String topicName, Schema schema) { return publishProducers.computeIfAbsent(producerId, s -> { try { return createProducer( client, topicName, producerName, - schema); + schema == null ? schema : this.schema); } catch (PulsarClientException e) { log.error("Failed to create Producer while doing user publish", e); throw new RuntimeException(e); @@ -181,15 +181,14 @@ public PulsarSinkAtMostOnceProcessor(Schema schema) { @Override public TypedMessageBuilder newMessage(Record record) { if (record.getSchema() != null) { - schema = record.getSchema(); return getProducer(record .getDestinationTopic() - .orElse(pulsarSinkConfig.getTopic())) + .orElse(pulsarSinkConfig.getTopic()), record.getSchema()) .newMessage(record.getSchema()); } else { return getProducer(record .getDestinationTopic() - .orElse(pulsarSinkConfig.getTopic())) + .orElse(pulsarSinkConfig.getTopic()), record.getSchema()) .newMessage(); } } @@ -232,10 +231,10 @@ public TypedMessageBuilder newMessage(Record record) { Producer producer = getProducer( String.format("%s-%s",record.getDestinationTopic().orElse(pulsarSinkConfig.getTopic()), record.getPartitionId().get()), record.getPartitionId().get(), - record.getDestinationTopic().orElse(pulsarSinkConfig.getTopic()) + record.getDestinationTopic().orElse(pulsarSinkConfig.getTopic()), + record.getSchema() ); if (record.getSchema() != null) { - schema = record.getSchema(); return producer.newMessage(record.getSchema()); } else { return producer.newMessage(); From d45f27a1a4e7b27a962cd5ec3bd4aee23eba0150 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Mon, 27 Apr 2020 16:11:02 +0800 Subject: [PATCH 29/36] fix test --- .../main/java/org/apache/pulsar/functions/sink/PulsarSink.java | 2 +- .../pulsar/tests/integration/functions/PulsarFunctionsTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java index 51c3b9d10e384..9007b3f71fa04 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java @@ -118,7 +118,7 @@ protected Producer getProducer(String producerId, String producerName, String client, topicName, producerName, - schema == null ? schema : this.schema); + schema != null ? schema : this.schema); } catch (PulsarClientException e) { log.error("Failed to create Producer while doing user publish", e); throw new RuntimeException(e); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index d9c8a6f727fbf..599d03a9ad8b7 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -141,7 +141,7 @@ public void testRabbitMQSink() throws Exception { testSink(new RabbitMQSinkTester(containerName), true, new RabbitMQSourceTester(containerName)); } - @Test(groups = "source") +// @Test(groups = "source") public void testDebeziumMySqlSourceJson() throws Exception { Thread.sleep(1000 * 30); testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter", true); From 7e91ab0a2a16bdf1c7e4b3e2453b0e33a07fc273 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Mon, 27 Apr 2020 18:21:02 +0800 Subject: [PATCH 30/36] fix test --- .../org/apache/pulsar/functions/instance/SinkRecord.java | 5 +++-- .../pulsar/io/kafka/connect/KafkaConnectSource.java | 8 ++++++++ .../tests/integration/functions/PulsarFunctionsTest.java | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java index 21fb94351c299..0593eedab7fbf 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java @@ -90,7 +90,7 @@ public Optional getDestinationTopic() { @Override public Schema getSchema() { if (sourceRecord == null) { - log.info("[SinkRecord] topic: {}, schema is null", sourceRecord.getDestinationTopic().isPresent() + log.info("[SinkRecord] topic: {}, sourceRecord is null", sourceRecord.getDestinationTopic().isPresent() ? sourceRecord.getDestinationTopic().get() : "null"); return null; } @@ -109,9 +109,10 @@ public Schema getSchema() { log.info("[SinkRecord] topic: {}, Schema: {}", sourceRecord.getDestinationTopic().isPresent() ? sourceRecord.getDestinationTopic().get() : "null", schema.getClass().getName()); + return schema; } - log.info("[SinkRecord] topic: {}, schema is null", sourceRecord.getDestinationTopic().isPresent() + log.info("[SinkRecord] topic: {}, schema is null or sourceRecord is not KVRecord.", sourceRecord.getDestinationTopic().isPresent() ? sourceRecord.getDestinationTopic().get() : "null"); return null; } diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index fbc2603726627..261a23d793e74 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -174,10 +174,18 @@ public synchronized Record> read() throws Exception { continue; } outstandingRecords.addAndGet(recordList.size()); + log.info("outstandingRecords - size: {}", outstandingRecords.get()); currentBatch = recordList.iterator(); } if (currentBatch.hasNext()) { Record> processRecord = processSourceRecord(currentBatch.next()); + KVRecord kvRecord = (KVRecord) processRecord; + KeyValue keyValue = processRecord.getValue(); + log.info("processRecord keyLength: {}, keySchema: {}, valueLength: {}, valueSchema: {}", + keyValue.getKey() != null ? keyValue.getKey().length : "null", + kvRecord.getKeySchema().getClass().getName(), + keyValue.getValue() != null ? keyValue.getValue().length : "null", + kvRecord.getValueSchema().getClass().getName()); if (processRecord.getValue().getValue() == null) { outstandingRecords.decrementAndGet(); continue; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 599d03a9ad8b7..d9c8a6f727fbf 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -141,7 +141,7 @@ public void testRabbitMQSink() throws Exception { testSink(new RabbitMQSinkTester(containerName), true, new RabbitMQSourceTester(containerName)); } -// @Test(groups = "source") + @Test(groups = "source") public void testDebeziumMySqlSourceJson() throws Exception { Thread.sleep(1000 * 30); testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter", true); From 5aa985c51bb73d0a46b64db69557e9e6899b4890 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Mon, 27 Apr 2020 20:43:45 +0800 Subject: [PATCH 31/36] fix test --- .../functions/PulsarFunctionsTest.java | 94 +++++++++++++------ 1 file changed, 64 insertions(+), 30 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index d9c8a6f727fbf..137633041b65e 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -71,9 +71,11 @@ import java.io.BufferedReader; import java.io.InputStreamReader; import java.time.Duration; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -2299,16 +2301,17 @@ private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWit @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); - try { - // If topic already exists, we should delete it so as not to affect the following tests. - admin.topics().getStats(consumeTopicName); - admin.topics().delete(consumeTopicName); - admin.schemas().deleteSchema(consumeTopicName); - } catch (PulsarAdminException e) { - // Expected results, ignoring the exception - log.info("Topic: {} does not exist, we can continue the following tests. Exceptions message: {}", - consumeTopicName, e.getMessage()); - } + deleteInventoryTopics(admin); +// try { +// // If topic already exists, we should delete it so as not to affect the following tests. +// admin.topics().getStats(consumeTopicName); +// admin.topics().delete(consumeTopicName); +// admin.schemas().deleteSchema(consumeTopicName); +// } catch (PulsarAdminException e) { +// // Expected results, ignoring the exception +// log.info("Topic: {} does not exist, we can continue the following tests. Exceptions message: {}", +// consumeTopicName, e.getMessage()); +// } try { SchemaInfo lastSchemaInfo = admin.schemas().getSchemaInfo(consumeTopicName); @@ -2408,16 +2411,17 @@ private void testDebeziumPostgreSqlConnect(String converterClassName, boolean j @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); - try { - // If topic already exists, we should delete it so as not to affect the following tests. - admin.topics().getStats(consumeTopicName); - admin.topics().delete(consumeTopicName); - admin.schemas().deleteSchema(consumeTopicName); - } catch (PulsarAdminException e) { - // Expected results, ignoring the exception - log.info("Topic: {} does not exist, we can continue the following tests. Exceptions message: {}", - consumeTopicName, e.getMessage()); - } + deleteInventoryTopics(admin); +// try { +// // If topic already exists, we should delete it so as not to affect the following tests. +// admin.topics().getStats(consumeTopicName); +// admin.topics().delete(consumeTopicName); +// admin.schemas().deleteSchema(consumeTopicName); +// } catch (PulsarAdminException e) { +// // Expected results, ignoring the exception +// log.info("Topic: {} does not exist, we can continue the following tests. Exceptions message: {}", +// consumeTopicName, e.getMessage()); +// } admin.topics().createNonPartitionedTopic(consumeTopicName); admin.topics().createNonPartitionedTopic(outputTopicName); @@ -2504,16 +2508,17 @@ private void testDebeziumMongoDbConnect(String converterClassName, boolean json @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); - try { - // If topic already exists, we should delete it so as not to affect the following tests. - admin.topics().getStats(consumeTopicName); - admin.topics().delete(consumeTopicName); - admin.schemas().deleteSchema(consumeTopicName); - } catch (PulsarAdminException e) { - // Expected results, ignoring the exception - log.info("Topic: {} does not exist, we can continue the following tests. Exceptions message: {}", - consumeTopicName, e.getMessage()); - } + deleteInventoryTopics(admin); +// try { +// // If topic already exists, we should delete it so as not to affect the following tests. +// admin.topics().getStats(consumeTopicName); +// admin.topics().delete(consumeTopicName); +// admin.schemas().deleteSchema(consumeTopicName); +// } catch (PulsarAdminException e) { +// // Expected results, ignoring the exception +// log.info("Topic: {} does not exist, we can continue the following tests. Exceptions message: {}", +// consumeTopicName, e.getMessage()); +// } admin.topics().createNonPartitionedTopic(consumeTopicName); admin.topics().createNonPartitionedTopic(outputTopicName); @@ -2575,6 +2580,35 @@ private void testDebeziumMongoDbConnect(String converterClassName, boolean json getSourceInfoNotFound(tenant, namespace, sourceName); } + private void deleteInventoryTopics(PulsarAdmin admin) { + log.info("[deleteInventoryTopics] start."); + List topics = Arrays.asList( + "persistent://public/default/dbserver1.inventory.products", + "persistent://public/default/dbserver1.inventory.customers", + "persistent://public/default/dbserver1.inventory.products_on_hand", + "persistent://public/default/dbserver1.inventory.addresses", + "persistent://public/default/dbserver1.inventory.orders"); + + for (String topic : topics) { + deleteTopic(admin, topic); + } + log.info("[deleteInventoryTopics] finish."); + } + + private void deleteTopic(PulsarAdmin admin, String topic) { + try { + // If topic already exists, we should delete it so as not to affect the following tests. + admin.topics().getStats(topic); + admin.topics().delete(topic); + admin.schemas().deleteSchema(topic); + } catch (PulsarAdminException e) { + // Expected results, ignoring the exception + log.info("Topic: {} does not exist, we can continue the following tests. Exceptions message: {}", + topic, e.getMessage()); + } + + } + private Schema getSchema(boolean jsonWithEnvelope) { if (jsonWithEnvelope) { return KeyValueSchema.kvBytes(); From dc83b12d0f9c17373ba60761d92743ae62903b6d Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Mon, 27 Apr 2020 23:20:39 +0800 Subject: [PATCH 32/36] fix test --- .../integration/functions/PulsarFunctionsTest.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 137633041b65e..76e3b25b2d658 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -85,6 +85,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -102,6 +103,11 @@ public abstract class PulsarFunctionsTest extends PulsarFunctionsTestBase { .withDelay(TEN_SECONDS) .onRetry(e -> log.error("Retry ... ")); + final RetryPolicy topicResetRetryPolicy = new RetryPolicy() + .withMaxDuration(ONE_MINUTE) + .withDelay(TEN_SECONDS) + .onRetry(e -> log.error("Retry reset topic")); + PulsarFunctionsTest(FunctionRuntimeType functionRuntimeType) { super(functionRuntimeType); } @@ -2590,21 +2596,25 @@ private void deleteInventoryTopics(PulsarAdmin admin) { "persistent://public/default/dbserver1.inventory.orders"); for (String topic : topics) { - deleteTopic(admin, topic); + Failsafe.with(topicResetRetryPolicy).run(() -> deleteTopic(admin, topic)); } log.info("[deleteInventoryTopics] finish."); } private void deleteTopic(PulsarAdmin admin, String topic) { + SchemaInfo schemaInfo = null; try { // If topic already exists, we should delete it so as not to affect the following tests. admin.topics().getStats(topic); admin.topics().delete(topic); admin.schemas().deleteSchema(topic); + schemaInfo = admin.schemas().getSchemaInfo(topic); + assertNull(schemaInfo); } catch (PulsarAdminException e) { // Expected results, ignoring the exception log.info("Topic: {} does not exist, we can continue the following tests. Exceptions message: {}", topic, e.getMessage()); + assertNull(schemaInfo); } } From a03f6dce9a574f0395a5704c72eb5b4084e38d3c Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Tue, 28 Apr 2020 00:46:33 +0800 Subject: [PATCH 33/36] fix --- .../tests/integration/functions/PulsarFunctionsTest.java | 4 ---- .../tests/integration/io/DebeziumMongoDbSourceTester.java | 1 + .../tests/integration/io/DebeziumMySqlSourceTester.java | 2 ++ .../tests/integration/io/DebeziumPostgreSqlSourceTester.java | 1 + 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 76e3b25b2d658..2725c4a7c8b9a 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -151,26 +151,22 @@ public void testRabbitMQSink() throws Exception { @Test(groups = "source") public void testDebeziumMySqlSourceJson() throws Exception { - Thread.sleep(1000 * 30); testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter", true); } @Test(groups = "source") public void testDebeziumMySqlSourceAvro() throws Exception { - Thread.sleep(1000 * 30); testDebeziumMySqlConnect( "org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter", false); } @Test(groups = "source") public void testDebeziumPostgreSqlSource() throws Exception { - Thread.sleep(1000 * 30); testDebeziumPostgreSqlConnect("org.apache.kafka.connect.json.JsonConverter", true); } @Test(groups = "source") public void testDebeziumMongoDbSource() throws Exception{ - Thread.sleep(1000 * 30); testDebeziumMongoDbConnect("org.apache.kafka.connect.json.JsonConverter", true); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMongoDbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMongoDbSourceTester.java index 23b5db43134e4..6fa35ef918376 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMongoDbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMongoDbSourceTester.java @@ -50,6 +50,7 @@ public DebeziumMongoDbSourceTester(PulsarCluster cluster) { sourceConfig.put("mongodb.task.id","1"); sourceConfig.put("database.whitelist", "inventory"); sourceConfig.put("pulsar.service.url", pulsarServiceUrl); + sourceConfig.put("topic.namespace", "debezium/mongodb"); } @Override diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMySqlSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMySqlSourceTester.java index 901e66453ce11..4b665065e0e01 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMySqlSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMySqlSourceTester.java @@ -63,6 +63,8 @@ public DebeziumMySqlSourceTester(PulsarCluster cluster, String converterClassNam sourceConfig.put("pulsar.service.url", pulsarServiceUrl); sourceConfig.put("key.converter", converterClassName); sourceConfig.put("value.converter", converterClassName); + sourceConfig.put("topic.namespace", "debezium/mysql-" + + (converterClassName.endsWith("AvroConverter") ? "avro" : "json")); } @Override diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumPostgreSqlSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumPostgreSqlSourceTester.java index e0efff2df1115..a8deb4ccc075d 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumPostgreSqlSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumPostgreSqlSourceTester.java @@ -63,6 +63,7 @@ public DebeziumPostgreSqlSourceTester(PulsarCluster cluster) { sourceConfig.put("schema.whitelist", "inventory"); sourceConfig.put("table.blacklist", "inventory.spatial_ref_sys,inventory.geom"); sourceConfig.put("pulsar.service.url", pulsarServiceUrl); + sourceConfig.put("topic.namespace", "debezium/postgresql"); } @Override From d07394a2e1d90828ad92f05740d407b3c372be4f Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Tue, 28 Apr 2020 01:54:29 +0800 Subject: [PATCH 34/36] fix --- .../functions/PulsarFunctionsTest.java | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 2725c4a7c8b9a..540a722a16a18 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -43,6 +43,7 @@ import org.apache.pulsar.common.policies.data.FunctionStatus; import org.apache.pulsar.common.policies.data.SinkStatus; import org.apache.pulsar.common.policies.data.SourceStatus; +import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; @@ -65,6 +66,7 @@ import org.apache.pulsar.tests.integration.utils.DockerUtils; import org.assertj.core.api.Assertions; import org.testcontainers.containers.GenericContainer; +import org.testcontainers.shaded.com.google.common.collect.Sets; import org.testng.annotations.Test; import org.testng.collections.Maps; @@ -2283,10 +2285,13 @@ private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWit final String tenant = TopicName.PUBLIC_TENANT; final String namespace = TopicName.DEFAULT_NAMESPACE; final String outputTopicName = "debe-output-topic-name"; - final String consumeTopicName = "public/default/dbserver1.inventory.products"; + boolean isJsonConverter = converterClassName.endsWith("JsonConverter"); + final String consumeTopicName = "debezium/mysql-" + + (isJsonConverter ? "json" : "avro") + + "/dbserver1.inventory.products"; // final String sourceName = "test-source-connector-" // + functionRuntimeType + "-name-mysql"; - final String sourceName = "test-source-debezium-mysql"; + final String sourceName = "test-source-debezium-mysql" + (isJsonConverter ? "json" : "avro"); // This is the binlog count that contained in mysql container. final int numMessages = 47; @@ -2303,6 +2308,7 @@ private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWit @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); + initNamespace(admin); deleteInventoryTopics(admin); // try { // // If topic already exists, we should delete it so as not to affect the following tests. @@ -2392,7 +2398,7 @@ private void testDebeziumPostgreSqlConnect(String converterClassName, boolean j final String tenant = TopicName.PUBLIC_TENANT; final String namespace = TopicName.DEFAULT_NAMESPACE; final String outputTopicName = "debe-output-topic-name"; - final String consumeTopicName = "public/default/dbserver1.inventory.products"; + final String consumeTopicName = "debezium/postgresql/dbserver1.inventory.products"; // final String sourceName = "test-source-connector-" // + functionRuntimeType + "-name-" + randomName(8); final String sourceName = "test-source-debezium-postgersql"; @@ -2413,6 +2419,7 @@ private void testDebeziumPostgreSqlConnect(String converterClassName, boolean j @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); + initNamespace(admin); deleteInventoryTopics(admin); // try { // // If topic already exists, we should delete it so as not to affect the following tests. @@ -2491,7 +2498,7 @@ private void testDebeziumMongoDbConnect(String converterClassName, boolean json final String tenant = TopicName.PUBLIC_TENANT; final String namespace = TopicName.DEFAULT_NAMESPACE; final String outputTopicName = "debe-output-topic-name"; - final String consumeTopicName = "public/default/dbserver1.inventory.products"; + final String consumeTopicName = "debezium/mongodb/dbserver1.inventory.products"; final String sourceName = "test-source-connector-" + functionRuntimeType + "-name-" + randomName(8); @@ -2510,6 +2517,7 @@ private void testDebeziumMongoDbConnect(String converterClassName, boolean json @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); + initNamespace(admin); deleteInventoryTopics(admin); // try { // // If topic already exists, we should delete it so as not to affect the following tests. @@ -2582,6 +2590,21 @@ private void testDebeziumMongoDbConnect(String converterClassName, boolean json getSourceInfoNotFound(tenant, namespace, sourceName); } + private void initNamespace(PulsarAdmin admin) { + log.info("[initNamespace] start."); + try { + admin.tenants().createTenant("debezium", new TenantInfo(Sets.newHashSet(), + Sets.newHashSet(pulsarCluster.getClusterName()))); + admin.namespaces().createNamespace("debezium/mysql-json"); + admin.namespaces().createNamespace("debezium/mysql-avro"); + admin.namespaces().createNamespace("debezium/mongodb"); + admin.namespaces().createNamespace("debezium/postgresql"); + } catch (Exception e) { + log.info("[initNamespace] msg: {}", e.getMessage()); + } + log.info("[initNamespace] finish."); + } + private void deleteInventoryTopics(PulsarAdmin admin) { log.info("[deleteInventoryTopics] start."); List topics = Arrays.asList( From 7698bc990b1226b4f88800b0d575b0a0e8d16ecb Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Tue, 28 Apr 2020 03:28:57 +0800 Subject: [PATCH 35/36] delete test log --- .../client/impl/TypedMessageBuilderImpl.java | 4 +- .../client/impl/schema/StructSchema.java | 8 +- .../schema/generic/GenericAvroReader.java | 5 +- .../schema/generic/GenericAvroSchema.java | 5 +- .../java/org/apache/pulsar/DebeziumTest.java | 125 ------------------ .../schema/generic/GenericAvroReaderTest.java | 2 +- .../pulsar/functions/instance/SinkRecord.java | 13 +- .../pulsar/functions/sink/PulsarSink.java | 8 -- .../apache/pulsar/functions/LocalRunner.java | 1 - .../io/kafka/connect/KafkaConnectSource.java | 11 +- .../schema/KafkaSchemaWrappedSchema.java | 4 +- .../containers/ChaosContainer.java | 27 ---- .../functions/PulsarFunctionsTest.java | 83 +----------- 13 files changed, 21 insertions(+), 275 deletions(-) delete mode 100644 pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java index 460e03827f723..44edbc986a537 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java @@ -144,13 +144,13 @@ public TypedMessageBuilder value(T value) { if (schema.getSchemaInfo() != null && schema.getSchemaInfo().getType() == SchemaType.KEY_VALUE) { KeyValueSchema kvSchema = (KeyValueSchema) schema; org.apache.pulsar.common.schema.KeyValue kv = (org.apache.pulsar.common.schema.KeyValue) value; - if (((KeyValueSchema) schema).getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) { + if (kvSchema.getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED) { // set key as the message key msgMetadataBuilder.setPartitionKey( Base64.getEncoder().encodeToString(kvSchema.getKeySchema().encode(kv.getKey()))); msgMetadataBuilder.setPartitionKeyB64Encoded(true); // set value as the payload - this.content = ByteBuffer.wrap((kvSchema.getValueSchema().encode(kv.getValue()))); + this.content = ByteBuffer.wrap(kvSchema.getValueSchema().encode(kv.getValue())); return this; } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java index bbbda12e1ce9d..fc0608bbb2ff6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java @@ -41,7 +41,7 @@ import org.apache.pulsar.client.api.schema.SchemaInfoProvider; import org.apache.pulsar.client.api.schema.SchemaReader; import org.apache.pulsar.client.api.schema.SchemaWriter; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroReader; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; import org.apache.pulsar.common.protocol.schema.BytesSchemaVersion; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -80,9 +80,9 @@ protected StructSchema(SchemaInfo schemaInfo) { this.schema = parseAvroSchema(new String(schemaInfo.getSchema(), UTF_8)); this.schemaInfo = schemaInfo; - if (schemaInfo.getProperties().containsKey(GenericAvroReader.OFFSET_PROP)) { - this.schema.addProp(GenericAvroReader.OFFSET_PROP, - schemaInfo.getProperties().get(GenericAvroReader.OFFSET_PROP)); + if (schemaInfo.getProperties().containsKey(GenericAvroSchema.OFFSET_PROP)) { + this.schema.addProp(GenericAvroSchema.OFFSET_PROP, + schemaInfo.getProperties().get(GenericAvroSchema.OFFSET_PROP)); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReader.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReader.java index 9e90080aea607..0b7547a1a4ab2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReader.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReader.java @@ -49,7 +49,6 @@ public class GenericAvroReader implements SchemaReader { private final byte[] schemaVersion; private int offset; - public final static String OFFSET_PROP = "AVRO_READ_OFFSET"; public GenericAvroReader(Schema schema) { this(null, schema, null); @@ -70,8 +69,8 @@ public GenericAvroReader(Schema writerSchema, Schema readerSchema, byte[] schema this.byteArrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(this.byteArrayOutputStream, encoder); - if (schema.getObjectProp(GenericAvroReader.OFFSET_PROP) != null) { - this.offset = Integer.parseInt(schema.getObjectProp(GenericAvroReader.OFFSET_PROP).toString()); + if (schema.getObjectProp(GenericAvroSchema.OFFSET_PROP) != null) { + this.offset = Integer.parseInt(schema.getObjectProp(GenericAvroSchema.OFFSET_PROP).toString()); } else { this.offset = 0; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java index a1f93b94474a3..94c5ba1f9911c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java @@ -33,6 +33,8 @@ @Slf4j public class GenericAvroSchema extends GenericSchemaImpl { + public final static String OFFSET_PROP = "__AVRO_READ_OFFSET__"; + public GenericAvroSchema(SchemaInfo schemaInfo) { this(schemaInfo, true); } @@ -73,8 +75,7 @@ protected SchemaReader loadReader(BytesSchemaVersion schemaVersio schemaInfo); Schema writerSchema = parseAvroSchema(schemaInfo.getSchemaDefinition()); Schema readerSchema = useProvidedSchemaAsReaderSchema ? schema : writerSchema; - readerSchema.addProp(GenericAvroReader.OFFSET_PROP, - schemaInfo.getProperties().getOrDefault(GenericAvroReader.OFFSET_PROP, "0")); + readerSchema.addProp(OFFSET_PROP, schemaInfo.getProperties().getOrDefault(OFFSET_PROP, "0")); return new GenericAvroReader( writerSchema, diff --git a/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java b/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java deleted file mode 100644 index e4c382899cdbc..0000000000000 --- a/pulsar-client/src/test/java/org/apache/pulsar/DebeziumTest.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar; - -import org.apache.pulsar.client.api.Consumer; -import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.SubscriptionInitialPosition; -import org.apache.pulsar.client.api.schema.Field; -import org.apache.pulsar.client.api.schema.GenericRecord; -import org.apache.pulsar.client.impl.schema.KeyValueSchema; -import org.apache.pulsar.common.schema.KeyValue; -import org.apache.pulsar.common.schema.KeyValueEncodingType; -import org.testng.annotations.Test; - -public class DebeziumTest { - -// @Test - private void testJsonConverterBytes() throws PulsarClientException { - PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); - - Schema> schema = - Schema.KeyValue(Schema.BYTES, Schema.BYTES, KeyValueEncodingType.INLINE); - - Consumer> consumer = pulsarClient.newConsumer(schema) - .topic("public/default/dbserver1.inventory.products") - .subscriptionName("journey-test") - .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) - .subscribe(); - - while (true) { - Message> message = consumer.receive(); - KeyValue keyValue = message.getValue(); - System.out.println("----------- get message -----------"); - System.out.println("key: " + new String(keyValue.getKey())); - System.out.println("value: " + new String(keyValue.getValue())); - } - } - -// @Test - private void testJsonConverter() throws PulsarClientException { - PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); - - Schema> schema = - Schema.KeyValue(Schema.AUTO_CONSUME(), Schema.AUTO_CONSUME(), KeyValueEncodingType.SEPARATED); - - Consumer> consumer = pulsarClient.newConsumer(schema) - .topic("public/default/dbserver1.inventory.products") - .subscriptionName("journey-test") - .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) - .subscribe(); - - while (true) { - Message> message = consumer.receive(); - KeyValue keyValue = message.getValue(); - System.out.println("----------- get message -----------"); - System.out.println("key: " + new String(message.getKeyBytes())); - System.out.println("value: " + new String(message.getData())); - } - } - -// @Test - private void testAvroConverter() throws PulsarClientException { - PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); - - Schema> schema = - Schema.KeyValue(Schema.AUTO_CONSUME(), Schema.AUTO_CONSUME(), KeyValueEncodingType.SEPARATED); - - Consumer> consumer = pulsarClient.newConsumer(schema) - .topic("public/default/dbserver1.inventory.products") - .subscriptionName("journey-test") - .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) - .subscribe(); - - while (true) { - Message> message = consumer.receive(); - try { - message.getKeyBytes(); - message.getData(); - KeyValue result = message.getValue(); - System.out.println("------------- got message -------------"); - - System.out.println("key >>>>>>>>>>> "); - for (Field field : result.getKey().getFields()) { - Object obj = result.getKey().getField(field); - System.out.println(field.getName() + ":" + (obj == null ? "null" : obj.toString())); - } - - System.out.println("value >>>>>>>>>>> "); - for (Field field : result.getValue().getFields()) { - Object obj = result.getValue().getField(field); - System.out.println(field.getName() + ":" + (obj == null ? "null" : obj.toString())); - if (obj != null && !"null".equalsIgnoreCase(obj.toString()) && (field.getName().equals("source") || - field.getName().equals("before") || field.getName().equals("after"))) { - for (Field innerField : ((GenericRecord) obj).getFields()) { - Object innerObj = ((GenericRecord) obj).getField(innerField); - System.out.println(" " + innerField.getName() + ":" + (innerObj == null ? "null" : innerObj.toString())); - } - } - } - } catch (Exception e) { - e.printStackTrace(); -// consumer.acknowledge(message); - } - } - } -} diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReaderTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReaderTest.java index 52e387fd88f5f..15717947a22d9 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReaderTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReaderTest.java @@ -54,7 +54,7 @@ public void setup() { .build()); fooOffsetSchema = AvroSchema.of(Foo.class); - fooOffsetSchema.getAvroSchema().addProp(GenericAvroReader.OFFSET_PROP, 5); + fooOffsetSchema.getAvroSchema().addProp(GenericAvroSchema.OFFSET_PROP, 5); foo = new Foo(); foo.setField1("foo1"); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java index 0593eedab7fbf..71a398484d6ab 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/SinkRecord.java @@ -90,30 +90,19 @@ public Optional getDestinationTopic() { @Override public Schema getSchema() { if (sourceRecord == null) { - log.info("[SinkRecord] topic: {}, sourceRecord is null", sourceRecord.getDestinationTopic().isPresent() - ? sourceRecord.getDestinationTopic().get() : "null"); return null; } if (sourceRecord.getSchema() != null) { - log.info("[SinkRecord] topic: {}, Schema: {}", - sourceRecord.getDestinationTopic().isPresent() ? sourceRecord.getDestinationTopic().get() : "null", - sourceRecord.getSchema().getClass().getName()); return sourceRecord.getSchema(); } if (sourceRecord instanceof KVRecord) { KVRecord kvRecord = (KVRecord) sourceRecord; - Schema schema = KeyValueSchema.of(kvRecord.getKeySchema(), kvRecord.getValueSchema(), + return KeyValueSchema.of(kvRecord.getKeySchema(), kvRecord.getValueSchema(), kvRecord.getKeyValueEncodingType()); - log.info("[SinkRecord] topic: {}, Schema: {}", - sourceRecord.getDestinationTopic().isPresent() ? sourceRecord.getDestinationTopic().get() : "null", - schema.getClass().getName()); - return schema; } - log.info("[SinkRecord] topic: {}, schema is null or sourceRecord is not KVRecord.", sourceRecord.getDestinationTopic().isPresent() - ? sourceRecord.getDestinationTopic().get() : "null"); return null; } diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java index 9007b3f71fa04..8cb67efa47b1d 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java @@ -87,7 +87,6 @@ protected PulsarSinkProcessorBase(Schema schema) { public Producer createProducer(PulsarClient client, String topic, String producerName, Schema schema) throws PulsarClientException { - log.info("[createProducer] topic: {}, schema: {}", topic, schema == null ? "null" : schema.getSchemaInfo()); ProducerBuilder builder = client.newProducer(schema) .blockIfQueueFull(true) .enableBatching(true) @@ -294,13 +293,6 @@ public void open(Map config, SinkContext sinkContext) throws Exc public void write(Record record) { TypedMessageBuilder msg = pulsarSinkProcessor.newMessage(record); - if (record != null && record.getSchema() != null) { - log.info("[write] KeyValueSchema className: {}, classLoader: {}", - KeyValueSchema.class.getName(), KeyValueSchema.class.getClassLoader()); - log.info("[write] schema className: {}, classLoader: {}", - record.getSchema().getClass().getName(), record.getSchema().getClass().getClassLoader()); - } - if (record.getKey().isPresent() && !(record.getSchema() instanceof KeyValueSchema && ((KeyValueSchema) record.getSchema()).getKeyValueEncodingType() == KeyValueEncodingType.SEPARATED)) { msg.key(record.getKey().get()); diff --git a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java index 9fb8de4668b35..b9ac10eea6918 100644 --- a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java +++ b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java @@ -249,7 +249,6 @@ public void start(boolean blocking) throws Exception { } String builtInSource = isBuiltInSource(userCodeFile); - log.info("builtInSource: {}", builtInSource); if (builtInSource != null) { sourceConfig.setArchive(builtInSource); } diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index 261a23d793e74..a178b4bcc00d3 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -174,18 +174,10 @@ public synchronized Record> read() throws Exception { continue; } outstandingRecords.addAndGet(recordList.size()); - log.info("outstandingRecords - size: {}", outstandingRecords.get()); currentBatch = recordList.iterator(); } if (currentBatch.hasNext()) { Record> processRecord = processSourceRecord(currentBatch.next()); - KVRecord kvRecord = (KVRecord) processRecord; - KeyValue keyValue = processRecord.getValue(); - log.info("processRecord keyLength: {}, keySchema: {}, valueLength: {}, valueSchema: {}", - keyValue.getKey() != null ? keyValue.getKey().length : "null", - kvRecord.getKeySchema().getClass().getName(), - keyValue.getValue() != null ? keyValue.getValue().length : "null", - kvRecord.getValueSchema().getClass().getName()); if (processRecord.getValue().getValue() == null) { outstandingRecords.decrementAndGet(); continue; @@ -375,8 +367,7 @@ public void ack() { @Override public void fail() { if (flushFuture != null) { - log.info("[fail] isCompletedExceptionally: {}", flushFuture.isCompletedExceptionally()); -// flushFuture.completeExceptionally(new Exception("Sink Error")); + flushFuture.completeExceptionally(new Exception("Sink Error")); } } } diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java index ba2aea30b1401..2db9d6cd93bc6 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/schema/KafkaSchemaWrappedSchema.java @@ -27,7 +27,7 @@ import org.apache.kafka.connect.json.JsonConverter; import org.apache.kafka.connect.storage.Converter; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.schema.generic.GenericAvroReader; +import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; @@ -43,7 +43,7 @@ public KafkaSchemaWrappedSchema(org.apache.pulsar.kafka.shade.avro.Schema schema Converter converter) { Map props = new HashMap<>(); boolean isJsonConverter = converter instanceof JsonConverter; - props.put(GenericAvroReader.OFFSET_PROP, isJsonConverter ? "0" : "5"); + props.put(GenericAvroSchema.OFFSET_PROP, isJsonConverter ? "0" : "5"); this.schemaInfo = SchemaInfo.builder() .name(isJsonConverter? "KafKaJson" : "KafkaAvro") .type(isJsonConverter ? SchemaType.JSON : SchemaType.AVRO) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java index 3fedea2d417d2..a66f635e8d8f2 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/ChaosContainer.java @@ -29,7 +29,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang.StringUtils; import org.apache.pulsar.tests.integration.docker.ContainerExecResult; import org.apache.pulsar.tests.integration.utils.DockerUtils; import org.testcontainers.containers.GenericContainer; @@ -139,30 +138,4 @@ public int hashCode() { clusterName); } - @Override - public void start() { - super.start(); - this.tailContainerLog(); - if (this.getContainerName().contains("pulsar-broker")) { - DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), - "tail", "-f", "/var/log/pulsar/broker.log"); - } else if (this.getContainerName().contains("bookie")) { - DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), - "tail", "-f", "/var/log/pulsar/bookie.log"); - } else if (this.getContainerName().contains("functions-worker")) { - DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), - "tail", "-f", "/var/log/pulsar/functions_worker.log"); - - DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), - "mkdir", "-p", - "/tmp/functions/public/default/test-source-debezium-mysql"); - DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), - "touch", - "/tmp/functions/public/default/test-source-debezium-mysql/test-source-debezium-mysql-0.log"); - DockerUtils.runCommandAsync(this.dockerClient, this.getContainerId(), - "tail", "-f", - "/tmp/functions/public/default/test-source-debezium-mysql/test-source-debezium-mysql-0.log"); - } - } - } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 540a722a16a18..07262eec4f384 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -105,11 +105,6 @@ public abstract class PulsarFunctionsTest extends PulsarFunctionsTestBase { .withDelay(TEN_SECONDS) .onRetry(e -> log.error("Retry ... ")); - final RetryPolicy topicResetRetryPolicy = new RetryPolicy() - .withMaxDuration(ONE_MINUTE) - .withDelay(TEN_SECONDS) - .onRetry(e -> log.error("Retry reset topic")); - PulsarFunctionsTest(FunctionRuntimeType functionRuntimeType) { super(functionRuntimeType); } @@ -2289,9 +2284,8 @@ private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWit final String consumeTopicName = "debezium/mysql-" + (isJsonConverter ? "json" : "avro") + "/dbserver1.inventory.products"; -// final String sourceName = "test-source-connector-" -// + functionRuntimeType + "-name-mysql"; - final String sourceName = "test-source-debezium-mysql" + (isJsonConverter ? "json" : "avro"); + final String sourceName = "test-source-debezium-mysql" + (isJsonConverter ? "json" : "avro") + + "-" + functionRuntimeType + "-" + randomName(8); // This is the binlog count that contained in mysql container. final int numMessages = 47; @@ -2309,17 +2303,6 @@ private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWit @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); initNamespace(admin); - deleteInventoryTopics(admin); -// try { -// // If topic already exists, we should delete it so as not to affect the following tests. -// admin.topics().getStats(consumeTopicName); -// admin.topics().delete(consumeTopicName); -// admin.schemas().deleteSchema(consumeTopicName); -// } catch (PulsarAdminException e) { -// // Expected results, ignoring the exception -// log.info("Topic: {} does not exist, we can continue the following tests. Exceptions message: {}", -// consumeTopicName, e.getMessage()); -// } try { SchemaInfo lastSchemaInfo = admin.schemas().getSchemaInfo(consumeTopicName); @@ -2329,7 +2312,6 @@ private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWit consumeTopicName, e.getMessage()); } -// admin.topics().createNonPartitionedTopic(consumeTopicName); admin.topics().createNonPartitionedTopic(outputTopicName); @Cleanup @@ -2399,9 +2381,7 @@ private void testDebeziumPostgreSqlConnect(String converterClassName, boolean j final String namespace = TopicName.DEFAULT_NAMESPACE; final String outputTopicName = "debe-output-topic-name"; final String consumeTopicName = "debezium/postgresql/dbserver1.inventory.products"; -// final String sourceName = "test-source-connector-" -// + functionRuntimeType + "-name-" + randomName(8); - final String sourceName = "test-source-debezium-postgersql"; + final String sourceName = "test-source-debezium-postgersql-" + functionRuntimeType + "-" + randomName(8); // This is the binlog count that contained in postgresql container. @@ -2420,17 +2400,7 @@ private void testDebeziumPostgreSqlConnect(String converterClassName, boolean j @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); initNamespace(admin); - deleteInventoryTopics(admin); -// try { -// // If topic already exists, we should delete it so as not to affect the following tests. -// admin.topics().getStats(consumeTopicName); -// admin.topics().delete(consumeTopicName); -// admin.schemas().deleteSchema(consumeTopicName); -// } catch (PulsarAdminException e) { -// // Expected results, ignoring the exception -// log.info("Topic: {} does not exist, we can continue the following tests. Exceptions message: {}", -// consumeTopicName, e.getMessage()); -// } + admin.topics().createNonPartitionedTopic(consumeTopicName); admin.topics().createNonPartitionedTopic(outputTopicName); @@ -2518,17 +2488,7 @@ private void testDebeziumMongoDbConnect(String converterClassName, boolean json @Cleanup PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); initNamespace(admin); - deleteInventoryTopics(admin); -// try { -// // If topic already exists, we should delete it so as not to affect the following tests. -// admin.topics().getStats(consumeTopicName); -// admin.topics().delete(consumeTopicName); -// admin.schemas().deleteSchema(consumeTopicName); -// } catch (PulsarAdminException e) { -// // Expected results, ignoring the exception -// log.info("Topic: {} does not exist, we can continue the following tests. Exceptions message: {}", -// consumeTopicName, e.getMessage()); -// } + admin.topics().createNonPartitionedTopic(consumeTopicName); admin.topics().createNonPartitionedTopic(outputTopicName); @@ -2605,39 +2565,6 @@ private void initNamespace(PulsarAdmin admin) { log.info("[initNamespace] finish."); } - private void deleteInventoryTopics(PulsarAdmin admin) { - log.info("[deleteInventoryTopics] start."); - List topics = Arrays.asList( - "persistent://public/default/dbserver1.inventory.products", - "persistent://public/default/dbserver1.inventory.customers", - "persistent://public/default/dbserver1.inventory.products_on_hand", - "persistent://public/default/dbserver1.inventory.addresses", - "persistent://public/default/dbserver1.inventory.orders"); - - for (String topic : topics) { - Failsafe.with(topicResetRetryPolicy).run(() -> deleteTopic(admin, topic)); - } - log.info("[deleteInventoryTopics] finish."); - } - - private void deleteTopic(PulsarAdmin admin, String topic) { - SchemaInfo schemaInfo = null; - try { - // If topic already exists, we should delete it so as not to affect the following tests. - admin.topics().getStats(topic); - admin.topics().delete(topic); - admin.schemas().deleteSchema(topic); - schemaInfo = admin.schemas().getSchemaInfo(topic); - assertNull(schemaInfo); - } catch (PulsarAdminException e) { - // Expected results, ignoring the exception - log.info("Topic: {} does not exist, we can continue the following tests. Exceptions message: {}", - topic, e.getMessage()); - assertNull(schemaInfo); - } - - } - private Schema getSchema(boolean jsonWithEnvelope) { if (jsonWithEnvelope) { return KeyValueSchema.kvBytes(); From 6c2f98d92f3aa57b67a6435e57730aa88053d6a9 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Tue, 28 Apr 2020 03:32:59 +0800 Subject: [PATCH 36/36] delete test log --- .../org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java | 2 -- .../pulsar/client/impl/schema/generic/GenericAvroReader.java | 1 - 2 files changed, 3 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java index 44edbc986a537..8d7884954bd9a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java @@ -30,7 +30,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClientException; @@ -44,7 +43,6 @@ import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString; -@Slf4j public class TypedMessageBuilderImpl implements TypedMessageBuilder { private static final long serialVersionUID = 0L; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReader.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReader.java index 0b7547a1a4ab2..22c63a94a5f18 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReader.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroReader.java @@ -47,7 +47,6 @@ public class GenericAvroReader implements SchemaReader { private final List fields; private final Schema schema; private final byte[] schemaVersion; - private int offset; public GenericAvroReader(Schema schema) {