From e4dfa86127643dbe027f9f48a9c03143df57b0e7 Mon Sep 17 00:00:00 2001 From: penghui Date: Sat, 1 Jun 2019 10:42:46 +0800 Subject: [PATCH 01/13] Introduce batch message container framework. --- .../client/impl/BatchMessageContainer.java | 195 ++++++----------- .../impl/BatchMessageContainerImpl.java | 205 ++++++++++++++++++ .../client/impl/BatchMessageContainers.java | 36 +++ .../pulsar/client/impl/ProducerImpl.java | 114 ++++------ .../impl/conf/ProducerConfigurationData.java | 2 + .../pulsar/PulsarConsumerMessageSource.java | 39 ++++ .../pulsar/PulsarTypeInformation.java | 66 ++++++ .../pulsar/PulsarTypeSerializer.java | 87 ++++++++ .../PulsarTypeSerializerConfigSnapshot.java | 21 ++ .../containers/FlinkContainer.java | 30 +++ .../integration/flink/FlinkContainerTest.java | 15 ++ 11 files changed, 614 insertions(+), 196 deletions(-) create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainers.java create mode 100644 pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerMessageSource.java create mode 100644 pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeInformation.java create mode 100644 pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializer.java create mode 100644 pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializerConfigSnapshot.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/FlinkContainer.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/flink/FlinkContainerTest.java diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainer.java index 0e1adee0da07c..6d7a57908519c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainer.java @@ -18,137 +18,78 @@ */ package org.apache.pulsar.client.impl; -import com.google.common.collect.Lists; - -import io.netty.buffer.ByteBuf; +import org.apache.pulsar.client.impl.ProducerImpl.OpSendMsg; +import java.io.IOException; import java.util.List; -import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; -import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.api.proto.PulsarApi; -import org.apache.pulsar.common.compression.CompressionCodec; -import org.apache.pulsar.common.compression.CompressionCodecProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * container for individual messages being published until they are batched and sent to broker */ - -class BatchMessageContainer { - - private SendCallback previousCallback = null; - private final PulsarApi.CompressionType compressionType; - private final CompressionCodec compressor; - private final String topicName; - private final String producerName; - - final int maxNumMessagesInBatch; - - PulsarApi.MessageMetadata.Builder messageMetadata = PulsarApi.MessageMetadata.newBuilder(); - int numMessagesInBatch = 0; - long currentBatchSizeBytes = 0; - // sequence id for this batch which will be persisted as a single entry by broker - long sequenceId = -1; - ByteBuf batchedMessageMetadataAndPayload; - List> messages = Lists.newArrayList(); - // keep track of callbacks for individual messages being published in a batch - SendCallback firstCallback; - - private static final int INITIAL_BATCH_BUFFER_SIZE = 1024; - protected static final int MAX_MESSAGE_BATCH_SIZE_BYTES = 128 * 1024; - - // This will be the largest size for a batch sent from this particular producer. This is used as a baseline to - // allocate a new buffer that can hold the entire batch without needing costly reallocations - private int maxBatchSize = INITIAL_BATCH_BUFFER_SIZE; - - BatchMessageContainer(int maxNumMessagesInBatch, PulsarApi.CompressionType compressionType, String topicName, - String producerName) { - this.maxNumMessagesInBatch = maxNumMessagesInBatch; - this.compressionType = compressionType; - this.compressor = CompressionCodecProvider.getCompressionCodec(compressionType); - this.topicName = topicName; - this.producerName = producerName; - } - - boolean hasSpaceInBatch(MessageImpl msg) { - int messageSize = msg.getDataBuffer().readableBytes(); - return ((messageSize + currentBatchSizeBytes) <= MAX_MESSAGE_BATCH_SIZE_BYTES - && numMessagesInBatch < maxNumMessagesInBatch); - } - - void add(MessageImpl msg, SendCallback callback) { - - if (log.isDebugEnabled()) { - log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName, producerName, - numMessagesInBatch); - } - - if (++numMessagesInBatch == 1) { - // some properties are common amongst the different messages in the batch, hence we just pick it up from - // the first message - sequenceId = Commands.initBatchMessageMetadata(messageMetadata, msg.getMessageBuilder()); - this.firstCallback = callback; - batchedMessageMetadataAndPayload = PulsarByteBufAllocator.DEFAULT - .buffer(Math.min(maxBatchSize, MAX_MESSAGE_BATCH_SIZE_BYTES)); - } - - if (previousCallback != null) { - previousCallback.addCallback(msg, callback); - } - previousCallback = callback; - - currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); - PulsarApi.MessageMetadata.Builder msgBuilder = msg.getMessageBuilder(); - batchedMessageMetadataAndPayload = Commands.serializeSingleMessageInBatchWithPayload(msgBuilder, - msg.getDataBuffer(), batchedMessageMetadataAndPayload); - messages.add(msg); - msgBuilder.recycle(); - } - - ByteBuf getCompressedBatchMetadataAndPayload() { - int uncompressedSize = batchedMessageMetadataAndPayload.readableBytes(); - ByteBuf compressedPayload = compressor.encode(batchedMessageMetadataAndPayload); - batchedMessageMetadataAndPayload.release(); - if (compressionType != PulsarApi.CompressionType.NONE) { - messageMetadata.setCompression(compressionType); - messageMetadata.setUncompressedSize(uncompressedSize); - } - - // Update the current max batch size using the uncompressed size, which is what we need in any case to - // accumulate the batch content - maxBatchSize = Math.max(maxBatchSize, uncompressedSize); - return compressedPayload; - } - - PulsarApi.MessageMetadata setBatchAndBuild() { - messageMetadata.setNumMessagesInBatch(numMessagesInBatch); - if (log.isDebugEnabled()) { - log.debug("[{}] [{}] num messages in batch being closed are {}", topicName, producerName, - numMessagesInBatch); - } - return messageMetadata.build(); - } - - ByteBuf getBatchedSingleMessageMetadataAndPayload() { - return batchedMessageMetadataAndPayload; - } - - void clear() { - messages = Lists.newArrayList(); - firstCallback = null; - previousCallback = null; - messageMetadata.clear(); - numMessagesInBatch = 0; - currentBatchSizeBytes = 0; - sequenceId = -1; - batchedMessageMetadataAndPayload = null; - } - - boolean isEmpty() { - return messages.isEmpty(); - } - - private static final Logger log = LoggerFactory.getLogger(BatchMessageContainer.class); +public interface BatchMessageContainer { + + /** + * Add message to the batch message container. + * + * @param msg message will add to the batch message container + * @param callback message send callback + */ + void add(MessageImpl msg, SendCallback callback); + + /** + * Check the batch message container have enough space for the message want to add. + * + * @param msg the message want to add + * @return return true if the container have enough space for the specific message, + * otherwise return false. + */ + boolean haveEnoughSpace(MessageImpl msg); + + /** + * Clear the message batch container. + */ + void clear(); + + /** + * Check the message batch container is empty. + * + * @return return true if empty, otherwise return false. + */ + boolean isEmpty(); + + /** + * Get count of messages in the message batch container. + * + * @return messages count + */ + int getNumMessagesInBatch(); + + /** + * Get current message batch size of the message batch container in bytes. + * + * @return message batch size in bytes + */ + long getCurrentBatchSizeBytes(); + + /** + * Set producer of the message batch container. + * + * @param producer producer + */ + void setProducer(ProducerImpl producer); + + /** + * Release the payload and clear the container. + * + * @param ex cause + */ + void handleException(Exception ex); + + /** + * Create list of OpSendMsg, producer use OpSendMsg to send to the broker. + * + * @return list of OpSendMsg + * @throws IOException + */ + List createOpSendMsgs() throws IOException; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java new file mode 100644 index 0000000000000..8075fdaf4c2ac --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -0,0 +1,205 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import com.google.common.collect.Lists; + +import io.netty.buffer.ByteBuf; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import io.netty.util.ReferenceCountUtil; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.ProducerImpl.OpSendMsg; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.api.ByteBufPair; +import org.apache.pulsar.common.api.Commands; +import org.apache.pulsar.common.api.proto.PulsarApi; +import org.apache.pulsar.common.compression.CompressionCodec; +import org.apache.pulsar.common.compression.CompressionCodecProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * container for individual messages being published until they are batched and sent to broker + */ + +class BatchMessageContainerImpl implements BatchMessageContainer { + + private SendCallback previousCallback = null; + private PulsarApi.CompressionType compressionType; + private CompressionCodec compressor; + private String topicName; + private String producerName; + private ProducerImpl producer; + + int maxNumMessagesInBatch; + + PulsarApi.MessageMetadata.Builder messageMetadata = PulsarApi.MessageMetadata.newBuilder(); + int numMessagesInBatch = 0; + long currentBatchSizeBytes = 0; + // sequence id for this batch which will be persisted as a single entry by broker + long sequenceId = -1; + ByteBuf batchedMessageMetadataAndPayload; + List> messages = Lists.newArrayList(); + // keep track of callbacks for individual messages being published in a batch + SendCallback firstCallback; + + private static final int INITIAL_BATCH_BUFFER_SIZE = 1024; + protected static final int MAX_MESSAGE_BATCH_SIZE_BYTES = 128 * 1024; + + // This will be the largest size for a batch sent from this particular producer. This is used as a baseline to + // allocate a new buffer that can hold the entire batch without needing costly reallocations + private int maxBatchSize = INITIAL_BATCH_BUFFER_SIZE; + + + @Override + public boolean haveEnoughSpace(MessageImpl msg) { + int messageSize = msg.getDataBuffer().readableBytes(); + return ((messageSize + currentBatchSizeBytes) <= MAX_MESSAGE_BATCH_SIZE_BYTES + && numMessagesInBatch < maxNumMessagesInBatch); + } + + @Override + public void add(MessageImpl msg, SendCallback callback) { + + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName, producerName, + numMessagesInBatch); + } + + if (++numMessagesInBatch == 1) { + // some properties are common amongst the different messages in the batch, hence we just pick it up from + // the first message + sequenceId = Commands.initBatchMessageMetadata(messageMetadata, msg.getMessageBuilder()); + this.firstCallback = callback; + batchedMessageMetadataAndPayload = PulsarByteBufAllocator.DEFAULT + .buffer(Math.min(maxBatchSize, MAX_MESSAGE_BATCH_SIZE_BYTES)); + } + + if (previousCallback != null) { + previousCallback.addCallback(msg, callback); + } + previousCallback = callback; + + currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); + PulsarApi.MessageMetadata.Builder msgBuilder = msg.getMessageBuilder(); + batchedMessageMetadataAndPayload = Commands.serializeSingleMessageInBatchWithPayload(msgBuilder, + msg.getDataBuffer(), batchedMessageMetadataAndPayload); + messages.add(msg); + msgBuilder.recycle(); + } + + private ByteBuf getCompressedBatchMetadataAndPayload() { + int uncompressedSize = batchedMessageMetadataAndPayload.readableBytes(); + ByteBuf compressedPayload = compressor.encode(batchedMessageMetadataAndPayload); + batchedMessageMetadataAndPayload.release(); + if (compressionType != PulsarApi.CompressionType.NONE) { + messageMetadata.setCompression(compressionType); + messageMetadata.setUncompressedSize(uncompressedSize); + } + + // Update the current max batch size using the uncompressed size, which is what we need in any case to + // accumulate the batch content + maxBatchSize = Math.max(maxBatchSize, uncompressedSize); + return compressedPayload; + } + + @Override + public void clear() { + messages = Lists.newArrayList(); + firstCallback = null; + previousCallback = null; + messageMetadata.clear(); + numMessagesInBatch = 0; + currentBatchSizeBytes = 0; + sequenceId = -1; + batchedMessageMetadataAndPayload = null; + } + + @Override + public boolean isEmpty() { + return messages.isEmpty(); + } + + @Override + public int getNumMessagesInBatch() { + return numMessagesInBatch; + } + + @Override + public long getCurrentBatchSizeBytes() { + return currentBatchSizeBytes; + } + + @Override + public void setProducer(ProducerImpl producer) { + this.producer = producer; + this.topicName = producer.getTopic(); + this.producerName = producer.getProducerName(); + this.compressionType = CompressionCodecProvider + .convertToWireProtocol(producer.getConfiguration().getCompressionType()); + this.compressor = CompressionCodecProvider.getCompressionCodec(compressionType); + this.maxNumMessagesInBatch = producer.getConfiguration().getBatchingMaxMessages(); + } + + @Override + public void handleException(Exception ex) { + try { + // Need to protect ourselves from any exception being thrown in the future handler from the application + firstCallback.sendComplete(ex); + } catch (Throwable t) { + log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topicName, producerName, + sequenceId, t); + } + ReferenceCountUtil.safeRelease(batchedMessageMetadataAndPayload); + clear(); + } + + private OpSendMsg createOpSendMsg() throws IOException { + ByteBuf encryptedPayload = producer.encryptMessage(messageMetadata, getCompressedBatchMetadataAndPayload()); + messageMetadata.setNumMessagesInBatch(numMessagesInBatch); + ByteBufPair cmd = producer.sendMessage(producer.producerId, sequenceId, numMessagesInBatch, + messageMetadata.build(), encryptedPayload); + + OpSendMsg op = OpSendMsg.create(messages, cmd, sequenceId, firstCallback); + + if (encryptedPayload.readableBytes() > ClientCnx.getMaxMessageSize()) { + cmd.release(); + if (op != null) { + op.callback.sendComplete(new PulsarClientException.InvalidMessageException( + "Message size is bigger than " + ClientCnx.getMaxMessageSize() + " bytes")); + } + throw new PulsarClientException("Message size is bigger than " + + ClientCnx.getMaxMessageSize() + " bytes"); + } + + op.setNumMessagesInBatch(numMessagesInBatch); + op.setBatchSizeByte(currentBatchSizeBytes); + return op; + } + + @Override + public List createOpSendMsgs() throws IOException { + return Collections.singletonList(createOpSendMsg()); + } + + private static final Logger log = LoggerFactory.getLogger(BatchMessageContainerImpl.class); +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainers.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainers.java new file mode 100644 index 0000000000000..7be209e47b09a --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainers.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +/** + * Batch message container utilities + */ +public class BatchMessageContainers { + + /** + * Create a default batch message container. + * Default batch message container is the most basic batch message container, + * it generates a single batch messages form multiple single messages. + * + * @return a new default batch message container. + */ + public static BatchMessageContainer newDefaultBatchMessageContainer() { + return new BatchMessageContainerImpl(); + } +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index ab6b5b6b00310..f3d664ae42585 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -75,7 +75,7 @@ public class ProducerImpl extends ProducerBase implements TimerTask, ConnectionHandler.Connection { // Producer id, used to identify a producer within a single connection - private final long producerId; + protected final long producerId; // Variable is used through the atomic updater private volatile long msgIdGenerator; @@ -163,8 +163,12 @@ public ProducerImpl(PulsarClientImpl client, String topic, ProducerConfiguration this.createProducerTimeout = System.currentTimeMillis() + client.getConfiguration().getOperationTimeoutMs(); if (conf.isBatchingEnabled()) { this.maxNumMessagesInBatch = conf.getBatchingMaxMessages(); - this.batchMessageContainer = new BatchMessageContainer(maxNumMessagesInBatch, - CompressionCodecProvider.convertToWireProtocol(conf.getCompressionType()), topic, producerName); + if (conf.getBatchMessageContainer() == null) { + this.batchMessageContainer = new BatchMessageContainerImpl(); + } else { + this.batchMessageContainer = conf.getBatchMessageContainer(); + } + this.batchMessageContainer.setProducer(this); } else { this.maxNumMessagesInBatch = 1; this.batchMessageContainer = null; @@ -363,12 +367,12 @@ public void sendAsync(Message message, SendCallback callback) { if (isBatchMessagingEnabled() && !msgMetadataBuilder.hasDeliverAtTime()) { // handle boundary cases where message being added would exceed // batch size and/or max message size - if (batchMessageContainer.hasSpaceInBatch(msg)) { + if (batchMessageContainer.haveEnoughSpace(msg)) { batchMessageContainer.add(msg, callback); lastSendFuture = callback.getFuture(); payload.release(); - if (batchMessageContainer.numMessagesInBatch == maxNumMessagesInBatch - || batchMessageContainer.currentBatchSizeBytes >= BatchMessageContainer.MAX_MESSAGE_BATCH_SIZE_BYTES) { + if (batchMessageContainer.getNumMessagesInBatch() == maxNumMessagesInBatch + || batchMessageContainer.getCurrentBatchSizeBytes() >= BatchMessageContainerImpl.MAX_MESSAGE_BATCH_SIZE_BYTES) { batchMessageAndSend(); } } else { @@ -425,7 +429,7 @@ public void sendAsync(Message message, SendCallback callback) { } } - private ByteBuf encryptMessage(MessageMetadata.Builder msgMetadata, ByteBuf compressedPayload) + protected ByteBuf encryptMessage(MessageMetadata.Builder msgMetadata, ByteBuf compressedPayload) throws PulsarClientException { ByteBuf encryptedPayload = compressedPayload; @@ -447,7 +451,7 @@ private ByteBuf encryptMessage(MessageMetadata.Builder msgMetadata, ByteBuf comp return encryptedPayload; } - private ByteBufPair sendMessage(long producerId, long sequenceId, int numMessages, MessageMetadata msgMetadata, + protected ByteBufPair sendMessage(long producerId, long sequenceId, int numMessages, MessageMetadata msgMetadata, ByteBuf compressedPayload) throws IOException { ChecksumType checksumType; @@ -1224,17 +1228,9 @@ private void failPendingBatchMessages(PulsarClientException ex) { if (batchMessageContainer.isEmpty()) { return; } - int numMessagesInBatch = batchMessageContainer.numMessagesInBatch; + int numMessagesInBatch = batchMessageContainer.getNumMessagesInBatch(); semaphore.release(numMessagesInBatch); - try { - // Need to protect ourselves from any exception being thrown in the future handler from the application - batchMessageContainer.firstCallback.sendComplete(ex); - } catch (Throwable t) { - log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topic, producerName, - batchMessageContainer.sequenceId, t); - } - ReferenceCountUtil.safeRelease(batchMessageContainer.getBatchedSingleMessageMetadataAndPayload()); - batchMessageContainer.clear(); + batchMessageContainer.handleException(ex); } TimerTask batchMessageAndSendTask = new TimerTask() { @@ -1288,68 +1284,48 @@ protected void triggerFlush() { private void batchMessageAndSend() { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Batching the messages from the batch container with {} messages", topic, producerName, - batchMessageContainer.numMessagesInBatch); + batchMessageContainer.getNumMessagesInBatch()); } - OpSendMsg op = null; - int numMessagesInBatch = 0; - try { - if (!batchMessageContainer.isEmpty()) { - numMessagesInBatch = batchMessageContainer.numMessagesInBatch; - ByteBuf compressedPayload = batchMessageContainer.getCompressedBatchMetadataAndPayload(); - long sequenceId = batchMessageContainer.sequenceId; - ByteBuf encryptedPayload = encryptMessage(batchMessageContainer.messageMetadata, compressedPayload); - - ByteBufPair cmd = sendMessage(producerId, sequenceId, batchMessageContainer.numMessagesInBatch, - batchMessageContainer.setBatchAndBuild(), encryptedPayload); - - op = OpSendMsg.create(batchMessageContainer.messages, cmd, sequenceId, - batchMessageContainer.firstCallback); - - if (encryptedPayload.readableBytes() > ClientCnx.getMaxMessageSize()) { - cmd.release(); - semaphore.release(numMessagesInBatch); - if (op != null) { - op.callback.sendComplete(new PulsarClientException.InvalidMessageException( - "Message size is bigger than " + ClientCnx.getMaxMessageSize() + " bytes")); - } - return; + if (!batchMessageContainer.isEmpty()) { + try { + for (OpSendMsg opSendMsg : batchMessageContainer.createOpSendMsgs()) { + processOpSendMsg(opSendMsg); } + } catch (PulsarClientException e) { + Thread.currentThread().interrupt(); + semaphore.release(batchMessageContainer.getNumMessagesInBatch()); + } catch (Throwable t) { + semaphore.release(batchMessageContainer.getNumMessagesInBatch()); + log.warn("[{}] [{}] error while create opSendMsg by batch message container -- {}", topic, producerName, t); + } + } + } - op.setNumMessagesInBatch(batchMessageContainer.numMessagesInBatch); - op.setBatchSizeByte(batchMessageContainer.currentBatchSizeBytes); - - batchMessageContainer.clear(); - - pendingMessages.put(op); - - ClientCnx cnx = cnx(); - if (isConnected()) { - // If we do have a connection, the message is sent immediately, otherwise we'll try again once a new - // connection is established - cmd.retain(); - cnx.ctx().channel().eventLoop().execute(WriteInEventLoopCallback.create(this, cnx, op)); - stats.updateNumMsgsSent(numMessagesInBatch, op.batchSizeByte); - } else { - if (log.isDebugEnabled()) { - log.debug("[{}] [{}] Connection is not ready -- sequenceId {}", topic, producerName, - sequenceId); - } + private void processOpSendMsg(OpSendMsg op) { + try { + batchMessageContainer.clear(); + pendingMessages.put(op); + ClientCnx cnx = cnx(); + if (isConnected()) { + // If we do have a connection, the message is sent immediately, otherwise we'll try again once a new + // connection is established + op.cmd.retain(); + cnx.ctx().channel().eventLoop().execute(WriteInEventLoopCallback.create(this, cnx, op)); + stats.updateNumMsgsSent(op.numMessagesInBatch, op.batchSizeByte); + } else { + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Connection is not ready -- sequenceId {}", topic, producerName, + op.sequenceId); } } } catch (InterruptedException ie) { Thread.currentThread().interrupt(); - semaphore.release(numMessagesInBatch); + semaphore.release(op.numMessagesInBatch); if (op != null) { op.callback.sendComplete(new PulsarClientException(ie)); } - } catch (PulsarClientException e) { - Thread.currentThread().interrupt(); - semaphore.release(numMessagesInBatch); - if (op != null) { - op.callback.sendComplete(e); - } } catch (Throwable t) { - semaphore.release(numMessagesInBatch); + semaphore.release(op.numMessagesInBatch); log.warn("[{}] [{}] error while closing out batch -- {}", topic, producerName, t); if (op != null) { op.callback.sendComplete(new PulsarClientException(t)); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java index 7ec8f9051f31f..ec97795c641ab 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java @@ -40,6 +40,7 @@ import com.google.common.collect.Sets; import lombok.Data; +import org.apache.pulsar.client.impl.BatchMessageContainer; @Data @NoArgsConstructor @@ -66,6 +67,7 @@ public class ProducerConfigurationData implements Serializable, Cloneable { private long batchingMaxPublishDelayMicros = TimeUnit.MILLISECONDS.toMicros(1); private int batchingMaxMessages = 1000; private boolean batchingEnabled = true; // enabled by default + private BatchMessageContainer batchMessageContainer = null; @JsonIgnore private CryptoKeyReader cryptoKeyReader; diff --git a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerMessageSource.java b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerMessageSource.java new file mode 100644 index 0000000000000..bbbfc8058d484 --- /dev/null +++ b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerMessageSource.java @@ -0,0 +1,39 @@ +package org.apache.flink.streaming.connectors.pulsar; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.functions.source.MessageAcknowledgingSourceBase; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; + +import java.util.Set; + +public class PulsarConsumerMessageSource + extends MessageAcknowledgingSourceBase, MessageId> + implements PulsarSourceBase> { + + PulsarConsumerMessageSource(PulsarSourceBuilder builder) { + super(MessageId.class); + + } + + @Override + public TypeInformation> getProducedType() { + return null; + } + + @Override + protected void acknowledgeIDs(long l, Set set) { + + } + + @Override + public void run(SourceContext> sourceContext) throws Exception { + + } + + @Override + public void cancel() { + + } + +} diff --git a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeInformation.java b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeInformation.java new file mode 100644 index 0000000000000..3db5e4d6f5b65 --- /dev/null +++ b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeInformation.java @@ -0,0 +1,66 @@ +package org.apache.flink.streaming.connectors.pulsar; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeutils.TypeSerializer; + +/** + * Convert a pulsar schema info into a flink type inform. + */ +public class PulsarTypeInformation extends TypeInformation { + + @Override + public boolean isBasicType() { + return false; + } + + @Override + public boolean isTupleType() { + return false; + } + + @Override + public int getArity() { + return 0; + } + + @Override + public int getTotalFields() { + return 0; + } + + @Override + public Class getTypeClass() { + return null; + } + + @Override + public boolean isKeyType() { + return false; + } + + @Override + public TypeSerializer createSerializer(ExecutionConfig config) { + return null; + } + + @Override + public String toString() { + return null; + } + + @Override + public boolean equals(Object obj) { + return false; + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public boolean canEqual(Object obj) { + return false; + } +} diff --git a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializer.java b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializer.java new file mode 100644 index 0000000000000..aec13145d9838 --- /dev/null +++ b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializer.java @@ -0,0 +1,87 @@ +package org.apache.flink.streaming.connectors.pulsar; + +import org.apache.flink.api.common.typeutils.CompatibilityResult; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; + +import java.io.IOException; + +public class PulsarTypeSerializer extends TypeSerializer { + + @Override + public boolean isImmutableType() { + return false; + } + + @Override + public TypeSerializer duplicate() { + return null; + } + + @Override + public T createInstance() { + return null; + } + + @Override + public T copy(T from) { + return null; + } + + @Override + public T copy(T from, T reuse) { + return null; + } + + @Override + public int getLength() { + return 0; + } + + @Override + public void serialize(T record, DataOutputView target) throws IOException { + + } + + @Override + public T deserialize(DataInputView source) throws IOException { + return null; + } + + @Override + public T deserialize(T reuse, DataInputView source) throws IOException { + return null; + } + + @Override + public void copy(DataInputView source, DataOutputView target) throws IOException { + + } + + @Override + public boolean equals(Object obj) { + return false; + } + + @Override + public boolean canEqual(Object obj) { + return false; + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public TypeSerializerConfigSnapshot snapshotConfiguration() { + return null; + } + + @Override + public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { + return null; + } +} diff --git a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializerConfigSnapshot.java b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializerConfigSnapshot.java new file mode 100644 index 0000000000000..95f06306218b2 --- /dev/null +++ b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializerConfigSnapshot.java @@ -0,0 +1,21 @@ +package org.apache.flink.streaming.connectors.pulsar; + +import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; + +public class PulsarTypeSerializerConfigSnapshot extends TypeSerializerConfigSnapshot { + + @Override + public boolean equals(Object obj) { + return false; + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public int getVersion() { + return 0; + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/FlinkContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/FlinkContainer.java new file mode 100644 index 0000000000000..3d7eec4efbd10 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/FlinkContainer.java @@ -0,0 +1,30 @@ +package org.apache.pulsar.tests.integration.containers; + +import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy; + +import java.time.Duration; + +public class FlinkContainer extends ChaosContainer { + + public static final String NAME = "Flink"; + static final Integer[] PORTS = { 8080 }; + + private static final String IMAGE_NAME = "flink:latest"; + + public FlinkContainer(String clusterName) { + super(clusterName, IMAGE_NAME); + } + + @Override + protected void configure() { + super.configure(); + this.withNetworkAliases(NAME) + .withExposedPorts(PORTS) + .withCreateContainerCmdModifier(createContainerCmd -> { + createContainerCmd.withHostName(NAME); + createContainerCmd.withName(clusterName + "-" + NAME); + }) + .withCommand("bin/start-cluster.sh") + .waitingFor(new HostPortWaitStrategy().withStartupTimeout(Duration.ofMinutes(3))); + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/flink/FlinkContainerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/flink/FlinkContainerTest.java new file mode 100644 index 0000000000000..7ea31bf2ba1f0 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/flink/FlinkContainerTest.java @@ -0,0 +1,15 @@ +package org.apache.pulsar.tests.integration.flink; + +import org.apache.pulsar.tests.integration.containers.FlinkContainer; +import org.testng.annotations.Test; + +public class FlinkContainerTest { + + @Test + public void test() throws Exception { + FlinkContainer flinkContainer = new FlinkContainer("test"); + flinkContainer.start(); + System.out.println(flinkContainer.isRunning()); + flinkContainer.stop(); + } +} From b4d404eb547cc2c4ab5eee6c28d47b623fc9ffcb Mon Sep 17 00:00:00 2001 From: lipenghui Date: Mon, 3 Jun 2019 10:49:57 +0800 Subject: [PATCH 02/13] Fix UI issues. --- .../apache/pulsar/client/impl/BatchMessageContainerImpl.java | 3 ++- .../main/java/org/apache/pulsar/client/impl/ProducerImpl.java | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java index 8075fdaf4c2ac..52040626fc234 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -187,8 +187,9 @@ private OpSendMsg createOpSendMsg() throws IOException { if (op != null) { op.callback.sendComplete(new PulsarClientException.InvalidMessageException( "Message size is bigger than " + ClientCnx.getMaxMessageSize() + " bytes")); + op.recycle(); } - throw new PulsarClientException("Message size is bigger than " + + ClientCnx.getMaxMessageSize() + " bytes"); + return null; } op.setNumMessagesInBatch(numMessagesInBatch); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index f3d664ae42585..ba6168e562f11 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -1289,6 +1289,9 @@ private void batchMessageAndSend() { if (!batchMessageContainer.isEmpty()) { try { for (OpSendMsg opSendMsg : batchMessageContainer.createOpSendMsgs()) { + if (opSendMsg == null) { + break; + } processOpSendMsg(opSendMsg); } } catch (PulsarClientException e) { From 3b986f94286a07388dfa32f7919a92a7cdb5ed8b Mon Sep 17 00:00:00 2001 From: lipenghui Date: Mon, 3 Jun 2019 16:19:28 +0800 Subject: [PATCH 03/13] Add batch message container builder. --- .../client/api/BatchMessageContainer.java | 58 +++++++++++++++++++ .../api/BatchMessageContainerBuilder.java | 36 ++++++++++++ .../pulsar/client/api/ProducerBuilder.java | 10 ++++ .../internal/DefaultImplementation.java | 7 +++ ...er.java => BatchMessageContainerBase.java} | 36 +----------- .../impl/BatchMessageContainerImpl.java | 2 +- ... DefaultBatchMessageContainerBuilder.java} | 18 ++---- .../client/impl/ProducerBuilderImpl.java | 8 +++ .../pulsar/client/impl/ProducerImpl.java | 8 +-- .../impl/conf/ProducerConfigurationData.java | 4 +- 10 files changed, 132 insertions(+), 55 deletions(-) create mode 100644 pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainer.java create mode 100644 pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java rename pulsar-client/src/main/java/org/apache/pulsar/client/impl/{BatchMessageContainer.java => BatchMessageContainerBase.java} (72%) rename pulsar-client/src/main/java/org/apache/pulsar/client/impl/{BatchMessageContainers.java => DefaultBatchMessageContainerBuilder.java} (67%) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainer.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainer.java new file mode 100644 index 0000000000000..f9d58f96710cb --- /dev/null +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainer.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +/** + * container for individual messages being published until they are batched and sent to broker + */ +public interface BatchMessageContainer { + + /** + * Clear the message batch container. + */ + void clear(); + + /** + * Check the message batch container is empty. + * + * @return return true if empty, otherwise return false. + */ + boolean isEmpty(); + + /** + * Get count of messages in the message batch container. + * + * @return messages count + */ + int getNumMessagesInBatch(); + + /** + * Get current message batch size of the message batch container in bytes. + * + * @return message batch size in bytes + */ + long getCurrentBatchSizeBytes(); + + /** + * Release the payload and clear the container. + * + * @param ex cause + */ + void handleException(Exception ex); +} diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java new file mode 100644 index 0000000000000..dbe06bdb9e269 --- /dev/null +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import org.apache.pulsar.client.internal.DefaultImplementation; + +/** + * Batch message container builder + */ +public interface BatchMessageContainerBuilder { + + BatchMessageContainerBuilder DEFAULT_BUILDER = DefaultImplementation.newDefaultBatchMessageContainerBuilder(); + + /** + * Build a new batch message container. + * @return new batch message container + */ + BatchMessageContainer build(); + +} diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java index a98036c475e20..cb6ac7dc861a9 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java @@ -365,6 +365,16 @@ public interface ProducerBuilder extends Cloneable { */ ProducerBuilder batchingMaxMessages(int batchMessagesMaxMessagesPerBatch); + /** + * Set the batch message container builder of the producer. Producer will use the batch message container builder to + * build a batch message container. + * + * @param batchingContainerBuilder + * batch message container builder + * @return the producer builder instance + */ + ProducerBuilder batchingContainerBuilder(BatchMessageContainerBuilder batchingContainerBuilder); + /** * Set the baseline for the sequence ids for messages published by the producer. *

diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java index d85b278844bb2..b3c71467c4280 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java @@ -36,6 +36,7 @@ import lombok.experimental.UtilityClass; import org.apache.pulsar.client.api.Authentication; +import org.apache.pulsar.client.api.BatchMessageContainerBuilder; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Schema; @@ -282,4 +283,10 @@ public static RecordSchemaBuilder newRecordSchemaBuilder(String name) { () -> (RecordSchemaBuilder) getConstructor("org.apache.pulsar.client.impl.schema.RecordSchemaBuilderImpl", String.class).newInstance(name)); } + + public static BatchMessageContainerBuilder newDefaultBatchMessageContainerBuilder() { + return catchExceptions( + () -> (BatchMessageContainerBuilder) getConstructor("org.apache.pulsar.client.impl.DefaultBatchMessageContainerBuilder") + .newInstance()); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java similarity index 72% rename from pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainer.java rename to pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java index 6d7a57908519c..930c97c3fb919 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl; +import org.apache.pulsar.client.api.BatchMessageContainer; import org.apache.pulsar.client.impl.ProducerImpl.OpSendMsg; import java.io.IOException; @@ -26,7 +27,7 @@ /** * container for individual messages being published until they are batched and sent to broker */ -public interface BatchMessageContainer { +public interface BatchMessageContainerBase extends BatchMessageContainer { /** * Add message to the batch message container. @@ -45,32 +46,6 @@ public interface BatchMessageContainer { */ boolean haveEnoughSpace(MessageImpl msg); - /** - * Clear the message batch container. - */ - void clear(); - - /** - * Check the message batch container is empty. - * - * @return return true if empty, otherwise return false. - */ - boolean isEmpty(); - - /** - * Get count of messages in the message batch container. - * - * @return messages count - */ - int getNumMessagesInBatch(); - - /** - * Get current message batch size of the message batch container in bytes. - * - * @return message batch size in bytes - */ - long getCurrentBatchSizeBytes(); - /** * Set producer of the message batch container. * @@ -78,13 +53,6 @@ public interface BatchMessageContainer { */ void setProducer(ProducerImpl producer); - /** - * Release the payload and clear the container. - * - * @param ex cause - */ - void handleException(Exception ex); - /** * Create list of OpSendMsg, producer use OpSendMsg to send to the broker. * diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java index 52040626fc234..7fef3eec8e0f4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -42,7 +42,7 @@ * container for individual messages being published until they are batched and sent to broker */ -class BatchMessageContainerImpl implements BatchMessageContainer { +class BatchMessageContainerImpl implements BatchMessageContainerBase { private SendCallback previousCallback = null; private PulsarApi.CompressionType compressionType; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainers.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/DefaultBatchMessageContainerBuilder.java similarity index 67% rename from pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainers.java rename to pulsar-client/src/main/java/org/apache/pulsar/client/impl/DefaultBatchMessageContainerBuilder.java index 7be209e47b09a..4806b2a484aac 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainers.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/DefaultBatchMessageContainerBuilder.java @@ -18,19 +18,13 @@ */ package org.apache.pulsar.client.impl; -/** - * Batch message container utilities - */ -public class BatchMessageContainers { +import org.apache.pulsar.client.api.BatchMessageContainer; +import org.apache.pulsar.client.api.BatchMessageContainerBuilder; + +public class DefaultBatchMessageContainerBuilder implements BatchMessageContainerBuilder { - /** - * Create a default batch message container. - * Default batch message container is the most basic batch message container, - * it generates a single batch messages form multiple single messages. - * - * @return a new default batch message container. - */ - public static BatchMessageContainer newDefaultBatchMessageContainer() { + @Override + public BatchMessageContainer build() { return new BatchMessageContainerImpl(); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java index 193d0341640e0..bb0436059e7be 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java @@ -29,6 +29,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.api.BatchMessageContainerBuilder; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.HashingScheme; @@ -223,6 +224,13 @@ public ProducerBuilder batchingMaxMessages(int batchMessagesMaxMessagesPerBat return this; } + @Override + public ProducerBuilder batchingContainerBuilder(BatchMessageContainerBuilder batchingContainerBuilder) { + conf.setBatchMessageContainerBuilder(batchingContainerBuilder); + return this; + } + + @Override public ProducerBuilder initialSequenceId(long initialSequenceId) { conf.setInitialSequenceId(initialSequenceId); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index ba6168e562f11..1a108ccd1476a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -87,7 +87,7 @@ public class ProducerImpl extends ProducerBase implements TimerTask, Conne private volatile Timeout batchMessageAndSendTimeout = null; private long createProducerTimeout; private final int maxNumMessagesInBatch; - private final BatchMessageContainer batchMessageContainer; + private final BatchMessageContainerBase batchMessageContainer; private CompletableFuture lastSendFuture = CompletableFuture.completedFuture(null); // Globally unique producer name @@ -163,11 +163,7 @@ public ProducerImpl(PulsarClientImpl client, String topic, ProducerConfiguration this.createProducerTimeout = System.currentTimeMillis() + client.getConfiguration().getOperationTimeoutMs(); if (conf.isBatchingEnabled()) { this.maxNumMessagesInBatch = conf.getBatchingMaxMessages(); - if (conf.getBatchMessageContainer() == null) { - this.batchMessageContainer = new BatchMessageContainerImpl(); - } else { - this.batchMessageContainer = conf.getBatchMessageContainer(); - } + this.batchMessageContainer = (BatchMessageContainerBase)conf.getBatchMessageContainerBuilder().build(); this.batchMessageContainer.setProducer(this); } else { this.maxNumMessagesInBatch = 1; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java index ec97795c641ab..eb2bcccac23ec 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java @@ -28,6 +28,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.NoArgsConstructor; +import org.apache.pulsar.client.api.BatchMessageContainerBuilder; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.HashingScheme; @@ -40,7 +41,6 @@ import com.google.common.collect.Sets; import lombok.Data; -import org.apache.pulsar.client.impl.BatchMessageContainer; @Data @NoArgsConstructor @@ -67,7 +67,7 @@ public class ProducerConfigurationData implements Serializable, Cloneable { private long batchingMaxPublishDelayMicros = TimeUnit.MILLISECONDS.toMicros(1); private int batchingMaxMessages = 1000; private boolean batchingEnabled = true; // enabled by default - private BatchMessageContainer batchMessageContainer = null; + private BatchMessageContainerBuilder batchMessageContainerBuilder = BatchMessageContainerBuilder.DEFAULT_BUILDER; @JsonIgnore private CryptoKeyReader cryptoKeyReader; From 5d5806ea05c6ce3a12b3a4cb579307e26b4d2d8d Mon Sep 17 00:00:00 2001 From: lipenghui Date: Tue, 4 Jun 2019 15:13:20 +0800 Subject: [PATCH 04/13] Add key based batch message container. --- .../broker/service/BatchMessageTest.java | 163 +++++++++++---- .../api/BatchMessageContainerBuilder.java | 4 +- .../internal/DefaultImplementation.java | 6 + .../impl/AbstractBatchMessageContainer.java | 71 +++++++ .../impl/BatchMessageContainerImpl.java | 73 ++----- .../impl/BatchMessageKeyBasedContainer.java | 195 ++++++++++++++++++ .../KeyBasedBatchMessageContainerBuilder.java | 30 +++ .../pulsar/client/impl/ProducerImpl.java | 6 +- .../impl/conf/ProducerConfigurationData.java | 2 +- .../pulsar/common/protocol/Commands.java | 3 + 10 files changed, 452 insertions(+), 101 deletions(-) create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java create mode 100644 pulsar-client/src/main/java/org/apache/pulsar/client/impl/KeyBasedBatchMessageContainerBuilder.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java index 516197d642f2e..b38c2c502be80 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java @@ -22,7 +22,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; -import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically; import com.google.common.collect.Lists; @@ -30,8 +29,10 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -39,12 +40,14 @@ import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.BatchMessageContainerBuilder; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; @@ -69,16 +72,31 @@ protected void cleanup() throws Exception { super.internalCleanup(); } - @DataProvider(name = "codec") - public Object[][] codecProvider() { - return new Object[][] { { CompressionType.NONE }, { CompressionType.LZ4 }, { CompressionType.ZLIB }, }; + @DataProvider(name = "codecAndContainerBuilder") + public Object[][] codecAndContainerBuilderProvider() { + return new Object[][] { + { CompressionType.NONE, BatchMessageContainerBuilder.DEFAULT }, + { CompressionType.LZ4, BatchMessageContainerBuilder.DEFAULT }, + { CompressionType.ZLIB, BatchMessageContainerBuilder.DEFAULT }, + { CompressionType.NONE, BatchMessageContainerBuilder.KEY_BASED }, + { CompressionType.LZ4, BatchMessageContainerBuilder.KEY_BASED }, + { CompressionType.ZLIB, BatchMessageContainerBuilder.KEY_BASED } + }; } - @Test(dataProvider = "codec") - public void testSimpleBatchProducerWithFixedBatchSize(CompressionType compressionType) throws Exception { + @DataProvider(name = "containerBuilder") + public Object[][] containerBuilderProvider() { + return new Object[][] { + { BatchMessageContainerBuilder.DEFAULT }, + { BatchMessageContainerBuilder.KEY_BASED } + }; + } + + @Test(dataProvider = "codecAndContainerBuilder") + public void testSimpleBatchProducerWithFixedBatchSize(CompressionType compressionType, BatchMessageContainerBuilder builder) throws Exception { int numMsgs = 50; int numMsgsInBatch = numMsgs / 2; - final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerWithFixedBatchSize"; + final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerWithFixedBatchSize-" + UUID.randomUUID(); final String subscriptionName = "sub-1" + compressionType.toString(); Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) @@ -87,6 +105,7 @@ public void testSimpleBatchProducerWithFixedBatchSize(CompressionType compressio Producer producer = pulsarClient.newProducer().topic(topicName).compressionType(compressionType) .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true) + .batchingContainerBuilder(builder) .create(); List> sendFutureList = Lists.newArrayList(); @@ -117,10 +136,10 @@ public void testSimpleBatchProducerWithFixedBatchSize(CompressionType compressio producer.close(); } - @Test(dataProvider = "codec") - public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressionType) throws Exception { + @Test(dataProvider = "codecAndContainerBuilder") + public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressionType, BatchMessageContainerBuilder builder) throws Exception { int numMsgs = 100; - final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerWithFixedBatchTime"; + final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerWithFixedBatchTime-" + UUID.randomUUID(); final String subscriptionName = "time-sub-1" + compressionType.toString(); Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) @@ -128,7 +147,9 @@ public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressio consumer.close(); Producer producer = pulsarClient.newProducer().topic(topicName).compressionType(compressionType) - .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS).enableBatching(true).create(); + .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS).enableBatching(true) + .batchingContainerBuilder(builder) + .create(); Random random = new Random(); List> sendFutureList = Lists.newArrayList(); @@ -151,10 +172,10 @@ public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressio producer.close(); } - @Test(dataProvider = "codec") - public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType compressionType) throws Exception { + @Test(dataProvider = "codecAndContainerBuilder") + public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType compressionType, BatchMessageContainerBuilder builder) throws Exception { int numMsgs = 100; - final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerWithFixedBatchSizeAndTime"; + final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerWithFixedBatchSizeAndTime-" + UUID.randomUUID(); final String subscriptionName = "time-size-sub-1" + compressionType.toString(); Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) @@ -163,6 +184,7 @@ public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType com Producer producer = pulsarClient.newProducer().topic(topicName) .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS).batchingMaxMessages(5) + .batchingContainerBuilder(builder) .compressionType(compressionType).enableBatching(true).create(); Random random = new Random(); @@ -186,11 +208,11 @@ public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType com producer.close(); } - @Test(dataProvider = "codec") - public void testBatchProducerWithLargeMessage(CompressionType compressionType) throws Exception { + @Test(dataProvider = "codecAndContainerBuilder") + public void testBatchProducerWithLargeMessage(CompressionType compressionType, BatchMessageContainerBuilder builder) throws Exception { int numMsgs = 50; int numMsgsInBatch = numMsgs / 2; - final String topicName = "persistent://prop/ns-abc/testBatchProducerWithLargeMessage"; + final String topicName = "persistent://prop/ns-abc/testBatchProducerWithLargeMessage-" + UUID.randomUUID(); final String subscriptionName = "large-message-sub-1" + compressionType.toString(); Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) @@ -199,6 +221,7 @@ public void testBatchProducerWithLargeMessage(CompressionType compressionType) t Producer producer = pulsarClient.newProducer().topic(topicName).compressionType(compressionType) .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true) + .batchingContainerBuilder(builder) .create(); List> sendFutureList = Lists.newArrayList(); @@ -238,11 +261,11 @@ public void testBatchProducerWithLargeMessage(CompressionType compressionType) t producer.close(); } - @Test(dataProvider = "codec") - public void testSimpleBatchProducerConsumer(CompressionType compressionType) throws Exception { + @Test(dataProvider = "codecAndContainerBuilder") + public void testSimpleBatchProducerConsumer(CompressionType compressionType, BatchMessageContainerBuilder builder) throws Exception { int numMsgs = 500; int numMsgsInBatch = numMsgs / 20; - final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerConsumer"; + final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerConsumer-" + UUID.randomUUID(); final String subscriptionName = "pc-sub-1" + compressionType.toString(); Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) @@ -257,6 +280,7 @@ public void testSimpleBatchProducerConsumer(CompressionType compressionType) thr // disabled size based batch .batchingMaxMessages(2 * numMsgs) .enableBatching(true) + .batchingContainerBuilder(builder) .create(); List> sendFutureList = Lists.newArrayList(); @@ -296,11 +320,11 @@ public void testSimpleBatchProducerConsumer(CompressionType compressionType) thr producer.close(); } - @Test - public void testSimpleBatchSyncProducerWithFixedBatchSize() throws Exception { + @Test(dataProvider = "containerBuilder") + public void testSimpleBatchSyncProducerWithFixedBatchSize(BatchMessageContainerBuilder builder) throws Exception { int numMsgs = 10; int numMsgsInBatch = numMsgs / 2; - final String topicName = "persistent://prop/ns-abc/testSimpleBatchSyncProducerWithFixedBatchSize"; + final String topicName = "persistent://prop/ns-abc/testSimpleBatchSyncProducerWithFixedBatchSize-" + UUID.randomUUID(); final String subscriptionName = "syncsub-1"; Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) @@ -309,6 +333,7 @@ public void testSimpleBatchSyncProducerWithFixedBatchSize() throws Exception { Producer producer = pulsarClient.newProducer().topic(topicName) .batchingMaxPublishDelay(1, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true) + .batchingContainerBuilder(builder) .create(); for (int i = 0; i < numMsgs; i++) { @@ -338,11 +363,11 @@ public void testSimpleBatchSyncProducerWithFixedBatchSize() throws Exception { } - @Test - public void testSimpleBatchProducerConsumer1kMessages() throws Exception { + @Test(dataProvider = "containerBuilder") + public void testSimpleBatchProducerConsumer1kMessages(BatchMessageContainerBuilder builder) throws Exception { int numMsgs = 2000; int numMsgsInBatch = 4; - final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerConsumer1kMessages"; + final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerConsumer1kMessages-" + UUID.randomUUID(); final String subscriptionName = "pc1k-sub-1"; Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) @@ -351,6 +376,7 @@ public void testSimpleBatchProducerConsumer1kMessages() throws Exception { Producer producer = pulsarClient.newProducer().topic(topicName).maxPendingMessages(numMsgs + 1) .batchingMaxPublishDelay(30, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true) + .batchingContainerBuilder(builder) .create(); List> sendFutureList = Lists.newArrayList(); @@ -465,11 +491,11 @@ public void testOutOfOrderAcksForBatchMessage() throws Exception { producer.close(); } - @Test - public void testNonBatchCumulativeAckAfterBatchPublish() throws Exception { + @Test(dataProvider = "containerBuilder") + public void testNonBatchCumulativeAckAfterBatchPublish(BatchMessageContainerBuilder builder) throws Exception { int numMsgs = 10; int numMsgsInBatch = numMsgs; - final String topicName = "persistent://prop/ns-abc/testNonBatchCumulativeAckAfterBatchPublish"; + final String topicName = "persistent://prop/ns-abc/testNonBatchCumulativeAckAfterBatchPublish-" + UUID.randomUUID(); final String subscriptionName = "nbcaabp-sub-1"; Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) @@ -478,6 +504,7 @@ public void testNonBatchCumulativeAckAfterBatchPublish() throws Exception { Producer producer = pulsarClient.newProducer().topic(topicName) .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true) + .batchingContainerBuilder(builder) .create(); // create producer to publish non batch messages Producer noBatchProducer = pulsarClient.newProducer().topic(topicName).create(); @@ -517,11 +544,11 @@ public void testNonBatchCumulativeAckAfterBatchPublish() throws Exception { noBatchProducer.close(); } - @Test - public void testBatchAndNonBatchCumulativeAcks() throws Exception { + @Test(dataProvider = "containerBuilder") + public void testBatchAndNonBatchCumulativeAcks(BatchMessageContainerBuilder builder) throws Exception { int numMsgs = 50; int numMsgsInBatch = numMsgs / 10; - final String topicName = "persistent://prop/ns-abc/testBatchAndNonBatchCumulativeAcks"; + final String topicName = "persistent://prop/ns-abc/testBatchAndNonBatchCumulativeAcks-" + UUID.randomUUID(); final String subscriptionName = "bnb-sub-1"; Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) @@ -532,6 +559,7 @@ public void testBatchAndNonBatchCumulativeAcks() throws Exception { .batchingMaxPublishDelay(5, TimeUnit.SECONDS) .batchingMaxMessages(numMsgsInBatch) .enableBatching(true) + .batchingContainerBuilder(builder) .messageRoutingMode(MessageRoutingMode.SinglePartition) .create(); // create producer to publish non batch messages @@ -590,10 +618,10 @@ public void testBatchAndNonBatchCumulativeAcks() throws Exception { * * @throws Exception */ - @Test(timeOut = 3000) - public void testConcurrentBatchMessageAck() throws Exception { + @Test(dataProvider = "containerBuilder", timeOut = 3000) + public void testConcurrentBatchMessageAck(BatchMessageContainerBuilder builder) throws Exception { int numMsgs = 10; - final String topicName = "persistent://prop/ns-abc/testConcurrentAck"; + final String topicName = "persistent://prop/ns-abc/testConcurrentAck-" + UUID.randomUUID(); final String subscriptionName = "sub-1"; Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName) @@ -602,6 +630,7 @@ public void testConcurrentBatchMessageAck() throws Exception { Producer producer = pulsarClient.newProducer().topic(topicName) .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgs).enableBatching(true) + .batchingContainerBuilder(builder) .create(); List> sendFutureList = Lists.newArrayList(); @@ -644,5 +673,69 @@ public void testConcurrentBatchMessageAck() throws Exception { producer.close(); } + @Test + public void testOrderingOfKeyBasedBatchMessageContainer() throws PulsarClientException, ExecutionException, InterruptedException { + final String topicName = "persistent://prop/ns-abc/testKeyBased"; + final String subscriptionName = "sub-1"; + Producer producer = pulsarClient.newProducer().topic(topicName) + .batchingMaxPublishDelay(5, TimeUnit.SECONDS) + .batchingMaxMessages(30) + .enableBatching(true) + .batchingContainerBuilder(BatchMessageContainerBuilder.KEY_BASED) + .create(); + Consumer consumer = pulsarClient.newConsumer().topic(topicName) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Key_Shared) + .subscribe(); + List> sendFutureList = Lists.newArrayList(); + String[] keys = new String[]{"key-1", "key-2", "key-3"}; + for (int i = 0; i < 10; i++) { + byte[] message = ("my-message-" + i).getBytes(); + for (String key : keys) { + sendFutureList.add(producer.newMessage().key(key).value(message).sendAsync()); + } + } + FutureUtil.waitForAll(sendFutureList).get(); + + for (int i = 0; i < 30; i++) { + Message received = consumer.receive(); + if (i < 10) { + assertEquals(received.getKey(), "key-1"); + } else if (i < 20) { + assertEquals(received.getKey(), "key-2"); + } else { + assertEquals(received.getKey(), "key-3"); + } + consumer.acknowledge(received); + } + + for (int i = 0; i < 10; i++) { + byte[] message = ("my-message-" + i).getBytes(); + for (String key : keys) { + sendFutureList.add(producer.newMessage() + .key(UUID.randomUUID().toString()) + .orderingKey(key.getBytes()) + .value(message) + .sendAsync()); + } + } + FutureUtil.waitForAll(sendFutureList).get(); + + for (int i = 0; i < 30; i++) { + Message received = consumer.receive(); + if (i < 10) { + assertEquals(new String(received.getOrderingKey()), "key-1"); + } else if (i < 20) { + assertEquals(new String(received.getOrderingKey()), "key-2"); + } else { + assertEquals(new String(received.getOrderingKey()), "key-3"); + } + consumer.acknowledge(received); + } + + consumer.close(); + producer.close(); + } + private static final Logger LOG = LoggerFactory.getLogger(BatchMessageTest.class); } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java index dbe06bdb9e269..b28a56da61af9 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java @@ -25,7 +25,9 @@ */ public interface BatchMessageContainerBuilder { - BatchMessageContainerBuilder DEFAULT_BUILDER = DefaultImplementation.newDefaultBatchMessageContainerBuilder(); + BatchMessageContainerBuilder DEFAULT = DefaultImplementation.newDefaultBatchMessageContainerBuilder(); + + BatchMessageContainerBuilder KEY_BASED = DefaultImplementation.newKeyBasedBatchMessageContainerBuilder(); /** * Build a new batch message container. diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java index b3c71467c4280..6588038a30456 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java @@ -289,4 +289,10 @@ public static BatchMessageContainerBuilder newDefaultBatchMessageContainerBuilde () -> (BatchMessageContainerBuilder) getConstructor("org.apache.pulsar.client.impl.DefaultBatchMessageContainerBuilder") .newInstance()); } + + public static BatchMessageContainerBuilder newKeyBasedBatchMessageContainerBuilder() { + return catchExceptions( + () -> (BatchMessageContainerBuilder) getConstructor("org.apache.pulsar.client.impl.KeyBasedBatchMessageContainerBuilder") + .newInstance()); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java new file mode 100644 index 0000000000000..f5d89477cc796 --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import org.apache.pulsar.common.api.proto.PulsarApi; +import org.apache.pulsar.common.compression.CompressionCodec; +import org.apache.pulsar.common.compression.CompressionCodecProvider; + +public abstract class AbstractBatchMessageContainer implements BatchMessageContainerBase { + + protected PulsarApi.CompressionType compressionType; + protected CompressionCodec compressor; + protected String topicName; + protected String producerName; + protected ProducerImpl producer; + + protected int maxNumMessagesInBatch; + protected int numMessagesInBatch = 0; + protected long currentBatchSizeBytes = 0; + + protected static final int INITIAL_BATCH_BUFFER_SIZE = 1024; + protected static final int MAX_MESSAGE_BATCH_SIZE_BYTES = 128 * 1024; + + // This will be the largest size for a batch sent from this particular producer. This is used as a baseline to + // allocate a new buffer that can hold the entire batch without needing costly reallocations + protected int maxBatchSize = INITIAL_BATCH_BUFFER_SIZE; + + @Override + public boolean haveEnoughSpace(MessageImpl msg) { + int messageSize = msg.getDataBuffer().readableBytes(); + return ((messageSize + currentBatchSizeBytes) <= MAX_MESSAGE_BATCH_SIZE_BYTES + && numMessagesInBatch < maxNumMessagesInBatch); + } + + @Override + public int getNumMessagesInBatch() { + return numMessagesInBatch; + } + + @Override + public long getCurrentBatchSizeBytes() { + return currentBatchSizeBytes; + } + + @Override + public void setProducer(ProducerImpl producer) { + this.producer = producer; + this.topicName = producer.getTopic(); + this.producerName = producer.getProducerName(); + this.compressionType = CompressionCodecProvider + .convertToWireProtocol(producer.getConfiguration().getCompressionType()); + this.compressor = CompressionCodecProvider.getCompressionCodec(compressionType); + this.maxNumMessagesInBatch = producer.getConfiguration().getBatchingMaxMessages(); + } +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java index 7fef3eec8e0f4..f78c2eae34873 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -33,8 +33,6 @@ import org.apache.pulsar.common.api.ByteBufPair; import org.apache.pulsar.common.api.Commands; import org.apache.pulsar.common.api.proto.PulsarApi; -import org.apache.pulsar.common.compression.CompressionCodec; -import org.apache.pulsar.common.compression.CompressionCodecProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,41 +40,16 @@ * container for individual messages being published until they are batched and sent to broker */ -class BatchMessageContainerImpl implements BatchMessageContainerBase { +class BatchMessageContainerImpl extends AbstractBatchMessageContainer { - private SendCallback previousCallback = null; - private PulsarApi.CompressionType compressionType; - private CompressionCodec compressor; - private String topicName; - private String producerName; - private ProducerImpl producer; - - int maxNumMessagesInBatch; - - PulsarApi.MessageMetadata.Builder messageMetadata = PulsarApi.MessageMetadata.newBuilder(); - int numMessagesInBatch = 0; - long currentBatchSizeBytes = 0; + private PulsarApi.MessageMetadata.Builder messageMetadata = PulsarApi.MessageMetadata.newBuilder(); // sequence id for this batch which will be persisted as a single entry by broker - long sequenceId = -1; - ByteBuf batchedMessageMetadataAndPayload; - List> messages = Lists.newArrayList(); + private long sequenceId = -1; + private ByteBuf batchedMessageMetadataAndPayload; + private List> messages = Lists.newArrayList(); + protected SendCallback previousCallback = null; // keep track of callbacks for individual messages being published in a batch - SendCallback firstCallback; - - private static final int INITIAL_BATCH_BUFFER_SIZE = 1024; - protected static final int MAX_MESSAGE_BATCH_SIZE_BYTES = 128 * 1024; - - // This will be the largest size for a batch sent from this particular producer. This is used as a baseline to - // allocate a new buffer that can hold the entire batch without needing costly reallocations - private int maxBatchSize = INITIAL_BATCH_BUFFER_SIZE; - - - @Override - public boolean haveEnoughSpace(MessageImpl msg) { - int messageSize = msg.getDataBuffer().readableBytes(); - return ((messageSize + currentBatchSizeBytes) <= MAX_MESSAGE_BATCH_SIZE_BYTES - && numMessagesInBatch < maxNumMessagesInBatch); - } + protected SendCallback firstCallback; @Override public void add(MessageImpl msg, SendCallback callback) { @@ -99,16 +72,17 @@ public void add(MessageImpl msg, SendCallback callback) { previousCallback.addCallback(msg, callback); } previousCallback = callback; - currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); - PulsarApi.MessageMetadata.Builder msgBuilder = msg.getMessageBuilder(); - batchedMessageMetadataAndPayload = Commands.serializeSingleMessageInBatchWithPayload(msgBuilder, - msg.getDataBuffer(), batchedMessageMetadataAndPayload); messages.add(msg); - msgBuilder.recycle(); } private ByteBuf getCompressedBatchMetadataAndPayload() { + for (MessageImpl msg : messages) { + PulsarApi.MessageMetadata.Builder msgBuilder = msg.getMessageBuilder(); + batchedMessageMetadataAndPayload = Commands.serializeSingleMessageInBatchWithPayload(msgBuilder, + msg.getDataBuffer(), batchedMessageMetadataAndPayload); + msgBuilder.recycle(); + } int uncompressedSize = batchedMessageMetadataAndPayload.readableBytes(); ByteBuf compressedPayload = compressor.encode(batchedMessageMetadataAndPayload); batchedMessageMetadataAndPayload.release(); @@ -140,27 +114,6 @@ public boolean isEmpty() { return messages.isEmpty(); } - @Override - public int getNumMessagesInBatch() { - return numMessagesInBatch; - } - - @Override - public long getCurrentBatchSizeBytes() { - return currentBatchSizeBytes; - } - - @Override - public void setProducer(ProducerImpl producer) { - this.producer = producer; - this.topicName = producer.getTopic(); - this.producerName = producer.getProducerName(); - this.compressionType = CompressionCodecProvider - .convertToWireProtocol(producer.getConfiguration().getCompressionType()); - this.compressor = CompressionCodecProvider.getCompressionCodec(compressionType); - this.maxNumMessagesInBatch = producer.getConfiguration().getBatchingMaxMessages(); - } - @Override public void handleException(Exception ex) { try { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java new file mode 100644 index 0000000000000..d65a315853127 --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java @@ -0,0 +1,195 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.Lists; +import io.netty.buffer.ByteBuf; +import io.netty.util.ReferenceCountUtil; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.api.ByteBufPair; +import org.apache.pulsar.common.api.Commands; +import org.apache.pulsar.common.api.proto.PulsarApi; +import org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +public class BatchMessageKeyBasedContainer extends AbstractBatchMessageContainer { + + private ConcurrentHashMap batches = new ConcurrentHashMap<>(); + + @Override + public void add(MessageImpl msg, SendCallback callback) { + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName, producerName, + numMessagesInBatch); + } + numMessagesInBatch++; + currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); + String key = peekKey(msg); + KeyBasedPart part = batches.get(key); + if (part == null) { + part = new KeyBasedPart(); + part.addMsg(msg, callback); + batches.putIfAbsent(key, part); + } else { + part.addMsg(msg, callback); + } + } + + @Override + public void clear() { + numMessagesInBatch = 0; + currentBatchSizeBytes = 0; + batches = new ConcurrentHashMap<>(); + } + + @Override + public boolean isEmpty() { + return batches.isEmpty(); + } + + @Override + public void handleException(Exception ex) { + try { + // Need to protect ourselves from any exception being thrown in the future handler from the application + batches.forEach((k, v) -> v.firstCallback.sendComplete(ex)); + } catch (Throwable t) { + log.warn("[{}] [{}] Got exception while completing the callback", topicName, producerName, t); + } + batches.forEach((k, v) -> ReferenceCountUtil.safeRelease(v.batchedMessageMetadataAndPayload)); + clear(); + } + + private ProducerImpl.OpSendMsg createOpSendMsg(KeyBasedPart keyBasedPart) throws IOException { + ByteBuf encryptedPayload = producer.encryptMessage(keyBasedPart.messageMetadata, keyBasedPart.getCompressedBatchMetadataAndPayload()); + final int numMessagesInBatch = keyBasedPart.messages.size(); + long currentBatchSizeBytes = 0; + for (MessageImpl message : keyBasedPart.messages) { + currentBatchSizeBytes += message.getDataBuffer().readableBytes(); + } + keyBasedPart.messageMetadata.setNumMessagesInBatch(numMessagesInBatch); + ByteBufPair cmd = producer.sendMessage(producer.producerId, keyBasedPart.sequenceId, numMessagesInBatch, + keyBasedPart.messageMetadata.build(), encryptedPayload); + + ProducerImpl.OpSendMsg op = ProducerImpl.OpSendMsg.create(keyBasedPart.messages, cmd, keyBasedPart.sequenceId, keyBasedPart.firstCallback); + + if (encryptedPayload.readableBytes() > ClientCnx.getMaxMessageSize()) { + cmd.release(); + if (op != null) { + op.callback.sendComplete(new PulsarClientException.InvalidMessageException( + "Message size is bigger than " + ClientCnx.getMaxMessageSize() + " bytes")); + op.recycle(); + } + return null; + } + op.setNumMessagesInBatch(numMessagesInBatch); + op.setBatchSizeByte(currentBatchSizeBytes); + return op; + } + + @Override + public List createOpSendMsgs() throws IOException { + List result = new ArrayList<>(); + List list = new ArrayList<>(batches.values()); + list.sort(((o1, o2) -> ComparisonChain.start() + .compare(o1.sequenceId, o2.sequenceId) + .result())); + for (KeyBasedPart keyBasedPart : list) { + ProducerImpl.OpSendMsg op = createOpSendMsg(keyBasedPart); + if (op != null) { + result.add(op); + } + } + return result; + } + + private String peekKey(MessageImpl msg) { + if (msg.hasOrderingKey()) { + return Base64.getEncoder().encodeToString(msg.getOrderingKey()); + } + return msg.getKey(); + } + + private class KeyBasedPart { + private PulsarApi.MessageMetadata.Builder messageMetadata = PulsarApi.MessageMetadata.newBuilder(); + // sequence id for this batch which will be persisted as a single entry by broker + private long sequenceId = -1; + private ByteBuf batchedMessageMetadataAndPayload; + private List> messages = Lists.newArrayList(); + private SendCallback previousCallback = null; + + // keep track of callbacks for individual messages being published in a batch + private SendCallback firstCallback; + + private ByteBuf getCompressedBatchMetadataAndPayload() { + for (MessageImpl msg : messages) { + PulsarApi.MessageMetadata.Builder msgBuilder = msg.getMessageBuilder(); + batchedMessageMetadataAndPayload = Commands.serializeSingleMessageInBatchWithPayload(msgBuilder, + msg.getDataBuffer(), batchedMessageMetadataAndPayload); + msgBuilder.recycle(); + } + int uncompressedSize = batchedMessageMetadataAndPayload.readableBytes(); + ByteBuf compressedPayload = compressor.encode(batchedMessageMetadataAndPayload); + batchedMessageMetadataAndPayload.release(); + if (compressionType != PulsarApi.CompressionType.NONE) { + messageMetadata.setCompression(compressionType); + messageMetadata.setUncompressedSize(uncompressedSize); + } + + // Update the current max batch size using the uncompressed size, which is what we need in any case to + // accumulate the batch content + maxBatchSize = Math.max(maxBatchSize, uncompressedSize); + return compressedPayload; + } + + private void addMsg(MessageImpl msg, SendCallback callback) { + if (messages.size() == 0) { + sequenceId = Commands.initBatchMessageMetadata(messageMetadata, msg.getMessageBuilder()); + if (msg.hasKey()) { + messageMetadata.setPartitionKey(msg.getKey()); + if (msg.hasBase64EncodedKey()) { + messageMetadata.setPartitionKeyB64Encoded(true); + } + } + if (msg.hasOrderingKey()) { + messageMetadata.setOrderingKey(ByteString.copyFrom(msg.getOrderingKey())); + } + batchedMessageMetadataAndPayload = PulsarByteBufAllocator.DEFAULT + .buffer(Math.min(maxBatchSize, MAX_MESSAGE_BATCH_SIZE_BYTES)); + firstCallback = callback; + } + if (previousCallback != null) { + previousCallback.addCallback(msg, callback); + } + previousCallback = callback; + messages.add(msg); + } + } + + private static final Logger log = LoggerFactory.getLogger(BatchMessageKeyBasedContainer.class); + +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/KeyBasedBatchMessageContainerBuilder.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/KeyBasedBatchMessageContainerBuilder.java new file mode 100644 index 0000000000000..4ad15b75b2c7d --- /dev/null +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/KeyBasedBatchMessageContainerBuilder.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import org.apache.pulsar.client.api.BatchMessageContainer; +import org.apache.pulsar.client.api.BatchMessageContainerBuilder; + +public class KeyBasedBatchMessageContainerBuilder implements BatchMessageContainerBuilder { + + @Override + public BatchMessageContainer build() { + return new BatchMessageKeyBasedContainer(); + } +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 1a108ccd1476a..340d333650f96 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -1284,10 +1284,8 @@ private void batchMessageAndSend() { } if (!batchMessageContainer.isEmpty()) { try { - for (OpSendMsg opSendMsg : batchMessageContainer.createOpSendMsgs()) { - if (opSendMsg == null) { - break; - } + List opSendMsgs = batchMessageContainer.createOpSendMsgs(); + for (OpSendMsg opSendMsg : opSendMsgs) { processOpSendMsg(opSendMsg); } } catch (PulsarClientException e) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java index eb2bcccac23ec..7af237aba545c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java @@ -67,7 +67,7 @@ public class ProducerConfigurationData implements Serializable, Cloneable { private long batchingMaxPublishDelayMicros = TimeUnit.MILLISECONDS.toMicros(1); private int batchingMaxMessages = 1000; private boolean batchingEnabled = true; // enabled by default - private BatchMessageContainerBuilder batchMessageContainerBuilder = BatchMessageContainerBuilder.DEFAULT_BUILDER; + private BatchMessageContainerBuilder batchMessageContainerBuilder = BatchMessageContainerBuilder.DEFAULT; @JsonIgnore private CryptoKeyReader cryptoKeyReader; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index 1de08cf7d8d29..ba801d90454c4 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -1158,6 +1158,9 @@ public static ByteBuf serializeSingleMessageInBatchWithPayload(PulsarApi.Message singleMessageMetadataBuilder = singleMessageMetadataBuilder.setPartitionKey(msgBuilder.getPartitionKey()) .setPartitionKeyB64Encoded(msgBuilder.getPartitionKeyB64Encoded()); } + if (msgBuilder.hasOrderingKey()) { + singleMessageMetadataBuilder = singleMessageMetadataBuilder.setOrderingKey(msgBuilder.getOrderingKey()); + } if (!msgBuilder.getPropertiesList().isEmpty()) { singleMessageMetadataBuilder = singleMessageMetadataBuilder .addAllProperties(msgBuilder.getPropertiesList()); From 199ef1335c033faf6eea815c70e195ee534d21a3 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Tue, 4 Jun 2019 15:37:32 +0800 Subject: [PATCH 05/13] Add document for batch message containers. --- .../client/api/KeySharedSubscriptionTest.java | 101 +++++++++++++----- .../client/api/BatchMessageContainer.java | 2 +- .../api/BatchMessageContainerBuilder.java | 18 ++++ .../pulsar/client/api/ProducerBuilder.java | 2 +- .../impl/AbstractBatchMessageContainer.java | 3 + .../impl/BatchMessageContainerBase.java | 3 - .../impl/BatchMessageContainerImpl.java | 9 +- .../impl/BatchMessageKeyBasedContainer.java | 11 +- 8 files changed, 113 insertions(+), 36 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java index a6f357ef8a847..96f4ecba7bd3d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java @@ -29,6 +29,7 @@ import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.util.ArrayList; @@ -37,6 +38,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.testng.Assert.assertTrue; @@ -46,6 +48,13 @@ public class KeySharedSubscriptionTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(KeySharedSubscriptionTest.class); private static final List keys = Arrays.asList("0", "1", "2", "3", "4", "5", "6", "7", "8", "9"); + @DataProvider(name = "batch") + public Object[][] batchProvider() { + return new Object[][] { + { false }, + { true } + }; + } @BeforeMethod @Override @@ -60,10 +69,10 @@ protected void cleanup() throws Exception { super.internalCleanup(); } - @Test - public void testSendAndReceiveWithHashRangeStickyKeyConsumerSelector() throws PulsarClientException { + @Test(dataProvider = "batch") + public void testSendAndReceiveWithHashRangeStickyKeyConsumerSelector(boolean enableBatch) throws PulsarClientException { this.conf.setSubscriptionKeySharedEnable(true); - String topic = "persistent://public/default/key_shared"; + String topic = "persistent://public/default/key_shared-" + UUID.randomUUID(); @Cleanup Consumer consumer1 = pulsarClient.newConsumer(Schema.INT32) @@ -90,10 +99,19 @@ public void testSendAndReceiveWithHashRangeStickyKeyConsumerSelector() throws Pu .subscribe(); @Cleanup - Producer producer = pulsarClient.newProducer(Schema.INT32) - .topic(topic) - .enableBatching(false) - .create(); + Producer producer = null; + if (enableBatch) { + producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(true) + .batchingContainerBuilder(BatchMessageContainerBuilder.KEY_BASED) + .create(); + } else { + producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(false) + .create(); + } int consumer1Slot = HashRangeStickyKeyConsumerSelector.DEFAULT_RANGE_SIZE; int consumer2Slot = consumer1Slot >> 1; @@ -129,11 +147,11 @@ public void testSendAndReceiveWithHashRangeStickyKeyConsumerSelector() throws Pu receiveAndCheck(checkList); } - @Test - public void testConsumerCrashSendAndReceiveWithHashRangeStickyKeyConsumerSelector() throws PulsarClientException, InterruptedException { + @Test(dataProvider = "batch") + public void testConsumerCrashSendAndReceiveWithHashRangeStickyKeyConsumerSelector(boolean enableBatch) throws PulsarClientException, InterruptedException { this.conf.setSubscriptionKeySharedEnable(true); - String topic = "persistent://public/default/key_shared_consumer_crash"; + String topic = "persistent://public/default/key_shared_consumer_crash-" + UUID.randomUUID(); @Cleanup Consumer consumer1 = pulsarClient.newConsumer(Schema.INT32) @@ -160,10 +178,19 @@ public void testConsumerCrashSendAndReceiveWithHashRangeStickyKeyConsumerSelecto .subscribe(); @Cleanup - Producer producer = pulsarClient.newProducer(Schema.INT32) - .topic(topic) - .enableBatching(false) - .create(); + Producer producer = null; + if (enableBatch) { + producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(true) + .batchingContainerBuilder(BatchMessageContainerBuilder.KEY_BASED) + .create(); + } else { + producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(false) + .create(); + } int consumer1Slot = HashRangeStickyKeyConsumerSelector.DEFAULT_RANGE_SIZE; int consumer2Slot = consumer1Slot >> 1; @@ -219,10 +246,10 @@ public void testConsumerCrashSendAndReceiveWithHashRangeStickyKeyConsumerSelecto } - @Test - public void testNonKeySendAndReceiveWithHashRangeStickyKeyConsumerSelector() throws PulsarClientException { + @Test(dataProvider = "batch") + public void testNonKeySendAndReceiveWithHashRangeStickyKeyConsumerSelector(boolean enableBatch) throws PulsarClientException { this.conf.setSubscriptionKeySharedEnable(true); - String topic = "persistent://public/default/key_shared_none_key"; + String topic = "persistent://public/default/key_shared_none_key-" + UUID.randomUUID(); @Cleanup Consumer consumer1 = pulsarClient.newConsumer(Schema.INT32) @@ -253,10 +280,19 @@ public void testNonKeySendAndReceiveWithHashRangeStickyKeyConsumerSelector() thr int consumer3Slot = consumer2Slot >> 1; @Cleanup - Producer producer = pulsarClient.newProducer(Schema.INT32) - .topic(topic) - .enableBatching(false) - .create(); + Producer producer = null; + if (enableBatch) { + producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(true) + .batchingContainerBuilder(BatchMessageContainerBuilder.KEY_BASED) + .create(); + } else { + producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(false) + .create(); + } for (int i = 0; i < 100; i++) { producer.newMessage() @@ -276,10 +312,10 @@ public void testNonKeySendAndReceiveWithHashRangeStickyKeyConsumerSelector() thr receiveAndCheck(checkList); } - @Test - public void testOrderingKeyWithHashRangeStickyKeyConsumerSelector() throws PulsarClientException { + @Test(dataProvider = "batch") + public void testOrderingKeyWithHashRangeStickyKeyConsumerSelector(boolean enableBatch) throws PulsarClientException { this.conf.setSubscriptionKeySharedEnable(true); - String topic = "persistent://public/default/key_shared_ordering_key"; + String topic = "persistent://public/default/key_shared_ordering_key-" + UUID.randomUUID(); @Cleanup Consumer consumer1 = pulsarClient.newConsumer(Schema.INT32) @@ -306,10 +342,19 @@ public void testOrderingKeyWithHashRangeStickyKeyConsumerSelector() throws Pulsa .subscribe(); @Cleanup - Producer producer = pulsarClient.newProducer(Schema.INT32) - .topic(topic) - .enableBatching(false) - .create(); + Producer producer = null; + if (enableBatch) { + producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(true) + .batchingContainerBuilder(BatchMessageContainerBuilder.KEY_BASED) + .create(); + } else { + producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(false) + .create(); + } int consumer1Slot = HashRangeStickyKeyConsumerSelector.DEFAULT_RANGE_SIZE; int consumer2Slot = consumer1Slot >> 1; diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainer.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainer.java index f9d58f96710cb..186c6fd425626 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainer.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainer.java @@ -19,7 +19,7 @@ package org.apache.pulsar.client.api; /** - * container for individual messages being published until they are batched and sent to broker + * Batch message container for individual messages being published until they are batched and sent to broker */ public interface BatchMessageContainer { diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java index b28a56da61af9..e3e39c91790a7 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java @@ -25,8 +25,26 @@ */ public interface BatchMessageContainerBuilder { + /** + * Default batch message container + * + * incoming single messages: + * (k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3) + * + * batched into single batch message: + * [(k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3)] + */ BatchMessageContainerBuilder DEFAULT = DefaultImplementation.newDefaultBatchMessageContainerBuilder(); + /** + * Key based batch message container + * + * incoming single messages: + * (k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3) + * + * batched into multiple batch messages: + * [(k1, v1), (k1, v2), (k1, v3)], [(k2, v1), (k2, v2), (k2, v3)], [(k3, v1), (k3, v2), (k3, v3)] + */ BatchMessageContainerBuilder KEY_BASED = DefaultImplementation.newKeyBasedBatchMessageContainerBuilder(); /** diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java index cb6ac7dc861a9..d833968d0e992 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java @@ -366,7 +366,7 @@ public interface ProducerBuilder extends Cloneable { ProducerBuilder batchingMaxMessages(int batchMessagesMaxMessagesPerBatch); /** - * Set the batch message container builder of the producer. Producer will use the batch message container builder to + * Set the batch message container builder {@link BatchMessageContainerBuilder} of the producer. Producer will use the batch message container builder to * build a batch message container. * * @param batchingContainerBuilder diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java index f5d89477cc796..546638bafb2a6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java @@ -22,6 +22,9 @@ import org.apache.pulsar.common.compression.CompressionCodec; import org.apache.pulsar.common.compression.CompressionCodecProvider; +/** + * Batch message container framework. + */ public abstract class AbstractBatchMessageContainer implements BatchMessageContainerBase { protected PulsarApi.CompressionType compressionType; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java index 930c97c3fb919..337ee0db9ef95 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java @@ -24,9 +24,6 @@ import java.io.IOException; import java.util.List; -/** - * container for individual messages being published until they are batched and sent to broker - */ public interface BatchMessageContainerBase extends BatchMessageContainer { /** diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java index f78c2eae34873..ac36954f2fff9 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -37,9 +37,14 @@ import org.slf4j.LoggerFactory; /** - * container for individual messages being published until they are batched and sent to broker + * Default batch message container + * + * incoming single messages: + * (k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3) + * + * batched into single batch message: + * [(k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3)] */ - class BatchMessageContainerImpl extends AbstractBatchMessageContainer { private PulsarApi.MessageMetadata.Builder messageMetadata = PulsarApi.MessageMetadata.newBuilder(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java index d65a315853127..467264aabbd5e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java @@ -37,7 +37,16 @@ import java.util.List; import java.util.concurrent.ConcurrentHashMap; -public class BatchMessageKeyBasedContainer extends AbstractBatchMessageContainer { +/** + * Key based batch message container + * + * incoming single messages: + * (k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3) + * + * batched into multiple batch messages: + * [(k1, v1), (k1, v2), (k1, v3)], [(k2, v1), (k2, v2), (k2, v3)], [(k3, v1), (k3, v2), (k3, v3)] + */ +class BatchMessageKeyBasedContainer extends AbstractBatchMessageContainer { private ConcurrentHashMap batches = new ConcurrentHashMap<>(); From b5c341bd6f9fbfcc8361028fe10483787512803e Mon Sep 17 00:00:00 2001 From: lipenghui Date: Tue, 4 Jun 2019 15:53:37 +0800 Subject: [PATCH 06/13] Fix configuration data test issue. --- .../pulsar/client/impl/conf/ConfigurationDataUtilsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java index 626c501015fc1..df150224701a1 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java @@ -59,6 +59,7 @@ public void testLoadProducerConfigurationData() { Map config = new HashMap<>(); config.put("producerName", "test-producer"); config.put("batchingEnabled", false); + confData.setBatchMessageContainerBuilder(null); confData = ConfigurationDataUtils.loadData(config, confData, ProducerConfigurationData.class); assertEquals("test-producer", confData.getProducerName()); assertEquals(false, confData.isBatchingEnabled()); From 9b648f00a488a1572c8522b74dcea69d0c05d505 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Wed, 5 Jun 2019 10:51:41 +0800 Subject: [PATCH 07/13] Fix storm UT issue --- .../java/org/apache/pulsar/client/impl/ProducerImpl.java | 7 ++++++- .../src/main/java/org/apache/pulsar/storm/PulsarBolt.java | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 340d333650f96..a26dff8f8c789 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -48,6 +48,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLongFieldUpdater; +import org.apache.pulsar.client.api.BatchMessageContainerBuilder; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; @@ -163,7 +164,11 @@ public ProducerImpl(PulsarClientImpl client, String topic, ProducerConfiguration this.createProducerTimeout = System.currentTimeMillis() + client.getConfiguration().getOperationTimeoutMs(); if (conf.isBatchingEnabled()) { this.maxNumMessagesInBatch = conf.getBatchingMaxMessages(); - this.batchMessageContainer = (BatchMessageContainerBase)conf.getBatchMessageContainerBuilder().build(); + BatchMessageContainerBuilder containerBuilder = conf.getBatchMessageContainerBuilder(); + if (containerBuilder == null) { + containerBuilder = BatchMessageContainerBuilder.DEFAULT; + } + this.batchMessageContainer = (BatchMessageContainerBase)containerBuilder.build(); this.batchMessageContainer.setProducer(this); } else { this.maxNumMessagesInBatch = 1; diff --git a/pulsar-storm/src/main/java/org/apache/pulsar/storm/PulsarBolt.java b/pulsar-storm/src/main/java/org/apache/pulsar/storm/PulsarBolt.java index 8432b1c38d084..8cdcbae8ea844 100644 --- a/pulsar-storm/src/main/java/org/apache/pulsar/storm/PulsarBolt.java +++ b/pulsar-storm/src/main/java/org/apache/pulsar/storm/PulsarBolt.java @@ -91,6 +91,7 @@ public PulsarBolt(PulsarBoltConfiguration pulsarBoltConf, ClientConfigurationDat this.producerConf = producerConf; this.clientConf.setServiceUrl(pulsarBoltConf.getServiceUrl()); this.producerConf.setTopicName(pulsarBoltConf.getTopic()); + this.producerConf.setBatchMessageContainerBuilder(null); } @SuppressWarnings({ "rawtypes" }) From 1657d24d85bca391c2dab6bbe10bbda45b53e4c9 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Wed, 5 Jun 2019 20:07:26 +0800 Subject: [PATCH 08/13] fix review comments. --- .../client/api/BatchMessageContainer.java | 10 +++- .../pulsar/client/api/ProducerBuilder.java | 2 +- .../impl/AbstractBatchMessageContainer.java | 15 +++++- .../impl/BatchMessageContainerBase.java | 8 +++ .../impl/BatchMessageContainerImpl.java | 15 +++--- .../impl/BatchMessageKeyBasedContainer.java | 50 ++++++++++++------- .../pulsar/client/impl/ProducerImpl.java | 17 +++++-- 7 files changed, 82 insertions(+), 35 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainer.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainer.java index 186c6fd425626..cf7296255de7d 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainer.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainer.java @@ -47,12 +47,18 @@ public interface BatchMessageContainer { * * @return message batch size in bytes */ - long getCurrentBatchSizeBytes(); + long getCurrentBatchSize(); /** * Release the payload and clear the container. * * @param ex cause */ - void handleException(Exception ex); + void discard(Exception ex); + + /** + * Return the batch container batch message in multiple batches + * @return + */ + boolean isMultiBatches(); } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java index d833968d0e992..f62d3ffe596bb 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java @@ -367,7 +367,7 @@ public interface ProducerBuilder extends Cloneable { /** * Set the batch message container builder {@link BatchMessageContainerBuilder} of the producer. Producer will use the batch message container builder to - * build a batch message container. + * build a batch message container.This is only be used when batching is enabled * * @param batchingContainerBuilder * batch message container builder diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java index 546638bafb2a6..3d6ca4a28584c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java @@ -22,6 +22,9 @@ import org.apache.pulsar.common.compression.CompressionCodec; import org.apache.pulsar.common.compression.CompressionCodecProvider; +import java.io.IOException; +import java.util.List; + /** * Batch message container framework. */ @@ -57,10 +60,20 @@ public int getNumMessagesInBatch() { } @Override - public long getCurrentBatchSizeBytes() { + public long getCurrentBatchSize() { return currentBatchSizeBytes; } + @Override + public List createOpSendMsgs() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public ProducerImpl.OpSendMsg createOpSendMsg() throws IOException { + throw new UnsupportedOperationException(); + } + @Override public void setProducer(ProducerImpl producer) { this.producer = producer; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java index 337ee0db9ef95..5f930bde580e8 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerBase.java @@ -57,4 +57,12 @@ public interface BatchMessageContainerBase extends BatchMessageContainer { * @throws IOException */ List createOpSendMsgs() throws IOException; + + /** + * Create OpSendMsg, producer use OpSendMsg to send to the broker. + * + * @return OpSendMsg + * @throws IOException + */ + OpSendMsg createOpSendMsg() throws IOException; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java index ac36954f2fff9..720480a9de7e4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -120,7 +120,7 @@ public boolean isEmpty() { } @Override - public void handleException(Exception ex) { + public void discard(Exception ex) { try { // Need to protect ourselves from any exception being thrown in the future handler from the application firstCallback.sendComplete(ex); @@ -132,7 +132,13 @@ public void handleException(Exception ex) { clear(); } - private OpSendMsg createOpSendMsg() throws IOException { + @Override + public boolean isMultiBatches() { + return false; + } + + @Override + public OpSendMsg createOpSendMsg() throws IOException { ByteBuf encryptedPayload = producer.encryptMessage(messageMetadata, getCompressedBatchMetadataAndPayload()); messageMetadata.setNumMessagesInBatch(numMessagesInBatch); ByteBufPair cmd = producer.sendMessage(producer.producerId, sequenceId, numMessagesInBatch, @@ -155,10 +161,5 @@ private OpSendMsg createOpSendMsg() throws IOException { return op; } - @Override - public List createOpSendMsgs() throws IOException { - return Collections.singletonList(createOpSendMsg()); - } - private static final Logger log = LoggerFactory.getLogger(BatchMessageContainerImpl.class); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java index 467264aabbd5e..614ada4299be8 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java @@ -27,6 +27,7 @@ import org.apache.pulsar.common.api.ByteBufPair; import org.apache.pulsar.common.api.Commands; import org.apache.pulsar.common.api.proto.PulsarApi; +import org.apache.pulsar.common.compression.CompressionCodec; import org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,21 +49,24 @@ */ class BatchMessageKeyBasedContainer extends AbstractBatchMessageContainer { - private ConcurrentHashMap batches = new ConcurrentHashMap<>(); + private ConcurrentHashMap batches = new ConcurrentHashMap<>(); @Override public void add(MessageImpl msg, SendCallback callback) { if (log.isDebugEnabled()) { - log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName, producerName, + log.debug("[{}] [{}] add message to batch, num messages in batch so far is {}", topicName, producerName, numMessagesInBatch); } numMessagesInBatch++; currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); - String key = peekKey(msg); - KeyBasedPart part = batches.get(key); + String key = getKey(msg); + KeyedBatch part = batches.get(key); if (part == null) { - part = new KeyBasedPart(); + part = new KeyedBatch(); part.addMsg(msg, callback); + part.compressionType = compressionType; + part.compressor = compressor; + part.maxBatchSize = maxBatchSize; batches.putIfAbsent(key, part); } else { part.addMsg(msg, callback); @@ -82,7 +86,7 @@ public boolean isEmpty() { } @Override - public void handleException(Exception ex) { + public void discard(Exception ex) { try { // Need to protect ourselves from any exception being thrown in the future handler from the application batches.forEach((k, v) -> v.firstCallback.sendComplete(ex)); @@ -93,18 +97,23 @@ public void handleException(Exception ex) { clear(); } - private ProducerImpl.OpSendMsg createOpSendMsg(KeyBasedPart keyBasedPart) throws IOException { - ByteBuf encryptedPayload = producer.encryptMessage(keyBasedPart.messageMetadata, keyBasedPart.getCompressedBatchMetadataAndPayload()); - final int numMessagesInBatch = keyBasedPart.messages.size(); + @Override + public boolean isMultiBatches() { + return true; + } + + private ProducerImpl.OpSendMsg createOpSendMsg(KeyedBatch keyedBatch) throws IOException { + ByteBuf encryptedPayload = producer.encryptMessage(keyedBatch.messageMetadata, keyedBatch.getCompressedBatchMetadataAndPayload()); + final int numMessagesInBatch = keyedBatch.messages.size(); long currentBatchSizeBytes = 0; - for (MessageImpl message : keyBasedPart.messages) { + for (MessageImpl message : keyedBatch.messages) { currentBatchSizeBytes += message.getDataBuffer().readableBytes(); } - keyBasedPart.messageMetadata.setNumMessagesInBatch(numMessagesInBatch); - ByteBufPair cmd = producer.sendMessage(producer.producerId, keyBasedPart.sequenceId, numMessagesInBatch, - keyBasedPart.messageMetadata.build(), encryptedPayload); + keyedBatch.messageMetadata.setNumMessagesInBatch(numMessagesInBatch); + ByteBufPair cmd = producer.sendMessage(producer.producerId, keyedBatch.sequenceId, numMessagesInBatch, + keyedBatch.messageMetadata.build(), encryptedPayload); - ProducerImpl.OpSendMsg op = ProducerImpl.OpSendMsg.create(keyBasedPart.messages, cmd, keyBasedPart.sequenceId, keyBasedPart.firstCallback); + ProducerImpl.OpSendMsg op = ProducerImpl.OpSendMsg.create(keyedBatch.messages, cmd, keyedBatch.sequenceId, keyedBatch.firstCallback); if (encryptedPayload.readableBytes() > ClientCnx.getMaxMessageSize()) { cmd.release(); @@ -123,12 +132,12 @@ private ProducerImpl.OpSendMsg createOpSendMsg(KeyBasedPart keyBasedPart) throws @Override public List createOpSendMsgs() throws IOException { List result = new ArrayList<>(); - List list = new ArrayList<>(batches.values()); + List list = new ArrayList<>(batches.values()); list.sort(((o1, o2) -> ComparisonChain.start() .compare(o1.sequenceId, o2.sequenceId) .result())); - for (KeyBasedPart keyBasedPart : list) { - ProducerImpl.OpSendMsg op = createOpSendMsg(keyBasedPart); + for (KeyedBatch keyedBatch : list) { + ProducerImpl.OpSendMsg op = createOpSendMsg(keyedBatch); if (op != null) { result.add(op); } @@ -136,20 +145,23 @@ public List createOpSendMsgs() throws IOException { return result; } - private String peekKey(MessageImpl msg) { + private String getKey(MessageImpl msg) { if (msg.hasOrderingKey()) { return Base64.getEncoder().encodeToString(msg.getOrderingKey()); } return msg.getKey(); } - private class KeyBasedPart { + private static class KeyedBatch { private PulsarApi.MessageMetadata.Builder messageMetadata = PulsarApi.MessageMetadata.newBuilder(); // sequence id for this batch which will be persisted as a single entry by broker private long sequenceId = -1; private ByteBuf batchedMessageMetadataAndPayload; private List> messages = Lists.newArrayList(); private SendCallback previousCallback = null; + private PulsarApi.CompressionType compressionType; + private CompressionCodec compressor; + private int maxBatchSize; // keep track of callbacks for individual messages being published in a batch private SendCallback firstCallback; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index a26dff8f8c789..55ac3d3b6d2fd 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -373,7 +373,7 @@ public void sendAsync(Message message, SendCallback callback) { lastSendFuture = callback.getFuture(); payload.release(); if (batchMessageContainer.getNumMessagesInBatch() == maxNumMessagesInBatch - || batchMessageContainer.getCurrentBatchSizeBytes() >= BatchMessageContainerImpl.MAX_MESSAGE_BATCH_SIZE_BYTES) { + || batchMessageContainer.getCurrentBatchSize() >= BatchMessageContainerImpl.MAX_MESSAGE_BATCH_SIZE_BYTES) { batchMessageAndSend(); } } else { @@ -1231,7 +1231,7 @@ private void failPendingBatchMessages(PulsarClientException ex) { } int numMessagesInBatch = batchMessageContainer.getNumMessagesInBatch(); semaphore.release(numMessagesInBatch); - batchMessageContainer.handleException(ex); + batchMessageContainer.discard(ex); } TimerTask batchMessageAndSendTask = new TimerTask() { @@ -1289,9 +1289,16 @@ private void batchMessageAndSend() { } if (!batchMessageContainer.isEmpty()) { try { - List opSendMsgs = batchMessageContainer.createOpSendMsgs(); - for (OpSendMsg opSendMsg : opSendMsgs) { - processOpSendMsg(opSendMsg); + if (batchMessageContainer.isMultiBatches()) { + List opSendMsgs = batchMessageContainer.createOpSendMsgs(); + for (OpSendMsg opSendMsg : opSendMsgs) { + processOpSendMsg(opSendMsg); + } + } else { + OpSendMsg opSendMsg = batchMessageContainer.createOpSendMsg(); + if (opSendMsg != null) { + processOpSendMsg(opSendMsg); + } } } catch (PulsarClientException e) { Thread.currentThread().interrupt(); From 3a6e78854e196f03f5802bfd1118ce0bc7df1be1 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Wed, 5 Jun 2019 20:16:08 +0800 Subject: [PATCH 09/13] clean code of UT. --- .../client/api/KeySharedSubscriptionTest.java | 170 +++++------------- 1 file changed, 44 insertions(+), 126 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java index 96f4ecba7bd3d..ec8c1aaeabf7c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java @@ -75,43 +75,16 @@ public void testSendAndReceiveWithHashRangeStickyKeyConsumerSelector(boolean ena String topic = "persistent://public/default/key_shared-" + UUID.randomUUID(); @Cleanup - Consumer consumer1 = pulsarClient.newConsumer(Schema.INT32) - .topic(topic) - .subscriptionName("key_shared") - .subscriptionType(SubscriptionType.Key_Shared) - .ackTimeout(3, TimeUnit.SECONDS) - .subscribe(); + Consumer consumer1 = createConsumer(topic); @Cleanup - Consumer consumer2 = pulsarClient.newConsumer(Schema.INT32) - .topic(topic) - .subscriptionName("key_shared") - .subscriptionType(SubscriptionType.Key_Shared) - .ackTimeout(3, TimeUnit.SECONDS) - .subscribe(); + Consumer consumer2 = createConsumer(topic); @Cleanup - Consumer consumer3 = pulsarClient.newConsumer(Schema.INT32) - .topic(topic) - .subscriptionName("key_shared") - .subscriptionType(SubscriptionType.Key_Shared) - .ackTimeout(3, TimeUnit.SECONDS) - .subscribe(); + Consumer consumer3 = createConsumer(topic); @Cleanup - Producer producer = null; - if (enableBatch) { - producer = pulsarClient.newProducer(Schema.INT32) - .topic(topic) - .enableBatching(true) - .batchingContainerBuilder(BatchMessageContainerBuilder.KEY_BASED) - .create(); - } else { - producer = pulsarClient.newProducer(Schema.INT32) - .topic(topic) - .enableBatching(false) - .create(); - } + Producer producer = createProducer(topic, enableBatch); int consumer1Slot = HashRangeStickyKeyConsumerSelector.DEFAULT_RANGE_SIZE; int consumer2Slot = consumer1Slot >> 1; @@ -154,43 +127,16 @@ public void testConsumerCrashSendAndReceiveWithHashRangeStickyKeyConsumerSelecto String topic = "persistent://public/default/key_shared_consumer_crash-" + UUID.randomUUID(); @Cleanup - Consumer consumer1 = pulsarClient.newConsumer(Schema.INT32) - .topic(topic) - .subscriptionName("key_shared") - .subscriptionType(SubscriptionType.Key_Shared) - .ackTimeout(3, TimeUnit.SECONDS) - .subscribe(); + Consumer consumer1 = createConsumer(topic); @Cleanup - Consumer consumer2 = pulsarClient.newConsumer(Schema.INT32) - .topic(topic) - .subscriptionName("key_shared") - .subscriptionType(SubscriptionType.Key_Shared) - .ackTimeout(3, TimeUnit.SECONDS) - .subscribe(); + Consumer consumer2 = createConsumer(topic); @Cleanup - Consumer consumer3 = pulsarClient.newConsumer(Schema.INT32) - .topic(topic) - .subscriptionName("key_shared") - .subscriptionType(SubscriptionType.Key_Shared) - .ackTimeout(3, TimeUnit.SECONDS) - .subscribe(); + Consumer consumer3 = createConsumer(topic); @Cleanup - Producer producer = null; - if (enableBatch) { - producer = pulsarClient.newProducer(Schema.INT32) - .topic(topic) - .enableBatching(true) - .batchingContainerBuilder(BatchMessageContainerBuilder.KEY_BASED) - .create(); - } else { - producer = pulsarClient.newProducer(Schema.INT32) - .topic(topic) - .enableBatching(false) - .create(); - } + Producer producer = createProducer(topic, enableBatch); int consumer1Slot = HashRangeStickyKeyConsumerSelector.DEFAULT_RANGE_SIZE; int consumer2Slot = consumer1Slot >> 1; @@ -252,48 +198,21 @@ public void testNonKeySendAndReceiveWithHashRangeStickyKeyConsumerSelector(boole String topic = "persistent://public/default/key_shared_none_key-" + UUID.randomUUID(); @Cleanup - Consumer consumer1 = pulsarClient.newConsumer(Schema.INT32) - .topic(topic) - .subscriptionName("key_shared") - .subscriptionType(SubscriptionType.Key_Shared) - .ackTimeout(3, TimeUnit.SECONDS) - .subscribe(); + Consumer consumer1 = createConsumer(topic); @Cleanup - Consumer consumer2 = pulsarClient.newConsumer(Schema.INT32) - .topic(topic) - .subscriptionName("key_shared") - .subscriptionType(SubscriptionType.Key_Shared) - .ackTimeout(3, TimeUnit.SECONDS) - .subscribe(); + Consumer consumer2 = createConsumer(topic); @Cleanup - Consumer consumer3 = pulsarClient.newConsumer(Schema.INT32) - .topic(topic) - .subscriptionName("key_shared") - .subscriptionType(SubscriptionType.Key_Shared) - .ackTimeout(3, TimeUnit.SECONDS) - .subscribe(); + Consumer consumer3 = createConsumer(topic); + + @Cleanup + Producer producer = createProducer(topic, enableBatch); int consumer1Slot = HashRangeStickyKeyConsumerSelector.DEFAULT_RANGE_SIZE; int consumer2Slot = consumer1Slot >> 1; int consumer3Slot = consumer2Slot >> 1; - @Cleanup - Producer producer = null; - if (enableBatch) { - producer = pulsarClient.newProducer(Schema.INT32) - .topic(topic) - .enableBatching(true) - .batchingContainerBuilder(BatchMessageContainerBuilder.KEY_BASED) - .create(); - } else { - producer = pulsarClient.newProducer(Schema.INT32) - .topic(topic) - .enableBatching(false) - .create(); - } - for (int i = 0; i < 100; i++) { producer.newMessage() .value(i) @@ -318,43 +237,16 @@ public void testOrderingKeyWithHashRangeStickyKeyConsumerSelector(boolean enable String topic = "persistent://public/default/key_shared_ordering_key-" + UUID.randomUUID(); @Cleanup - Consumer consumer1 = pulsarClient.newConsumer(Schema.INT32) - .topic(topic) - .subscriptionName("key_shared") - .subscriptionType(SubscriptionType.Key_Shared) - .ackTimeout(3, TimeUnit.SECONDS) - .subscribe(); + Consumer consumer1 = createConsumer(topic); @Cleanup - Consumer consumer2 = pulsarClient.newConsumer(Schema.INT32) - .topic(topic) - .subscriptionName("key_shared") - .subscriptionType(SubscriptionType.Key_Shared) - .ackTimeout(3, TimeUnit.SECONDS) - .subscribe(); + Consumer consumer2 = createConsumer(topic); @Cleanup - Consumer consumer3 = pulsarClient.newConsumer(Schema.INT32) - .topic(topic) - .subscriptionName("key_shared") - .subscriptionType(SubscriptionType.Key_Shared) - .ackTimeout(3, TimeUnit.SECONDS) - .subscribe(); + Consumer consumer3 = createConsumer(topic); @Cleanup - Producer producer = null; - if (enableBatch) { - producer = pulsarClient.newProducer(Schema.INT32) - .topic(topic) - .enableBatching(true) - .batchingContainerBuilder(BatchMessageContainerBuilder.KEY_BASED) - .create(); - } else { - producer = pulsarClient.newProducer(Schema.INT32) - .topic(topic) - .enableBatching(false) - .create(); - } + Producer producer = createProducer(topic, enableBatch); int consumer1Slot = HashRangeStickyKeyConsumerSelector.DEFAULT_RANGE_SIZE; int consumer2Slot = consumer1Slot >> 1; @@ -403,6 +295,32 @@ public void testDisableKeySharedSubscription() throws PulsarClientException { .subscribe(); } + private Producer createProducer(String topic, boolean enableBatch) throws PulsarClientException { + Producer producer = null; + if (enableBatch) { + producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(true) + .batchingContainerBuilder(BatchMessageContainerBuilder.KEY_BASED) + .create(); + } else { + producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(false) + .create(); + } + return producer; + } + + private Consumer createConsumer(String topic) throws PulsarClientException { + return pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName("key_shared") + .subscriptionType(SubscriptionType.Key_Shared) + .ackTimeout(3, TimeUnit.SECONDS) + .subscribe(); + } + private void receiveAndCheck(List, Integer>> checkList) throws PulsarClientException { Map> consumerKeys = new HashMap<>(); for (KeyValue, Integer> check : checkList) { From c8810ff98e0cc399e3aee9d9a798964429b6c534 Mon Sep 17 00:00:00 2001 From: lipenghui Date: Mon, 10 Jun 2019 11:12:48 +0800 Subject: [PATCH 10/13] fix review comments. --- .../broker/service/BatchMessageTest.java | 60 +++++++++---------- .../client/api/KeySharedSubscriptionTest.java | 2 +- ...tainerBuilder.java => BatcherBuilder.java} | 8 +-- .../pulsar/client/api/ProducerBuilder.java | 10 ++-- .../internal/DefaultImplementation.java | 10 ++-- .../impl/BatchMessageContainerImpl.java | 6 +- .../impl/BatchMessageKeyBasedContainer.java | 8 ++- ...uilder.java => DefaultBatcherBuilder.java} | 4 +- ...ilder.java => KeyBasedBatcherBuilder.java} | 4 +- .../client/impl/ProducerBuilderImpl.java | 6 +- .../pulsar/client/impl/ProducerImpl.java | 6 +- .../impl/conf/ProducerConfigurationData.java | 5 +- .../impl/conf/ConfigurationDataUtilsTest.java | 2 +- .../org/apache/pulsar/storm/PulsarBolt.java | 2 +- 14 files changed, 66 insertions(+), 67 deletions(-) rename pulsar-client-api/src/main/java/org/apache/pulsar/client/api/{BatchMessageContainerBuilder.java => BatcherBuilder.java} (85%) rename pulsar-client/src/main/java/org/apache/pulsar/client/impl/{DefaultBatchMessageContainerBuilder.java => DefaultBatcherBuilder.java} (86%) rename pulsar-client/src/main/java/org/apache/pulsar/client/impl/{KeyBasedBatchMessageContainerBuilder.java => KeyBasedBatcherBuilder.java} (86%) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java index b38c2c502be80..265e3ecba6e8f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageTest.java @@ -40,7 +40,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentTopic; -import org.apache.pulsar.client.api.BatchMessageContainerBuilder; +import org.apache.pulsar.client.api.BatcherBuilder; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -75,25 +75,25 @@ protected void cleanup() throws Exception { @DataProvider(name = "codecAndContainerBuilder") public Object[][] codecAndContainerBuilderProvider() { return new Object[][] { - { CompressionType.NONE, BatchMessageContainerBuilder.DEFAULT }, - { CompressionType.LZ4, BatchMessageContainerBuilder.DEFAULT }, - { CompressionType.ZLIB, BatchMessageContainerBuilder.DEFAULT }, - { CompressionType.NONE, BatchMessageContainerBuilder.KEY_BASED }, - { CompressionType.LZ4, BatchMessageContainerBuilder.KEY_BASED }, - { CompressionType.ZLIB, BatchMessageContainerBuilder.KEY_BASED } + { CompressionType.NONE, BatcherBuilder.DEFAULT }, + { CompressionType.LZ4, BatcherBuilder.DEFAULT }, + { CompressionType.ZLIB, BatcherBuilder.DEFAULT }, + { CompressionType.NONE, BatcherBuilder.KEY_BASED }, + { CompressionType.LZ4, BatcherBuilder.KEY_BASED }, + { CompressionType.ZLIB, BatcherBuilder.KEY_BASED } }; } @DataProvider(name = "containerBuilder") public Object[][] containerBuilderProvider() { return new Object[][] { - { BatchMessageContainerBuilder.DEFAULT }, - { BatchMessageContainerBuilder.KEY_BASED } + { BatcherBuilder.DEFAULT }, + { BatcherBuilder.KEY_BASED } }; } @Test(dataProvider = "codecAndContainerBuilder") - public void testSimpleBatchProducerWithFixedBatchSize(CompressionType compressionType, BatchMessageContainerBuilder builder) throws Exception { + public void testSimpleBatchProducerWithFixedBatchSize(CompressionType compressionType, BatcherBuilder builder) throws Exception { int numMsgs = 50; int numMsgsInBatch = numMsgs / 2; final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerWithFixedBatchSize-" + UUID.randomUUID(); @@ -105,7 +105,7 @@ public void testSimpleBatchProducerWithFixedBatchSize(CompressionType compressio Producer producer = pulsarClient.newProducer().topic(topicName).compressionType(compressionType) .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true) - .batchingContainerBuilder(builder) + .batcherBuilder(builder) .create(); List> sendFutureList = Lists.newArrayList(); @@ -137,7 +137,7 @@ public void testSimpleBatchProducerWithFixedBatchSize(CompressionType compressio } @Test(dataProvider = "codecAndContainerBuilder") - public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressionType, BatchMessageContainerBuilder builder) throws Exception { + public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressionType, BatcherBuilder builder) throws Exception { int numMsgs = 100; final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerWithFixedBatchTime-" + UUID.randomUUID(); final String subscriptionName = "time-sub-1" + compressionType.toString(); @@ -148,7 +148,7 @@ public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressio Producer producer = pulsarClient.newProducer().topic(topicName).compressionType(compressionType) .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS).enableBatching(true) - .batchingContainerBuilder(builder) + .batcherBuilder(builder) .create(); Random random = new Random(); @@ -173,7 +173,7 @@ public void testSimpleBatchProducerWithFixedBatchTime(CompressionType compressio } @Test(dataProvider = "codecAndContainerBuilder") - public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType compressionType, BatchMessageContainerBuilder builder) throws Exception { + public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType compressionType, BatcherBuilder builder) throws Exception { int numMsgs = 100; final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerWithFixedBatchSizeAndTime-" + UUID.randomUUID(); final String subscriptionName = "time-size-sub-1" + compressionType.toString(); @@ -184,7 +184,7 @@ public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType com Producer producer = pulsarClient.newProducer().topic(topicName) .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS).batchingMaxMessages(5) - .batchingContainerBuilder(builder) + .batcherBuilder(builder) .compressionType(compressionType).enableBatching(true).create(); Random random = new Random(); @@ -209,7 +209,7 @@ public void testSimpleBatchProducerWithFixedBatchSizeAndTime(CompressionType com } @Test(dataProvider = "codecAndContainerBuilder") - public void testBatchProducerWithLargeMessage(CompressionType compressionType, BatchMessageContainerBuilder builder) throws Exception { + public void testBatchProducerWithLargeMessage(CompressionType compressionType, BatcherBuilder builder) throws Exception { int numMsgs = 50; int numMsgsInBatch = numMsgs / 2; final String topicName = "persistent://prop/ns-abc/testBatchProducerWithLargeMessage-" + UUID.randomUUID(); @@ -221,7 +221,7 @@ public void testBatchProducerWithLargeMessage(CompressionType compressionType, B Producer producer = pulsarClient.newProducer().topic(topicName).compressionType(compressionType) .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true) - .batchingContainerBuilder(builder) + .batcherBuilder(builder) .create(); List> sendFutureList = Lists.newArrayList(); @@ -262,7 +262,7 @@ public void testBatchProducerWithLargeMessage(CompressionType compressionType, B } @Test(dataProvider = "codecAndContainerBuilder") - public void testSimpleBatchProducerConsumer(CompressionType compressionType, BatchMessageContainerBuilder builder) throws Exception { + public void testSimpleBatchProducerConsumer(CompressionType compressionType, BatcherBuilder builder) throws Exception { int numMsgs = 500; int numMsgsInBatch = numMsgs / 20; final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerConsumer-" + UUID.randomUUID(); @@ -280,7 +280,7 @@ public void testSimpleBatchProducerConsumer(CompressionType compressionType, Bat // disabled size based batch .batchingMaxMessages(2 * numMsgs) .enableBatching(true) - .batchingContainerBuilder(builder) + .batcherBuilder(builder) .create(); List> sendFutureList = Lists.newArrayList(); @@ -321,7 +321,7 @@ public void testSimpleBatchProducerConsumer(CompressionType compressionType, Bat } @Test(dataProvider = "containerBuilder") - public void testSimpleBatchSyncProducerWithFixedBatchSize(BatchMessageContainerBuilder builder) throws Exception { + public void testSimpleBatchSyncProducerWithFixedBatchSize(BatcherBuilder builder) throws Exception { int numMsgs = 10; int numMsgsInBatch = numMsgs / 2; final String topicName = "persistent://prop/ns-abc/testSimpleBatchSyncProducerWithFixedBatchSize-" + UUID.randomUUID(); @@ -333,7 +333,7 @@ public void testSimpleBatchSyncProducerWithFixedBatchSize(BatchMessageContainerB Producer producer = pulsarClient.newProducer().topic(topicName) .batchingMaxPublishDelay(1, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true) - .batchingContainerBuilder(builder) + .batcherBuilder(builder) .create(); for (int i = 0; i < numMsgs; i++) { @@ -364,7 +364,7 @@ public void testSimpleBatchSyncProducerWithFixedBatchSize(BatchMessageContainerB } @Test(dataProvider = "containerBuilder") - public void testSimpleBatchProducerConsumer1kMessages(BatchMessageContainerBuilder builder) throws Exception { + public void testSimpleBatchProducerConsumer1kMessages(BatcherBuilder builder) throws Exception { int numMsgs = 2000; int numMsgsInBatch = 4; final String topicName = "persistent://prop/ns-abc/testSimpleBatchProducerConsumer1kMessages-" + UUID.randomUUID(); @@ -376,7 +376,7 @@ public void testSimpleBatchProducerConsumer1kMessages(BatchMessageContainerBuild Producer producer = pulsarClient.newProducer().topic(topicName).maxPendingMessages(numMsgs + 1) .batchingMaxPublishDelay(30, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true) - .batchingContainerBuilder(builder) + .batcherBuilder(builder) .create(); List> sendFutureList = Lists.newArrayList(); @@ -492,7 +492,7 @@ public void testOutOfOrderAcksForBatchMessage() throws Exception { } @Test(dataProvider = "containerBuilder") - public void testNonBatchCumulativeAckAfterBatchPublish(BatchMessageContainerBuilder builder) throws Exception { + public void testNonBatchCumulativeAckAfterBatchPublish(BatcherBuilder builder) throws Exception { int numMsgs = 10; int numMsgsInBatch = numMsgs; final String topicName = "persistent://prop/ns-abc/testNonBatchCumulativeAckAfterBatchPublish-" + UUID.randomUUID(); @@ -504,7 +504,7 @@ public void testNonBatchCumulativeAckAfterBatchPublish(BatchMessageContainerBuil Producer producer = pulsarClient.newProducer().topic(topicName) .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgsInBatch).enableBatching(true) - .batchingContainerBuilder(builder) + .batcherBuilder(builder) .create(); // create producer to publish non batch messages Producer noBatchProducer = pulsarClient.newProducer().topic(topicName).create(); @@ -545,7 +545,7 @@ public void testNonBatchCumulativeAckAfterBatchPublish(BatchMessageContainerBuil } @Test(dataProvider = "containerBuilder") - public void testBatchAndNonBatchCumulativeAcks(BatchMessageContainerBuilder builder) throws Exception { + public void testBatchAndNonBatchCumulativeAcks(BatcherBuilder builder) throws Exception { int numMsgs = 50; int numMsgsInBatch = numMsgs / 10; final String topicName = "persistent://prop/ns-abc/testBatchAndNonBatchCumulativeAcks-" + UUID.randomUUID(); @@ -559,7 +559,7 @@ public void testBatchAndNonBatchCumulativeAcks(BatchMessageContainerBuilder buil .batchingMaxPublishDelay(5, TimeUnit.SECONDS) .batchingMaxMessages(numMsgsInBatch) .enableBatching(true) - .batchingContainerBuilder(builder) + .batcherBuilder(builder) .messageRoutingMode(MessageRoutingMode.SinglePartition) .create(); // create producer to publish non batch messages @@ -619,7 +619,7 @@ public void testBatchAndNonBatchCumulativeAcks(BatchMessageContainerBuilder buil * @throws Exception */ @Test(dataProvider = "containerBuilder", timeOut = 3000) - public void testConcurrentBatchMessageAck(BatchMessageContainerBuilder builder) throws Exception { + public void testConcurrentBatchMessageAck(BatcherBuilder builder) throws Exception { int numMsgs = 10; final String topicName = "persistent://prop/ns-abc/testConcurrentAck-" + UUID.randomUUID(); final String subscriptionName = "sub-1"; @@ -630,7 +630,7 @@ public void testConcurrentBatchMessageAck(BatchMessageContainerBuilder builder) Producer producer = pulsarClient.newProducer().topic(topicName) .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgs).enableBatching(true) - .batchingContainerBuilder(builder) + .batcherBuilder(builder) .create(); List> sendFutureList = Lists.newArrayList(); @@ -681,7 +681,7 @@ public void testOrderingOfKeyBasedBatchMessageContainer() throws PulsarClientExc .batchingMaxPublishDelay(5, TimeUnit.SECONDS) .batchingMaxMessages(30) .enableBatching(true) - .batchingContainerBuilder(BatchMessageContainerBuilder.KEY_BASED) + .batcherBuilder(BatcherBuilder.KEY_BASED) .create(); Consumer consumer = pulsarClient.newConsumer().topic(topicName) .subscriptionName(subscriptionName) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java index ec8c1aaeabf7c..494a8c8985bf9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java @@ -301,7 +301,7 @@ private Producer createProducer(String topic, boolean enableBatch) thro producer = pulsarClient.newProducer(Schema.INT32) .topic(topic) .enableBatching(true) - .batchingContainerBuilder(BatchMessageContainerBuilder.KEY_BASED) + .batcherBuilder(BatcherBuilder.KEY_BASED) .create(); } else { producer = pulsarClient.newProducer(Schema.INT32) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatcherBuilder.java similarity index 85% rename from pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java rename to pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatcherBuilder.java index e3e39c91790a7..94e337db15fd2 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatchMessageContainerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatcherBuilder.java @@ -21,9 +21,9 @@ import org.apache.pulsar.client.internal.DefaultImplementation; /** - * Batch message container builder + * Batcher builder */ -public interface BatchMessageContainerBuilder { +public interface BatcherBuilder { /** * Default batch message container @@ -34,7 +34,7 @@ public interface BatchMessageContainerBuilder { * batched into single batch message: * [(k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3)] */ - BatchMessageContainerBuilder DEFAULT = DefaultImplementation.newDefaultBatchMessageContainerBuilder(); + BatcherBuilder DEFAULT = DefaultImplementation.newDefaultBatchMessageContainerBuilder(); /** * Key based batch message container @@ -45,7 +45,7 @@ public interface BatchMessageContainerBuilder { * batched into multiple batch messages: * [(k1, v1), (k1, v2), (k1, v3)], [(k2, v1), (k2, v2), (k2, v3)], [(k3, v1), (k3, v2), (k3, v3)] */ - BatchMessageContainerBuilder KEY_BASED = DefaultImplementation.newKeyBasedBatchMessageContainerBuilder(); + BatcherBuilder KEY_BASED = DefaultImplementation.newKeyBasedBatchMessageContainerBuilder(); /** * Build a new batch message container. diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java index f62d3ffe596bb..2e1a8588728ac 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ProducerBuilder.java @@ -22,8 +22,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import javax.swing.plaf.basic.BasicInternalFrameTitlePane.MaximizeAction; - import org.apache.pulsar.client.api.PulsarClientException.ProducerQueueIsFullError; /** @@ -366,14 +364,14 @@ public interface ProducerBuilder extends Cloneable { ProducerBuilder batchingMaxMessages(int batchMessagesMaxMessagesPerBatch); /** - * Set the batch message container builder {@link BatchMessageContainerBuilder} of the producer. Producer will use the batch message container builder to + * Set the batcher builder {@link BatcherBuilder} of the producer. Producer will use the batcher builder to * build a batch message container.This is only be used when batching is enabled * - * @param batchingContainerBuilder - * batch message container builder + * @param batcherBuilder + * batcher builder * @return the producer builder instance */ - ProducerBuilder batchingContainerBuilder(BatchMessageContainerBuilder batchingContainerBuilder); + ProducerBuilder batcherBuilder(BatcherBuilder batcherBuilder); /** * Set the baseline for the sequence ids for messages published by the producer. diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java index 6588038a30456..fbccde24c86d3 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java @@ -36,7 +36,7 @@ import lombok.experimental.UtilityClass; import org.apache.pulsar.client.api.Authentication; -import org.apache.pulsar.client.api.BatchMessageContainerBuilder; +import org.apache.pulsar.client.api.BatcherBuilder; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Schema; @@ -284,15 +284,15 @@ public static RecordSchemaBuilder newRecordSchemaBuilder(String name) { String.class).newInstance(name)); } - public static BatchMessageContainerBuilder newDefaultBatchMessageContainerBuilder() { + public static BatcherBuilder newDefaultBatchMessageContainerBuilder() { return catchExceptions( - () -> (BatchMessageContainerBuilder) getConstructor("org.apache.pulsar.client.impl.DefaultBatchMessageContainerBuilder") + () -> (BatcherBuilder) getConstructor("org.apache.pulsar.client.impl.DefaultBatcherBuilder") .newInstance()); } - public static BatchMessageContainerBuilder newKeyBasedBatchMessageContainerBuilder() { + public static BatcherBuilder newKeyBasedBatchMessageContainerBuilder() { return catchExceptions( - () -> (BatchMessageContainerBuilder) getConstructor("org.apache.pulsar.client.impl.KeyBasedBatchMessageContainerBuilder") + () -> (BatcherBuilder) getConstructor("org.apache.pulsar.client.impl.KeyBasedBatcherBuilder") .newInstance()); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java index 720480a9de7e4..b293572b6f1ca 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -23,16 +23,16 @@ import io.netty.buffer.ByteBuf; import java.io.IOException; -import java.util.Collections; import java.util.List; import io.netty.util.ReferenceCountUtil; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.ProducerImpl.OpSendMsg; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; -import org.apache.pulsar.common.api.ByteBufPair; -import org.apache.pulsar.common.api.Commands; + import org.apache.pulsar.common.api.proto.PulsarApi; +import org.apache.pulsar.common.protocol.ByteBufPair; +import org.apache.pulsar.common.protocol.Commands; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java index 614ada4299be8..a1cd090726614 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java @@ -24,10 +24,10 @@ import io.netty.util.ReferenceCountUtil; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; -import org.apache.pulsar.common.api.ByteBufPair; -import org.apache.pulsar.common.api.Commands; import org.apache.pulsar.common.api.proto.PulsarApi; import org.apache.pulsar.common.compression.CompressionCodec; +import org.apache.pulsar.common.protocol.ByteBufPair; +import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +35,9 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Base64; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; /** @@ -49,7 +51,7 @@ */ class BatchMessageKeyBasedContainer extends AbstractBatchMessageContainer { - private ConcurrentHashMap batches = new ConcurrentHashMap<>(); + private Map batches = new HashMap<>(); @Override public void add(MessageImpl msg, SendCallback callback) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/DefaultBatchMessageContainerBuilder.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/DefaultBatcherBuilder.java similarity index 86% rename from pulsar-client/src/main/java/org/apache/pulsar/client/impl/DefaultBatchMessageContainerBuilder.java rename to pulsar-client/src/main/java/org/apache/pulsar/client/impl/DefaultBatcherBuilder.java index 4806b2a484aac..17c2eb4e18ae4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/DefaultBatchMessageContainerBuilder.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/DefaultBatcherBuilder.java @@ -19,9 +19,9 @@ package org.apache.pulsar.client.impl; import org.apache.pulsar.client.api.BatchMessageContainer; -import org.apache.pulsar.client.api.BatchMessageContainerBuilder; +import org.apache.pulsar.client.api.BatcherBuilder; -public class DefaultBatchMessageContainerBuilder implements BatchMessageContainerBuilder { +public class DefaultBatcherBuilder implements BatcherBuilder { @Override public BatchMessageContainer build() { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/KeyBasedBatchMessageContainerBuilder.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/KeyBasedBatcherBuilder.java similarity index 86% rename from pulsar-client/src/main/java/org/apache/pulsar/client/impl/KeyBasedBatchMessageContainerBuilder.java rename to pulsar-client/src/main/java/org/apache/pulsar/client/impl/KeyBasedBatcherBuilder.java index 4ad15b75b2c7d..8a4e09ee61f59 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/KeyBasedBatchMessageContainerBuilder.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/KeyBasedBatcherBuilder.java @@ -19,9 +19,9 @@ package org.apache.pulsar.client.impl; import org.apache.pulsar.client.api.BatchMessageContainer; -import org.apache.pulsar.client.api.BatchMessageContainerBuilder; +import org.apache.pulsar.client.api.BatcherBuilder; -public class KeyBasedBatchMessageContainerBuilder implements BatchMessageContainerBuilder { +public class KeyBasedBatcherBuilder implements BatcherBuilder { @Override public BatchMessageContainer build() { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java index bb0436059e7be..42dfdcb01e32c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerBuilderImpl.java @@ -29,7 +29,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.api.BatchMessageContainerBuilder; +import org.apache.pulsar.client.api.BatcherBuilder; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.HashingScheme; @@ -225,8 +225,8 @@ public ProducerBuilder batchingMaxMessages(int batchMessagesMaxMessagesPerBat } @Override - public ProducerBuilder batchingContainerBuilder(BatchMessageContainerBuilder batchingContainerBuilder) { - conf.setBatchMessageContainerBuilder(batchingContainerBuilder); + public ProducerBuilder batcherBuilder(BatcherBuilder batcherBuilder) { + conf.setBatcherBuilder(batcherBuilder); return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 55ac3d3b6d2fd..48656f8f4360a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -48,7 +48,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLongFieldUpdater; -import org.apache.pulsar.client.api.BatchMessageContainerBuilder; +import org.apache.pulsar.client.api.BatcherBuilder; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; @@ -164,9 +164,9 @@ public ProducerImpl(PulsarClientImpl client, String topic, ProducerConfiguration this.createProducerTimeout = System.currentTimeMillis() + client.getConfiguration().getOperationTimeoutMs(); if (conf.isBatchingEnabled()) { this.maxNumMessagesInBatch = conf.getBatchingMaxMessages(); - BatchMessageContainerBuilder containerBuilder = conf.getBatchMessageContainerBuilder(); + BatcherBuilder containerBuilder = conf.getBatcherBuilder(); if (containerBuilder == null) { - containerBuilder = BatchMessageContainerBuilder.DEFAULT; + containerBuilder = BatcherBuilder.DEFAULT; } this.batchMessageContainer = (BatchMessageContainerBase)containerBuilder.build(); this.batchMessageContainer.setProducer(this); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java index 7af237aba545c..57204200d6faa 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ProducerConfigurationData.java @@ -26,9 +26,8 @@ import java.util.concurrent.TimeUnit; import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.NoArgsConstructor; -import org.apache.pulsar.client.api.BatchMessageContainerBuilder; +import org.apache.pulsar.client.api.BatcherBuilder; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.HashingScheme; @@ -67,7 +66,7 @@ public class ProducerConfigurationData implements Serializable, Cloneable { private long batchingMaxPublishDelayMicros = TimeUnit.MILLISECONDS.toMicros(1); private int batchingMaxMessages = 1000; private boolean batchingEnabled = true; // enabled by default - private BatchMessageContainerBuilder batchMessageContainerBuilder = BatchMessageContainerBuilder.DEFAULT; + private BatcherBuilder batcherBuilder = BatcherBuilder.DEFAULT; @JsonIgnore private CryptoKeyReader cryptoKeyReader; diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java index df150224701a1..bb0092f2f25ca 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/conf/ConfigurationDataUtilsTest.java @@ -59,7 +59,7 @@ public void testLoadProducerConfigurationData() { Map config = new HashMap<>(); config.put("producerName", "test-producer"); config.put("batchingEnabled", false); - confData.setBatchMessageContainerBuilder(null); + confData.setBatcherBuilder(null); confData = ConfigurationDataUtils.loadData(config, confData, ProducerConfigurationData.class); assertEquals("test-producer", confData.getProducerName()); assertEquals(false, confData.isBatchingEnabled()); diff --git a/pulsar-storm/src/main/java/org/apache/pulsar/storm/PulsarBolt.java b/pulsar-storm/src/main/java/org/apache/pulsar/storm/PulsarBolt.java index 8cdcbae8ea844..d331ca2e46494 100644 --- a/pulsar-storm/src/main/java/org/apache/pulsar/storm/PulsarBolt.java +++ b/pulsar-storm/src/main/java/org/apache/pulsar/storm/PulsarBolt.java @@ -91,7 +91,7 @@ public PulsarBolt(PulsarBoltConfiguration pulsarBoltConf, ClientConfigurationDat this.producerConf = producerConf; this.clientConf.setServiceUrl(pulsarBoltConf.getServiceUrl()); this.producerConf.setTopicName(pulsarBoltConf.getTopic()); - this.producerConf.setBatchMessageContainerBuilder(null); + this.producerConf.setBatcherBuilder(null); } @SuppressWarnings({ "rawtypes" }) From 7089cbe524470235cb6134b468a639dc2054df9d Mon Sep 17 00:00:00 2001 From: lipenghui Date: Mon, 10 Jun 2019 14:44:13 +0800 Subject: [PATCH 11/13] rename to batcher builder. --- .../java/org/apache/pulsar/client/api/BatcherBuilder.java | 4 ++-- .../apache/pulsar/client/internal/DefaultImplementation.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatcherBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatcherBuilder.java index 94e337db15fd2..705a0ac167dd8 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatcherBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/BatcherBuilder.java @@ -34,7 +34,7 @@ public interface BatcherBuilder { * batched into single batch message: * [(k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3)] */ - BatcherBuilder DEFAULT = DefaultImplementation.newDefaultBatchMessageContainerBuilder(); + BatcherBuilder DEFAULT = DefaultImplementation.newDefaultBatcherBuilder(); /** * Key based batch message container @@ -45,7 +45,7 @@ public interface BatcherBuilder { * batched into multiple batch messages: * [(k1, v1), (k1, v2), (k1, v3)], [(k2, v1), (k2, v2), (k2, v3)], [(k3, v1), (k3, v2), (k3, v3)] */ - BatcherBuilder KEY_BASED = DefaultImplementation.newKeyBasedBatchMessageContainerBuilder(); + BatcherBuilder KEY_BASED = DefaultImplementation.newKeyBasedBatcherBuilder(); /** * Build a new batch message container. diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java index fbccde24c86d3..80282bfd48247 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/DefaultImplementation.java @@ -284,13 +284,13 @@ public static RecordSchemaBuilder newRecordSchemaBuilder(String name) { String.class).newInstance(name)); } - public static BatcherBuilder newDefaultBatchMessageContainerBuilder() { + public static BatcherBuilder newDefaultBatcherBuilder() { return catchExceptions( () -> (BatcherBuilder) getConstructor("org.apache.pulsar.client.impl.DefaultBatcherBuilder") .newInstance()); } - public static BatcherBuilder newKeyBasedBatchMessageContainerBuilder() { + public static BatcherBuilder newKeyBasedBatcherBuilder() { return catchExceptions( () -> (BatcherBuilder) getConstructor("org.apache.pulsar.client.impl.KeyBasedBatcherBuilder") .newInstance()); From 5a5df347141fb5b5d3254ef80ca0a13159c731ca Mon Sep 17 00:00:00 2001 From: lipenghui Date: Mon, 10 Jun 2019 14:51:40 +0800 Subject: [PATCH 12/13] remove unused import. --- .../pulsar/client/impl/BatchMessageKeyBasedContainer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java index a1cd090726614..a43afdac66f64 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java @@ -38,7 +38,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; /** * Key based batch message container @@ -79,7 +78,7 @@ public void add(MessageImpl msg, SendCallback callback) { public void clear() { numMessagesInBatch = 0; currentBatchSizeBytes = 0; - batches = new ConcurrentHashMap<>(); + batches = new HashMap<>(); } @Override From a84f88a8f42123f82b34c836be8b0b52113fcabf Mon Sep 17 00:00:00 2001 From: lipenghui Date: Tue, 11 Jun 2019 10:03:57 +0800 Subject: [PATCH 13/13] remove unused code. --- .../pulsar/PulsarConsumerMessageSource.java | 39 --------- .../pulsar/PulsarTypeInformation.java | 66 -------------- .../pulsar/PulsarTypeSerializer.java | 87 ------------------- .../PulsarTypeSerializerConfigSnapshot.java | 21 ----- .../containers/FlinkContainer.java | 30 ------- .../integration/flink/FlinkContainerTest.java | 15 ---- 6 files changed, 258 deletions(-) delete mode 100644 pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerMessageSource.java delete mode 100644 pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeInformation.java delete mode 100644 pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializer.java delete mode 100644 pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializerConfigSnapshot.java delete mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/FlinkContainer.java delete mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/flink/FlinkContainerTest.java diff --git a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerMessageSource.java b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerMessageSource.java deleted file mode 100644 index bbbfc8058d484..0000000000000 --- a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerMessageSource.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.apache.flink.streaming.connectors.pulsar; - -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.streaming.api.functions.source.MessageAcknowledgingSourceBase; -import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.MessageId; - -import java.util.Set; - -public class PulsarConsumerMessageSource - extends MessageAcknowledgingSourceBase, MessageId> - implements PulsarSourceBase> { - - PulsarConsumerMessageSource(PulsarSourceBuilder builder) { - super(MessageId.class); - - } - - @Override - public TypeInformation> getProducedType() { - return null; - } - - @Override - protected void acknowledgeIDs(long l, Set set) { - - } - - @Override - public void run(SourceContext> sourceContext) throws Exception { - - } - - @Override - public void cancel() { - - } - -} diff --git a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeInformation.java b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeInformation.java deleted file mode 100644 index 3db5e4d6f5b65..0000000000000 --- a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeInformation.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.apache.flink.streaming.connectors.pulsar; - -import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeutils.TypeSerializer; - -/** - * Convert a pulsar schema info into a flink type inform. - */ -public class PulsarTypeInformation extends TypeInformation { - - @Override - public boolean isBasicType() { - return false; - } - - @Override - public boolean isTupleType() { - return false; - } - - @Override - public int getArity() { - return 0; - } - - @Override - public int getTotalFields() { - return 0; - } - - @Override - public Class getTypeClass() { - return null; - } - - @Override - public boolean isKeyType() { - return false; - } - - @Override - public TypeSerializer createSerializer(ExecutionConfig config) { - return null; - } - - @Override - public String toString() { - return null; - } - - @Override - public boolean equals(Object obj) { - return false; - } - - @Override - public int hashCode() { - return 0; - } - - @Override - public boolean canEqual(Object obj) { - return false; - } -} diff --git a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializer.java b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializer.java deleted file mode 100644 index aec13145d9838..0000000000000 --- a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializer.java +++ /dev/null @@ -1,87 +0,0 @@ -package org.apache.flink.streaming.connectors.pulsar; - -import org.apache.flink.api.common.typeutils.CompatibilityResult; -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; -import org.apache.flink.core.memory.DataInputView; -import org.apache.flink.core.memory.DataOutputView; - -import java.io.IOException; - -public class PulsarTypeSerializer extends TypeSerializer { - - @Override - public boolean isImmutableType() { - return false; - } - - @Override - public TypeSerializer duplicate() { - return null; - } - - @Override - public T createInstance() { - return null; - } - - @Override - public T copy(T from) { - return null; - } - - @Override - public T copy(T from, T reuse) { - return null; - } - - @Override - public int getLength() { - return 0; - } - - @Override - public void serialize(T record, DataOutputView target) throws IOException { - - } - - @Override - public T deserialize(DataInputView source) throws IOException { - return null; - } - - @Override - public T deserialize(T reuse, DataInputView source) throws IOException { - return null; - } - - @Override - public void copy(DataInputView source, DataOutputView target) throws IOException { - - } - - @Override - public boolean equals(Object obj) { - return false; - } - - @Override - public boolean canEqual(Object obj) { - return false; - } - - @Override - public int hashCode() { - return 0; - } - - @Override - public TypeSerializerConfigSnapshot snapshotConfiguration() { - return null; - } - - @Override - public CompatibilityResult ensureCompatibility(TypeSerializerConfigSnapshot configSnapshot) { - return null; - } -} diff --git a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializerConfigSnapshot.java b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializerConfigSnapshot.java deleted file mode 100644 index 95f06306218b2..0000000000000 --- a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarTypeSerializerConfigSnapshot.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.apache.flink.streaming.connectors.pulsar; - -import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot; - -public class PulsarTypeSerializerConfigSnapshot extends TypeSerializerConfigSnapshot { - - @Override - public boolean equals(Object obj) { - return false; - } - - @Override - public int hashCode() { - return 0; - } - - @Override - public int getVersion() { - return 0; - } -} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/FlinkContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/FlinkContainer.java deleted file mode 100644 index 3d7eec4efbd10..0000000000000 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/FlinkContainer.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.apache.pulsar.tests.integration.containers; - -import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy; - -import java.time.Duration; - -public class FlinkContainer extends ChaosContainer { - - public static final String NAME = "Flink"; - static final Integer[] PORTS = { 8080 }; - - private static final String IMAGE_NAME = "flink:latest"; - - public FlinkContainer(String clusterName) { - super(clusterName, IMAGE_NAME); - } - - @Override - protected void configure() { - super.configure(); - this.withNetworkAliases(NAME) - .withExposedPorts(PORTS) - .withCreateContainerCmdModifier(createContainerCmd -> { - createContainerCmd.withHostName(NAME); - createContainerCmd.withName(clusterName + "-" + NAME); - }) - .withCommand("bin/start-cluster.sh") - .waitingFor(new HostPortWaitStrategy().withStartupTimeout(Duration.ofMinutes(3))); - } -} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/flink/FlinkContainerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/flink/FlinkContainerTest.java deleted file mode 100644 index 7ea31bf2ba1f0..0000000000000 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/flink/FlinkContainerTest.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.apache.pulsar.tests.integration.flink; - -import org.apache.pulsar.tests.integration.containers.FlinkContainer; -import org.testng.annotations.Test; - -public class FlinkContainerTest { - - @Test - public void test() throws Exception { - FlinkContainer flinkContainer = new FlinkContainer("test"); - flinkContainer.start(); - System.out.println(flinkContainer.isRunning()); - flinkContainer.stop(); - } -}