Skip to content

[improve][java-client] Improve performance of multi-topic consumer with more than one IO thread #16336

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
codelipenghui merged 7 commits into from
Jul 5, 2022
Merged

[improve][java-client] Improve performance of multi-topic consumer with more than one IO thread #16336

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e441bf6
[improve][java-client] Improve consumption performance of multi-topic…
codelipenghui Jul 1, 2022
d58a82f
Apply comments.
codelipenghui Jul 2, 2022
602ba66
Remove schedule.
codelipenghui Jul 2, 2022
714ae60
Remove schedule.
codelipenghui Jul 2, 2022
4d8ce0b
Remove public method.
codelipenghui Jul 3, 2022
853909d
Address comments.
codelipenghui Jul 4, 2022
07280ff
remove queue size check
codelipenghui Jul 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessagesImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
@NotThreadSafe
public class MessagesImpl<T> implements Messages<T> {

private List<Message<T>> messageList;
private final List<Message<T>> messageList;

private final int maxNumberOfMessages;
private final long maxSizeOfMessages;
Expand Down Expand Up @@ -80,6 +80,10 @@ public void clear() {
this.messageList.clear();
}

List<Message<T>> getMessageList() {
return messageList;
}

@Override
public Iterator<Message<T>> iterator() {
return messageList.iterator();
Expand Down
59 changes: 37 additions & 22 deletions pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.client.api.BatchReceivePolicy;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.ConsumerStats;
import org.apache.pulsar.client.api.Message;
Expand Down Expand Up @@ -238,19 +239,25 @@ private void startReceivingMessages(List<ConsumerImpl<T>> newConsumers) {
newConsumers.forEach(consumer -> {
consumer.increaseAvailablePermits(consumer.getConnectionHandler().cnx(),
consumer.getCurrentReceiverQueueSize());
internalPinnedExecutor.execute(() -> receiveMessageFromConsumer(consumer));
internalPinnedExecutor.execute(() -> receiveMessageFromConsumer(consumer, true));
});
}
}

private void receiveMessageFromConsumer(ConsumerImpl<T> consumer) {
consumer.receiveAsync().thenAcceptAsync(message -> {
private void receiveMessageFromConsumer(ConsumerImpl<T> consumer, boolean batchReceive) {
CompletableFuture<List<Message<T>>> messagesFuture;
if (batchReceive) {
messagesFuture = consumer.batchReceiveAsync().thenApply(msgs -> ((MessagesImpl<T>) msgs).getMessageList());
} else {
messagesFuture = consumer.receiveAsync().thenApply(Collections::singletonList);
}
messagesFuture.thenAcceptAsync(messages -> {
if (log.isDebugEnabled()) {
log.debug("[{}] [{}] Receive message from sub consumer:{}",
topic, subscription, consumer.getTopic());
}
// Process the message, add to the queue and trigger listener or async callback
messageReceived(consumer, message);
messages.forEach(msg -> messageReceived(consumer, msg));

int size = incomingMessages.size();
int maxReceiverQueueSize = getCurrentReceiverQueueSize();
Expand All @@ -268,7 +275,7 @@ private void receiveMessageFromConsumer(ConsumerImpl<T> consumer) {
} else {
// Call receiveAsync() if the incoming queue is not full. Because this block is run with
// thenAcceptAsync, there is no chance for recursion that would lead to stack overflow.
receiveMessageFromConsumer(consumer);
receiveMessageFromConsumer(consumer, messages.size() > 0);

@hangc0276 hangc0276 Jul 4, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If entered receiveMessageFromConsumer from this line and the messages.size() = 0 in this cycle, is there any chance to call batch receive in the future?

@codelipenghui codelipenghui Jul 4, 2022

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, if messages.size() = 0, the current round of receive messages from the internal consumer will use consumer.receiveAsync(). After the internal consumer has new incoming messages, the next round will use the batchReceiveAsync() again.

}
}, internalPinnedExecutor).exceptionally(ex -> {
if (ex instanceof PulsarClientException.AlreadyClosedException
Expand All @@ -277,8 +284,8 @@ private void receiveMessageFromConsumer(ConsumerImpl<T> consumer) {
return null;
}
log.error("Receive operation failed on consumer {} - Retrying later", consumer, ex);
((ScheduledExecutorService) client.getScheduledExecutorProvider())
.schedule(() -> receiveMessageFromConsumer(consumer), 10, TimeUnit.SECONDS);
((ScheduledExecutorService) client.getScheduledExecutorProvider().getExecutor())
.schedule(() -> receiveMessageFromConsumer(consumer, true), 10, TimeUnit.SECONDS);
return null;
});
}
Expand Down Expand Up @@ -321,7 +328,7 @@ private void resumeReceivingFromPausedConsumersIfNeeded() {
}

internalPinnedExecutor.execute(() -> {
receiveMessageFromConsumer(consumer);
receiveMessageFromConsumer(consumer, true);
});
}
}
Expand Down Expand Up @@ -1045,11 +1052,8 @@ private void doSubscribeTopicPartitions(Schema<T> schema,
String partitionName = TopicName.get(topicName).getPartition(partitionIndex).toString();
CompletableFuture<Consumer<T>> subFuture = new CompletableFuture<>();
configurationData.setStartPaused(paused);
ConsumerImpl<T> newConsumer = ConsumerImpl.newConsumerImpl(client, partitionName,
configurationData, client.externalExecutorProvider(),
partitionIndex, true, listener != null, subFuture,
startMessageId, schema, interceptors,
createIfDoesNotExist, startMessageRollbackDurationInSec);
ConsumerImpl<T> newConsumer = createInternalConsumer(configurationData, partitionName,
partitionIndex, subFuture, createIfDoesNotExist, schema);
synchronized (pauseMutex) {
if (paused) {
newConsumer.pause();
Expand All @@ -1075,10 +1079,8 @@ private void doSubscribeTopicPartitions(Schema<T> schema,
return existingValue;
} else {
internalConfig.setStartPaused(paused);
ConsumerImpl<T> newConsumer = ConsumerImpl.newConsumerImpl(client, topicName, internalConfig,
client.externalExecutorProvider(), -1,
true, listener != null, subFuture, startMessageId, schema, interceptors,
createIfDoesNotExist, startMessageRollbackDurationInSec);
ConsumerImpl<T> newConsumer = createInternalConsumer(internalConfig, topicName,
-1, subFuture, createIfDoesNotExist, schema);

synchronized (pauseMutex) {
if (paused) {
Expand Down Expand Up @@ -1121,6 +1123,22 @@ private void doSubscribeTopicPartitions(Schema<T> schema,
});
}

private ConsumerImpl<T> createInternalConsumer(ConsumerConfigurationData<T> configurationData, String partitionName,
int partitionIndex, CompletableFuture<Consumer<T>> subFuture,
boolean createIfDoesNotExist, Schema<T> schema) {
BatchReceivePolicy internalBatchReceivePolicy = BatchReceivePolicy.builder()
.maxNumMessages(Math.max(configurationData.getReceiverQueueSize() / 2, 1))
.maxNumBytes(-1)
.timeout(1, TimeUnit.MILLISECONDS)
.build();
configurationData.setBatchReceivePolicy(internalBatchReceivePolicy);
return ConsumerImpl.newConsumerImpl(client, partitionName,
configurationData, client.externalExecutorProvider(),
partitionIndex, true, listener != null, subFuture,
startMessageId, schema, interceptors,
createIfDoesNotExist, startMessageRollbackDurationInSec);
}

// handling failure during subscribe new topic, unsubscribe success created partitions
private void handleSubscribeOneTopicError(String topicName,
Throwable error,
Expand Down Expand Up @@ -1378,11 +1396,8 @@ private CompletableFuture<Void> subscribeIncreasedTopicPartitions(String topicNa
CompletableFuture<Consumer<T>> subFuture = new CompletableFuture<>();
ConsumerConfigurationData<T> configurationData = getInternalConsumerConfig();
configurationData.setStartPaused(paused);
ConsumerImpl<T> newConsumer = ConsumerImpl.newConsumerImpl(
client, partitionName, configurationData,
client.externalExecutorProvider(),
partitionIndex, true, listener != null, subFuture, startMessageId, schema, interceptors,
true /* createTopicIfDoesNotExist */, startMessageRollbackDurationInSec);
ConsumerImpl<T> newConsumer = createInternalConsumer(configurationData, partitionName,
partitionIndex, subFuture, true, schema);
synchronized (pauseMutex) {
if (paused) {
newConsumer.pause();
Expand Down