From 1d86b4d5c0a12e3246bdbdb75ec62de867aa2c0e Mon Sep 17 00:00:00 2001 From: congbo Date: Wed, 9 Dec 2020 19:08:38 +0800 Subject: [PATCH 01/18] Transaction pending ack persistent --- pom.xml | 2 + .../pulsar/broker/ServiceConfiguration.java | 26 + pulsar-broker/generate_protobuf.sh | 23 + pulsar-broker/generate_protobuf_docker.sh | 42 + .../apache/pulsar/broker/PulsarService.java | 23 + .../persistent/PersistentSubscription.java | 14 +- .../pendingack/PendingAckHandle.java | 7 + .../pendingack/PendingAckReplyCallBack.java | 39 + .../pendingack/PendingAckStore.java | 83 ++ .../TransactionPendingAckStoreProvider.java | 69 + ...ctionPendingAckStoreProviderException.java | 32 + .../impl/InMemoryPendingAckStore.java | 66 + .../impl/MLPendingAckReplyCallBack.java | 107 ++ .../pendingack/impl/MLPendingAckStore.java | 336 +++++ .../impl/MLPendingAckStoreProvider.java | 120 ++ .../impl/MLPendingAckStoreTimerTask.java | 200 +++ .../impl/PendingAckHandleDisabled.java | 8 +- .../pendingack/impl/PendingAckHandleImpl.java | 444 ++++-- .../impl/PendingAckHandleState.java | 70 + .../proto/TransactionPendingAck.java | 1254 +++++++++++++++++ .../transaction/TransactionTestBase.java | 2 + .../PendingAckInMemoryDeleteTest.java | 79 +- .../pendingack/PendingAckPersistentTest.java | 214 +++ 23 files changed, 3114 insertions(+), 146 deletions(-) create mode 100755 pulsar-broker/generate_protobuf.sh create mode 100755 pulsar-broker/generate_protobuf_docker.sh create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTimerTask.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java diff --git a/pom.xml b/pom.xml index cdd720ff163a2..f2e93897758e2 100644 --- a/pom.xml +++ b/pom.xml @@ -1163,6 +1163,7 @@ flexible messaging model and an intuitive client API. **/*.hgrm **/src/main/java/org/apache/bookkeeper/mledger/proto/MLDataFormats.java **/src/main/java/org/apache/pulsar/transaction/coordinator/proto/PulsarTransactionMetadata.java + **/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java **/src/main/java/org/apache/pulsar/broker/service/schema/proto/SchemaRegistryFormat.java **/src/main/java/org/apache/pulsar/common/api/proto/*.java **/src/test/java/org/apache/pulsar/common/api/proto/*.java @@ -1253,6 +1254,7 @@ flexible messaging model and an intuitive client API. src/main/java/org/apache/bookkeeper/mledger/proto/MLDataFormats.java src/main/java/org/apache/pulsar/common/api/proto/PulsarApi.java src/main/java/org/apache/pulsar/transaction/coordinator/proto/PulsarTransactionMetadata.java + **/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java src/test/java/org/apache/pulsar/common/api/proto/TestApi.java src/main/java/org/apache/pulsar/common/api/proto/PulsarMarkers.java src/main/java/org/apache/pulsar/broker/service/schema/proto/SchemaRegistryFormat.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index c456de066997e..62833c71eb9f4 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1871,6 +1871,32 @@ public class ServiceConfiguration implements PulsarConfiguration { private String transactionBufferProviderClassName = "org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferProvider"; + @FieldContext( + category = CATEGORY_TRANSACTION, + doc = "Class name for transaction pending ack store provider" + ) + private String transactionPendingAckStoreProviderClassName = + "org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStoreProvider"; + + @FieldContext( + category = CATEGORY_SERVER, + doc = "Number of threads to use for pulsar transaction replay PendingAckStore or TransactionBuffer." + + "Default is 5" + ) + private int numTransactionReplayThreadPoolSize = 5; + + @FieldContext( + category = CATEGORY_SERVER, + doc = "Interval time of pending ack timer task, unit is seconds." + ) + private int minPendingAckTimerTaskIntervalTime = 3600; + + @FieldContext( + category = CATEGORY_SERVER, + doc = "Interval time of pending ack timer task, unit is seconds." + ) + private int maxPendingAckTimerTaskIntervalTime = 36000; + /**** --- KeyStore TLS config variables --- ****/ @FieldContext( category = CATEGORY_KEYSTORE_TLS, diff --git a/pulsar-broker/generate_protobuf.sh b/pulsar-broker/generate_protobuf.sh new file mode 100755 index 0000000000000..d13e17db79605 --- /dev/null +++ b/pulsar-broker/generate_protobuf.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +PROTOC=${PROTOC:-protoc} +${PROTOC} --java_out=pulsar-broker/src/main/java pulsar-broker/src/main/proto/TransactionPendingAck.proto diff --git a/pulsar-broker/generate_protobuf_docker.sh b/pulsar-broker/generate_protobuf_docker.sh new file mode 100755 index 0000000000000..c51417f7f0eb4 --- /dev/null +++ b/pulsar-broker/generate_protobuf_docker.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Fail script in case of errors +set -e + +ROOT_DIR=$(git rev-parse --show-toplevel) +COMMON_DIR=$ROOT_DIR/ +cd $COMMON_DIR + +BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}" +BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-16.04}" + +IMAGE="$BUILD_IMAGE_NAME:$BUILD_IMAGE_VERSION" + +echo $IMAGE + +# Force to pull image in case it was updated +docker pull $IMAGE + +WORKDIR=/workdir +docker run -i \ + -v ${COMMON_DIR}:${WORKDIR} $IMAGE \ + bash -c "cd ${WORKDIR}; PROTOC=/pulsar/protobuf/src/protoc ./pulsar-broker/generate_protobuf.sh" + diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 8b28810a318f7..21d119cf18a35 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -93,6 +93,7 @@ import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsServlet; import org.apache.pulsar.broker.transaction.buffer.TransactionBufferProvider; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferClientImpl; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; import org.apache.pulsar.broker.validator.MultipleListenerValidator; import org.apache.pulsar.broker.web.WebService; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -210,6 +211,8 @@ public class PulsarService implements AutoCloseable { private TransactionMetadataStoreService transactionMetadataStoreService; private TransactionBufferProvider transactionBufferProvider; private TransactionBufferClient transactionBufferClient; + private TransactionPendingAckStoreProvider transactionPendingAckStoreProvider; + private final ScheduledExecutorService transactionReplayExecutor; private BrokerInterceptor brokerInterceptor; @@ -266,6 +269,12 @@ public PulsarService(ServiceConfiguration config, Optional functi new DefaultThreadFactory("pulsar")); this.cacheExecutor = Executors.newScheduledThreadPool(config.getNumCacheExecutorThreadPoolSize(), new DefaultThreadFactory("zk-cache-callback")); + if (config.isTransactionCoordinatorEnabled()) { + this.transactionReplayExecutor = Executors.newScheduledThreadPool(config.getNumExecutorThreadPoolSize(), + new DefaultThreadFactory("transaction-replay")); + } else { + this.transactionReplayExecutor = null; + } } /** @@ -352,6 +361,10 @@ public void close() throws PulsarServerException { executor.shutdown(); } + if (transactionPendingAckStoreProvider != null) { + transactionPendingAckStoreProvider.close(); + } + if (orderedExecutor != null) { orderedExecutor.shutdown(); } @@ -377,6 +390,10 @@ public void close() throws PulsarServerException { transactionBufferClient.close(); } + if (transactionReplayExecutor != null) { + transactionReplayExecutor.shutdown(); + } + state = State.Closed; isClosedCondition.signalAll(); } catch (Exception e) { @@ -594,6 +611,8 @@ public Boolean get() { transactionBufferProvider = TransactionBufferProvider .newProvider(config.getTransactionBufferProviderClassName()); + transactionPendingAckStoreProvider = TransactionPendingAckStoreProvider + .newProvider(config.getTransactionPendingAckStoreProviderClassName()); } this.metricsGenerator = new MetricsGenerator(this); @@ -990,6 +1009,10 @@ public ScheduledExecutorService getExecutor() { return executor; } + public ScheduledExecutorService getTransactionReplayExecutor() { + return transactionReplayExecutor; + } + public ScheduledExecutorService getCacheExecutor() { return cacheExecutor; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 491542a4edd48..232d262f21c5f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -161,6 +161,12 @@ void setReplicated(boolean replicated) { @Override public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { + if (pendingAckHandle instanceof PendingAckHandleImpl) { + if (!((PendingAckHandleImpl) pendingAckHandle).checkIfReady()) { + throw new SubscriptionBusyException("Pending ack handle not init " + + "complete! topicName : " + topicName + " subName : " + subName); + } + } cursor.updateLastActive(); if (IS_FENCED_UPDATER.get(this) == TRUE) { log.warn("Attempting to add consumer {} on a fenced subscription", consumer); @@ -713,11 +719,11 @@ public CompletableFuture close() { if (dispatcher != null && dispatcher.isConsumerConnected()) { return FutureUtil.failedFuture(new SubscriptionBusyException("Subscription has active consumers")); } - IS_FENCED_UPDATER.set(this, TRUE); - log.info("[{}][{}] Successfully closed subscription [{}]", topicName, subName, cursor); + return this.pendingAckHandle.close().thenAccept(v -> { + IS_FENCED_UPDATER.set(this, TRUE); + log.info("[{}][{}] Successfully closed subscription [{}]", topicName, subName, cursor); + }); } - - return CompletableFuture.completedFuture(null); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java index d43e93f1358e2..3ef5b2b8807b0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java @@ -117,4 +117,11 @@ public interface PendingAckHandle { * @param position {@link Position} which position need to clear */ void clearIndividualPosition(Position position); + + /** + * Close the pending ack handle. + * + * @return the future of this operation. + */ + CompletableFuture close(); } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java new file mode 100644 index 0000000000000..b34133974863b --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack; + +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry; + +/** + * Call back for pending ack reply. + */ +public interface PendingAckReplyCallBack { + + /** + * Pending ack replay complete callback for pending ack store. + */ + void replayComplete(); + + /** + * Handle metadata entry. + * + * @param pendingAckMetadataEntry {@link PendingAckMetadataEntry} the metadata entry of pending ack + */ + void handleMetadataEntry(PendingAckMetadataEntry pendingAckMetadataEntry); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java new file mode 100644 index 0000000000000..9d20f05f5c685 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack; + +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; + +/** + * To store transaction pending ack. + */ +public interface PendingAckStore { + /** + * Replay pending ack to recover the pending ack subscription pending ack state. + * + * @param pendingAckHandle the handle of pending ack + */ + void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService getExecutor); + + /** + * Close the transaction pending ack store. + */ + CompletableFuture closeAsync(); + + /** + * Append the individual pending ack operation to the ack persistent store. + * + * @param txnID {@link TxnID} transaction id. + * @param positions {@link PositionImpl} the pending ack postion. + * @return a future represents the result of this operation + */ + CompletableFuture appendIndividualAck(TxnID txnID, List> positions); + + /** + * Append the cumulative pending ack operation to the ack persistent store. + * + * @param txnID {@link TxnID} transaction id. + * @param position {@link PositionImpl} the pending ack position. + * @return a future represents the result of this operation + */ + CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position); + + /** + * Append the pending ack commit mark to the ack persistent store. + * + * @param txnID {@link TxnID} the transaction id for add commit mark. + * @param ackType {@link AckType} the ack type of the commit. + * @return a future represents the result of this operation + */ + CompletableFuture appendCommitMark(TxnID txnID, AckType ackType); + + /** + * Append the pending ack abort mark to the ack persistent store. + * + * @param txnID {@link Position} the txnID + * @param ackType {@link AckType} the ack type of the abort. + * @return a future represents the result of this operation + */ + CompletableFuture appendAbortMark(TxnID txnID, AckType ackType); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java new file mode 100644 index 0000000000000..4104e36f75b41 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack; + +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * Provider of transaction pending ack store. + */ +public interface TransactionPendingAckStoreProvider { + + /** + * Construct a provider from the provided class. + * + * @param providerClassName {@link String} the provider class name + * @return an instance of transaction buffer provider. + */ + static TransactionPendingAckStoreProvider newProvider(String providerClassName) throws IOException { + Class providerClass; + try { + providerClass = Class.forName(providerClassName); + Object obj = providerClass.newInstance(); + checkArgument(obj instanceof TransactionPendingAckStoreProvider, + "The factory has to be an instance of " + + TransactionPendingAckStoreProvider.class.getName()); + + return (TransactionPendingAckStoreProvider) obj; + } catch (Exception e) { + throw new IOException(e); + } + } + + /** + * Open the pending ack store. + * + * @param subscription {@link PersistentSubscription} + * @param subName {@link String} + * @return a future represents the result of the operation. + * an instance of {@link PendingAckStore} is returned + * if the operation succeeds. + */ + CompletableFuture newPendingAckStore(PersistentSubscription subscription, String subName); + + /** + * Close the provider and release all the resources. + */ + void close() throws IOException; +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java new file mode 100644 index 0000000000000..0f41e2cd3c455 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.exceptions; + +import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionBufferException; + +/** + * Transaction pending ack store provider exception. + */ +public class TransactionPendingAckStoreProviderException extends TransactionBufferException { + + public TransactionPendingAckStoreProviderException(String message) { + super(message); + } + +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java new file mode 100644 index 0000000000000..39cb094d55cef --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.api.proto.PulsarApi; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; + +/** + * The In memory implementation of {@link PendingAckStore}. + */ +public class InMemoryPendingAckStore implements PendingAckStore { + + @Override + public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService scheduledExecutorService) { + pendingAckHandle.changeToReadyState(); + } + + @Override + public CompletableFuture closeAsync() { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendIndividualAck(TxnID txnID, List> positions) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendCommitMark(TxnID txnID, PulsarApi.CommandAck.AckType ackType) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendAbortMark(TxnID txnID, PulsarApi.CommandAck.AckType ackType) { + return CompletableFuture.completedFuture(null); + } + +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java new file mode 100644 index 0000000000000..ee8f5f57f2557 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; +import org.apache.pulsar.common.util.SafeCollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * MLPendingAckStore reply call back. + */ +public class MLPendingAckReplyCallBack implements PendingAckReplyCallBack { + + private final MLPendingAckStore mlPendingAckStore; + + private final PendingAckHandleImpl pendingAckHandle; + + public MLPendingAckReplyCallBack(MLPendingAckStore mlPendingAckStore, + PendingAckHandleImpl pendingAckHandle) { + this.mlPendingAckStore = mlPendingAckStore; + this.pendingAckHandle = pendingAckHandle; + } + + @Override + public void replayComplete() { + log.info("Topic name : [{}], SubName : [{}] pending ack state reply success!", + pendingAckHandle.getTopicName(), pendingAckHandle.getSubName()); + + if (pendingAckHandle.changeToReadyState()) { + log.info("Topic name : [{}], SubName : [{}] pending ack state reply success!", + pendingAckHandle.getTopicName(), pendingAckHandle.getSubName()); + mlPendingAckStore.startTimerTask(); + } else { + log.error("Topic name : [{}], SubName : [{}] pending ack state reply fail!", + pendingAckHandle.getTopicName(), pendingAckHandle.getSubName()); + } + } + + @Override + public void handleMetadataEntry(PendingAckMetadataEntry pendingAckMetadataEntry) { + TxnID txnID = new TxnID(pendingAckMetadataEntry.getTxnidMostBits(), + pendingAckMetadataEntry.getTxnidLeastBits()); + AckType ackType = pendingAckMetadataEntry.getAckType(); + switch (pendingAckMetadataEntry.getPendingAckOp()) { + case ABORT: + pendingAckHandle.handleAbort(txnID, ackType); + break; + case COMMIT: + pendingAckHandle.handleCommit(txnID, ackType, null); + break; + case ACK: + if (ackType == AckType.Cumulative) { + PendingAckMetadata pendingAckMetadata = + pendingAckMetadataEntry.getPendingAckMetadataList().get(0); + pendingAckHandle.handleCumulativeAck(txnID, + PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId())); + } else { + List> positions = new ArrayList<>(); + pendingAckMetadataEntry.getPendingAckMetadataList().forEach(pendingAckMetadata -> { + if (pendingAckMetadata.getAckSetCount() == 0) { + positions.add(new MutablePair<>(PositionImpl.get(pendingAckMetadata.getLedgerId(), + pendingAckMetadata.getEntryId()), pendingAckMetadata.getBatchSize())); + } else { + PositionImpl position = + PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId()); + position.setAckSet(SafeCollectionUtils.longListToArray(pendingAckMetadata.getAckSetList())); + positions.add(new MutablePair<>(position, pendingAckMetadata.getBatchSize())); + } + }); + pendingAckHandle.handleIndividualAck(txnID, positions); + } + break; + default: + throw new IllegalStateException("Transaction pending ack replay " + + "error with illegal state : " + pendingAckMetadataEntry.getPendingAckOp()); + + } + } + + private static final Logger log = LoggerFactory.getLogger(MLPendingAckReplyCallBack.class); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java new file mode 100644 index 0000000000000..9afca676e00e2 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -0,0 +1,336 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import io.netty.buffer.ByteBuf; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import io.netty.util.Timer; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.SafeCollectionUtils; +import org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream; +import org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream; +import org.jctools.queues.MessagePassingQueue; +import org.jctools.queues.SpscArrayQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The implement of the pending ack store by manageLedger. + */ +public class MLPendingAckStore implements PendingAckStore { + + + private final ManagedLedger managedLedger; + + private final ManagedCursor cursor; + + private static final String PENDING_ACK_STORE_SUFFIX = "-transaction-pendingack"; + + private static final String PENDING_ACK_STORE_CURSOR_NAME = "pendingack"; + + private final SpscArrayQueue entryQueue; + + //this is for replay + private final PositionImpl lastConfirmedEntry; + + private PositionImpl currentLoadPosition; + + private final Timer timer; + + private final MLPendingAckStoreTimerTask mlPendingAckStoreTimerTask; + + private final int intervalTime; + + public MLPendingAckStore(ManagedLedger managedLedger, ManagedCursor cursor, + Timer timer, ManagedCursor subManagedCursor, int maxIntervalTime, int minIntervalTime) { + this.managedLedger = managedLedger; + this.cursor = cursor; + this.currentLoadPosition = (PositionImpl) this.cursor.getMarkDeletedPosition(); + this.entryQueue = new SpscArrayQueue<>(2000); + this.lastConfirmedEntry = (PositionImpl) managedLedger.getLastConfirmedEntry(); + this.timer = timer; + this.intervalTime = minIntervalTime; + this.mlPendingAckStoreTimerTask = new MLPendingAckStoreTimerTask(cursor, managedLedger, + minIntervalTime, maxIntervalTime, subManagedCursor, this.timer); + } + + @Override + public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService transactionReplayExecutor) { + transactionReplayExecutor + .execute(new PendingAckReplay(new MLPendingAckReplyCallBack(this, pendingAckHandle))); + } + + //TODO can control the number of entry to read + private void readAsync(int numberOfEntriesToRead, + AsyncCallbacks.ReadEntriesCallback readEntriesCallback) { + cursor.asyncReadEntries(numberOfEntriesToRead, readEntriesCallback, System.nanoTime()); + } + + @Override + public CompletableFuture closeAsync() { + CompletableFuture completableFuture = new CompletableFuture<>(); + cursor.asyncClose(new AsyncCallbacks.CloseCallback() { + @Override + public void closeComplete(Object ctx) { + try { + managedLedger.close(); + } catch (Exception e) { + completableFuture.completeExceptionally(e); + } + completableFuture.complete(null); + } + + @Override + public void closeFailed(ManagedLedgerException exception, Object ctx) { + completableFuture.completeExceptionally(exception); + } + }, null); + return completableFuture; + } + + @Override + public CompletableFuture appendIndividualAck(TxnID txnID, List> positions) { + PendingAckMetadataEntry.Builder builder = PendingAckMetadataEntry.newBuilder(); + builder.setPendingAckOp(PendingAckOp.ACK); + builder.setAckType(AckType.Individual); + positions.forEach(positionIntegerMutablePair -> { + PendingAckMetadata.Builder metadataBuilder = PendingAckMetadata.newBuilder(); + PositionImpl position = positionIntegerMutablePair.getLeft(); + int batchSize = positionIntegerMutablePair.getRight(); + if (positionIntegerMutablePair.getLeft().getAckSet() != null) { + metadataBuilder.addAllAckSet(SafeCollectionUtils.longArrayToList(position.getAckSet())); + metadataBuilder.setBatchSize(batchSize); + } + metadataBuilder.setLedgerId(position.getLedgerId()); + metadataBuilder.setEntryId(position.getEntryId()); + PendingAckMetadata pendingAckMetadata = metadataBuilder.build(); + metadataBuilder.recycle(); + builder.addPendingAckMetadata(pendingAckMetadata); + }); + return appendCommon(builder, txnID); + } + + @Override + public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { + PendingAckMetadataEntry.Builder builder = PendingAckMetadataEntry.newBuilder(); + builder.setPendingAckOp(PendingAckOp.ACK); + builder.setAckType(AckType.Cumulative); + PendingAckMetadata.Builder metadataBuilder = PendingAckMetadata.newBuilder(); + if (position.getAckSet() != null) { + metadataBuilder.addAllAckSet(SafeCollectionUtils.longArrayToList(position.getAckSet())); + } + metadataBuilder.setLedgerId(position.getLedgerId()); + metadataBuilder.setEntryId(position.getEntryId()); + PendingAckMetadata pendingAckMetadata = metadataBuilder.build(); + metadataBuilder.recycle(); + builder.addPendingAckMetadata(pendingAckMetadata); + return appendCommon(builder, txnID); + } + + @Override + public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) { + PendingAckMetadataEntry.Builder builder = PendingAckMetadataEntry.newBuilder(); + builder.setPendingAckOp(PendingAckOp.COMMIT); + builder.setAckType(ackType); + return appendCommon(builder, txnID); + } + + @Override + public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) { + PendingAckMetadataEntry.Builder builder = PendingAckMetadataEntry.newBuilder(); + builder.setPendingAckOp(PendingAckOp.ABORT); + builder.setAckType(ackType); + return appendCommon(builder, txnID); + } + + private CompletableFuture appendCommon(PendingAckMetadataEntry.Builder builder, TxnID txnID) { + CompletableFuture completableFuture = new CompletableFuture<>(); + builder.setTxnidLeastBits(txnID.getLeastSigBits()); + builder.setTxnidMostBits(txnID.getMostSigBits()); + PendingAckMetadataEntry pendingAckMetadataEntry = builder.build(); + int transactionMetadataEntrySize = pendingAckMetadataEntry.getSerializedSize(); + ByteBuf buf = PulsarByteBufAllocator.DEFAULT.buffer(transactionMetadataEntrySize, transactionMetadataEntrySize); + ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(buf); + try { + pendingAckMetadataEntry.writeTo(outStream); + managedLedger.asyncAddEntry(buf, new AsyncCallbacks.AddEntryCallback() { + @Override + public void addComplete(Position position, Object ctx) { + if (log.isDebugEnabled()) { + log.debug("[{}][{}] MLPendingAckStore message append success at {}, operation : {}", + managedLedger.getName(), ctx, position, builder.getPendingAckOp()); + } + builder.recycle(); + pendingAckMetadataEntry.recycle(); + buf.release(); + completableFuture.complete(null); + } + + @Override + public void addFailed(ManagedLedgerException exception, Object ctx) { + log.error("[{}][{}] MLPendingAckStore message append fail exception : {}, operation : {}", + managedLedger.getName(), ctx, exception, builder.getPendingAckOp()); + builder.recycle(); + pendingAckMetadataEntry.recycle(); + buf.release(); + completableFuture.completeExceptionally(exception); + } + } , null); + } catch (Exception e) { + log.error("[{}] MLPendingAckStore message append fail exception : {}", + managedLedger.getName(), e); + builder.recycle(); + pendingAckMetadataEntry.recycle(); + buf.release(); + completableFuture.completeExceptionally(e); + } finally { + outStream.recycle(); + } + return completableFuture; + } + + class PendingAckReplay implements Runnable { + + private final FillEntryQueueCallback fillEntryQueueCallback; + private final PendingAckReplyCallBack pendingAckReplyCallBack; + + PendingAckReplay(PendingAckReplyCallBack pendingAckReplyCallBack) { + this.fillEntryQueueCallback = new FillEntryQueueCallback(); + this.pendingAckReplyCallBack = pendingAckReplyCallBack; + } + + @Override + public void run() { + while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0) { + fillEntryQueueCallback.fillQueue(); + Entry entry = entryQueue.poll(); + if (entry != null) { + ByteBuf buffer = entry.getDataBuffer(); + currentLoadPosition = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); + ByteBufCodedInputStream stream = ByteBufCodedInputStream.get(buffer); + PendingAckMetadataEntry.Builder pendingAckMetadataEntryBuilder = + PendingAckMetadataEntry.newBuilder(); + PendingAckMetadataEntry pendingAckMetadataEntry = null; + try { + pendingAckMetadataEntry = + pendingAckMetadataEntryBuilder.mergeFrom(stream, null).build(); + pendingAckReplyCallBack.handleMetadataEntry(pendingAckMetadataEntry); + } catch (Exception e) { + if (pendingAckMetadataEntry != null) { + log.error("TxnId : [{}:{}] MLPendingAckStore reply error!", + pendingAckMetadataEntry.getTxnidMostBits(), + pendingAckMetadataEntry.getTxnidLeastBits(), e); + } else { + log.error("MLPendingAckStore reply error!", e); + } + } + entry.release(); + if (pendingAckMetadataEntry != null) { + pendingAckMetadataEntry.recycle(); + } + pendingAckMetadataEntryBuilder.recycle(); + stream.recycle(); + } else { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + //no-op + } + } + } + pendingAckReplyCallBack.replayComplete(); + } + } + + class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallback { + + private final AtomicLong outstandingReadsRequests = new AtomicLong(0); + + public FillEntryQueueCallback() { + } + + void fillQueue() { + if (entryQueue.size() < entryQueue.capacity() && outstandingReadsRequests.get() == 0) { + if (cursor.hasMoreEntries()) { + outstandingReadsRequests.incrementAndGet(); + readAsync(100, this); + } + } + } + + @Override + public void readEntriesComplete(List entries, Object ctx) { + entryQueue.fill(new MessagePassingQueue.Supplier() { + private int i = 0; + @Override + public Entry get() { + Entry entry = entries.get(i); + i++; + return entry; + } + }, entries.size()); + + outstandingReadsRequests.decrementAndGet(); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + log.error("MLPendingAckStore stat reply fail!", exception); + outstandingReadsRequests.decrementAndGet(); + } + + } + + //this task is for delete the pending ack store + protected void startTimerTask() { + this.timer.newTimeout(this.mlPendingAckStoreTimerTask, intervalTime, TimeUnit.MILLISECONDS); + } + + public static String getTransactionPendingAckStoreSuffix(String originTopicName, String subName) { + return TopicName.get(originTopicName) + "-" + subName + PENDING_ACK_STORE_SUFFIX; + } + + public static String getTransactionPendingAckStoreCursorName() { + return PENDING_ACK_STORE_CURSOR_NAME; + } + + private static final Logger log = LoggerFactory.getLogger(MLPendingAckStore.class); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java new file mode 100644 index 0000000000000..b39909173ed66 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timer; +import io.netty.util.concurrent.DefaultThreadFactory; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; +import org.apache.pulsar.broker.transaction.pendingack.exceptions.TransactionPendingAckStoreProviderException; +import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition; +import org.apache.pulsar.common.naming.TopicName; + +/** + * Provider is for MLPendingAckStore. + */ +@Slf4j +public class MLPendingAckStoreProvider implements TransactionPendingAckStoreProvider { + + private volatile Timer timer; + + private static final long tickTimeMillis = 1L; + + @Override + public CompletableFuture newPendingAckStore(PersistentSubscription subscription, String subName) { + CompletableFuture pendingAckStoreFuture = new CompletableFuture<>(); + + if (subscription == null) { + pendingAckStoreFuture.completeExceptionally( + new TransactionPendingAckStoreProviderException("The subscription is null.")); + return pendingAckStoreFuture; + } + + PersistentTopic originPersistentTopic = (PersistentTopic) subscription.getTopic(); + String pendingAckTopicName = MLPendingAckStore + .getTransactionPendingAckStoreSuffix(originPersistentTopic.getName(), subName); + + originPersistentTopic.getBrokerService().getManagedLedgerFactory() + .asyncOpen(TopicName.get(pendingAckTopicName).getPersistenceNamingEncoding(), + new AsyncCallbacks.OpenLedgerCallback() { + @Override + public void openLedgerComplete(ManagedLedger ledger, Object ctx) { + ledger.asyncOpenCursor(MLPendingAckStore.getTransactionPendingAckStoreCursorName(), + InitialPosition.Earliest, new AsyncCallbacks.OpenCursorCallback() { + @Override + public void openCursorComplete(ManagedCursor cursor, Object ctx) { + if (timer == null) { + synchronized (MLPendingAckStoreProvider.this) { + if (timer == null) { + initialize(); + } + } + } + int maxIntervalTime = + originPersistentTopic.getBrokerService().getPulsar() + .getConfig().getMaxPendingAckTimerTaskIntervalTime(); + int minIntervalTime = + originPersistentTopic.getBrokerService().getPulsar() + .getConfig().getMaxPendingAckTimerTaskIntervalTime(); + pendingAckStoreFuture + .complete(new MLPendingAckStore(ledger, cursor, timer, + subscription.getCursor(), + maxIntervalTime, minIntervalTime)); + } + + @Override + public void openCursorFailed(ManagedLedgerException exception, Object ctx) { + log.error("Open MLPendingAckStore cursor failed.", exception); + pendingAckStoreFuture.completeExceptionally(exception); + } + }, null); + } + + @Override + public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { + log.error("Open MLPendingAckStore managedLedger failed.", exception); + pendingAckStoreFuture.completeExceptionally(exception); + } + }, null); + return pendingAckStoreFuture; + } + + public void initialize() { + this.timer = new HashedWheelTimer(new DefaultThreadFactory("pending-ack-provider-timer"), + tickTimeMillis, TimeUnit.MILLISECONDS); + } + + @Override + public void close() { + if (timer != null) { + timer.stop(); + } + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTimerTask.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTimerTask.java new file mode 100644 index 0000000000000..2a1deaf721f62 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTimerTask.java @@ -0,0 +1,200 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import io.netty.buffer.ByteBuf; +import io.netty.util.Timeout; +import io.netty.util.Timer; +import io.netty.util.TimerTask; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry; +import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; +import org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +/** + * Pending ack timer task. + */ +@Slf4j +public class MLPendingAckStoreTimerTask implements TimerTask { + + private int intervalTime; + + private final int maxIntervalTime; + + private final int minIntervalTime; + + private final ManagedCursorImpl subManagedCursor; + + private final ManagedLedgerImpl storeManagedLedger; + + private final ManagedCursorImpl managedCursor; + + private final Timer timer; + + private volatile PositionImpl markDeletePosition; + + public MLPendingAckStoreTimerTask(ManagedCursor managedCursor, ManagedLedger storeManagedLedger, + int maxIntervalTime, int minIntervalTime, + ManagedCursor subManageCursor, Timer timer) { + this.intervalTime = minIntervalTime; + this.maxIntervalTime = maxIntervalTime; + this.minIntervalTime = minIntervalTime; + this.managedCursor = (ManagedCursorImpl) managedCursor; + this.subManagedCursor = (ManagedCursorImpl) subManageCursor; + this.storeManagedLedger = (ManagedLedgerImpl) storeManagedLedger; + this.markDeletePosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); + this.timer = timer; + } + + @Override + public void run(Timeout timeout) { + if (this.managedCursor.getState().equals("Closed")) { + return; + } + try { + // when no transaction ack operation in this pending ack store, it will increase the interval time + if (markDeletePosition.compareTo((PositionImpl) storeManagedLedger.getLastConfirmedEntry()) == 0) { + int time = intervalTime + minIntervalTime; + if (time > maxIntervalTime) { + intervalTime = maxIntervalTime; + } else { + intervalTime = time; + } + managedCursor.markDelete(markDeletePosition); + timer.newTimeout(MLPendingAckStoreTimerTask.this, intervalTime, TimeUnit.SECONDS); + return; + } else { + int time = intervalTime - minIntervalTime; + if (time < minIntervalTime) { + intervalTime = minIntervalTime; + } else { + intervalTime = time; + } + } + // this while in order to find the last position witch can mark delete + while (true) { + PositionImpl nextPosition = storeManagedLedger.getNextValidPosition(markDeletePosition); + Entry entry = getEntry(nextPosition).get(); + ByteBuf buffer = entry.getDataBuffer(); + ByteBufCodedInputStream stream = ByteBufCodedInputStream.get(buffer); + PendingAckMetadataEntry.Builder pendingAckMetadataEntryBuilder = + PendingAckMetadataEntry.newBuilder(); + PendingAckMetadataEntry pendingAckMetadataEntry = null; + try { + pendingAckMetadataEntry = + pendingAckMetadataEntryBuilder.mergeFrom(stream, null).build(); + switch (pendingAckMetadataEntry.getPendingAckOp()) { + case ACK: + if (pendingAckMetadataEntry.getAckType() == AckType.Cumulative) { + PendingAckMetadata pendingAckMetadata = + pendingAckMetadataEntry.getPendingAckMetadata(0); + handleAckCommon(PositionImpl.get(pendingAckMetadata.getLedgerId(), + pendingAckMetadata.getEntryId()), nextPosition); + } else { + //this judge the pendingAckMetadataEntry is can delete + PositionImpl largestPosition = null; + List metadataList = + pendingAckMetadataEntry.getPendingAckMetadataList(); + for (int i = 0; i < metadataList.size(); i++) { + PendingAckMetadata pendingAckMetadata = metadataList.get(0); + if (largestPosition == null) { + largestPosition = PositionImpl.get(pendingAckMetadata.getLedgerId(), + pendingAckMetadata.getEntryId()); + } else { + PositionImpl comparePosition = PositionImpl + .get(pendingAckMetadata.getLedgerId(), + pendingAckMetadata.getEntryId()); + if (largestPosition.compareTo(comparePosition) < 0) { + largestPosition = comparePosition; + } + + } + } + if (largestPosition != null) { + handleAckCommon(largestPosition, nextPosition); + } + } + break; + case ABORT: + case COMMIT: + markDeletePosition = nextPosition; + break; + default: + log.error("PendingAck timer task read illegal metadata state! {}", + pendingAckMetadataEntry.getPendingAckOp()); + } + } finally { + entry.release(); + if (pendingAckMetadataEntry != null) { + pendingAckMetadataEntry.recycle(); + } + pendingAckMetadataEntryBuilder.recycle(); + stream.recycle(); + } + // when markDeletePosition is not nextPosition, before markDeletePosition can delete + if (markDeletePosition != nextPosition) { + this.managedCursor.markDelete(markDeletePosition); + break; + } + } + this.timer.newTimeout(this, intervalTime, TimeUnit.MILLISECONDS); + } catch (Exception e) { + log.error("PendingAck timer task error!", e); + if ("Cursor was already closed".equals(e.getCause().getMessage())) { + return; + } + this.timer.newTimeout(this, intervalTime, TimeUnit.MILLISECONDS); + } + } + + private CompletableFuture getEntry(PositionImpl position) { + CompletableFuture completableFuture = new CompletableFuture<>(); + this.storeManagedLedger.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + completableFuture.complete(entry); + } + + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + completableFuture.completeExceptionally(exception); + } + }, null); + return completableFuture; + } + + private void handleAckCommon(PositionImpl readPosition, PositionImpl storePosition) { + if (readPosition.compareTo((PositionImpl) subManagedCursor.getMarkDeletedPosition()) <= 0) { + markDeletePosition = storePosition; + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java index f315f8b5eb854..39c78148377e4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java @@ -68,6 +68,12 @@ public boolean checkIsCanDeleteConsumerPendingAck(PositionImpl position) { @Override public void clearIndividualPosition(Position position) { - //no operation + //no-op } + + @Override + public CompletableFuture close() { + return CompletableFuture.completedFuture(null); + } + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 8812d4a95939c..91245d8551b24 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -26,8 +26,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Semaphore; + import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; @@ -35,9 +36,13 @@ import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; +import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.transaction.pendingack.PendingAckHandle; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; import org.apache.pulsar.common.util.FutureUtil; @@ -48,7 +53,7 @@ * The default implementation of {@link PendingAckHandle}. */ @Slf4j -public class PendingAckHandleImpl implements PendingAckHandle { +public class PendingAckHandleImpl extends PendingAckHandleState implements PendingAckHandle { /** * The map is for transaction with position witch was individual acked by this transaction. @@ -93,14 +98,42 @@ public class PendingAckHandleImpl implements PendingAckHandle { private final PersistentSubscription persistentSubscription; + private final CompletableFuture pendingAckStoreFuture; + + /** + * This is for multi thread handle transaction pending ack, + * we should think a better method to handle the multi thread problem. + */ + private final Semaphore semaphore; + public PendingAckHandleImpl(PersistentSubscription persistentSubscription) { + super(State.None); this.topicName = persistentSubscription.getTopicName(); this.subName = persistentSubscription.getName(); this.persistentSubscription = persistentSubscription; + this.semaphore = new Semaphore(1); + + TransactionPendingAckStoreProvider pendingAckStoreProvider = + ((PersistentTopic) this.persistentSubscription.getTopic()) + .getBrokerService().getPulsar().getTransactionPendingAckStoreProvider(); + this.pendingAckStoreFuture = + pendingAckStoreProvider.newPendingAckStore(persistentSubscription, subName); + + this.pendingAckStoreFuture.thenAccept(pendingAckStore -> { + if (pendingAckStore instanceof MLPendingAckStore) { + changeToInitializingState(); + pendingAckStore.replayAsync(this, + ((PersistentTopic) persistentSubscription.getTopic()).getBrokerService() + .getPulsar().getTransactionReplayExecutor()); + } + }).exceptionally(e -> { + log.error("PendingAckHandleImpl init fail! TopicName : {}, SubName: {}", topicName, subName, e); + return null; + }); } @Override - public synchronized CompletableFuture individualAcknowledgeMessage(TxnID txnID, + public CompletableFuture individualAcknowledgeMessage(TxnID txnID, List> positions) { if (txnID == null) { return FutureUtil.failedFuture(new NotAllowedException("TransactionID can not be null.")); @@ -108,6 +141,7 @@ public synchronized CompletableFuture individualAcknowledgeMessage(TxnID t if (positions == null) { return FutureUtil.failedFuture(new NotAllowedException("Positions can not be null.")); } + this.semaphore.acquireUninterruptibly(); CompletableFuture completableFuture = new CompletableFuture<>(); for (MutablePair positionIntegerMutablePair : positions) { PositionImpl position = positionIntegerMutablePair.left; @@ -118,6 +152,7 @@ public synchronized CompletableFuture individualAcknowledgeMessage(TxnID t String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" + position + " already acked before."; log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); } @@ -136,6 +171,7 @@ public synchronized CompletableFuture individualAcknowledgeMessage(TxnID t String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" + position + " already acked before."; log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); } @@ -144,6 +180,7 @@ && isAckSetOverlap(individualAckPositions.get(position).getLeft().getAckSet(), a String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack batch message:" + position + " in pending ack status."; log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); } } else { @@ -151,56 +188,32 @@ && isAckSetOverlap(individualAckPositions.get(position).getLeft().getAckSet(), a String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to ack message:" + position + " in pending ack status."; log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); } } } - for (int i = 0; i < positions.size(); i++) { - if (log.isDebugEnabled()) { - log.debug("[{}][{}] TxnID:[{}] Individual acks on {}", topicName, subName, txnID.toString(), positions); - } - if (individualAckOfTransaction == null) { - individualAckOfTransaction = new HashMap<>(); - } - - if (individualAckPositions == null) { - individualAckPositions = new HashMap<>(); - } - - PositionImpl position = positions.get(i).left; - - if (position.hasAckSet()) { - - HashMap pendingAckMessageForCurrentTxn = - individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); - - if (pendingAckMessageForCurrentTxn.containsKey(position)) { - andAckSet(pendingAckMessageForCurrentTxn.get(position), position); - } else { - pendingAckMessageForCurrentTxn.put(position, position); - } - - if (!individualAckPositions.containsKey(position)) { - this.individualAckPositions.put(position, positions.get(i)); - } else { - MutablePair positionPair = this.individualAckPositions.get(position); - positionPair.setRight(positions.get(i).right); - andAckSet(positionPair.getLeft(), position); - } - - } else { - HashMap pendingAckMessageForCurrentTxn = - individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); - pendingAckMessageForCurrentTxn.put(position, position); - this.individualAckPositions.putIfAbsent(position, positions.get(i)); - } - } - completableFuture.complete(null); + this.pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendIndividualAck(txnID, positions).whenComplete((v, e) -> { + if (e != null) { + handleIndividualAck(txnID, positions); + semaphore.release(); + completableFuture.completeExceptionally(e); + } else { + handleIndividualAck(txnID, positions); + semaphore.release(); + completableFuture.complete(null); + } + })).exceptionally(e -> { + semaphore.release(); + completableFuture.completeExceptionally(e); + return null; + }); return completableFuture; } @Override - public synchronized CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, + public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, List positions) { if (txnID == null) { @@ -209,11 +222,13 @@ public synchronized CompletableFuture cumulativeAcknowledgeMessage(TxnID t if (positions == null) { return FutureUtil.failedFuture(new NotAllowedException("Positions can not be null.")); } + this.semaphore.acquireUninterruptibly(); if (positions.size() != 1) { String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " invalid cumulative ack received with multiple message ids."; log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new NotAllowedException(errorMsg)); } @@ -224,124 +239,335 @@ public synchronized CompletableFuture cumulativeAcknowledgeMessage(TxnID t " try to cumulative ack position: " + position + " within range of cursor's " + "markDeletePosition: " + persistentSubscription.getCursor().getMarkDeletedPosition(); log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); } if (log.isDebugEnabled()) { log.debug("[{}][{}] TxnID:[{}] Cumulative ack on {}.", topicName, subName, txnID.toString(), position); } - - if (this.cumulativeAckOfTransaction == null) { - this.cumulativeAckOfTransaction = MutablePair.of(txnID, position); - } else if (this.cumulativeAckOfTransaction.getKey().equals(txnID) - && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) > 0) { - this.cumulativeAckOfTransaction.setValue(position); - - } else { + CompletableFuture completableFuture = new CompletableFuture<>(); + if (cumulativeAckOfTransaction != null && this.cumulativeAckOfTransaction.getKey().equals(txnID) + && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) <= 0) { String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to cumulative batch ack position: " + position + " within range of current " + "currentPosition: " + this.cumulativeAckOfTransaction.getValue(); log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); } - return CompletableFuture.completedFuture(null); + this.pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendCumulativeAck(txnID, position).whenComplete((v, e) -> { + if (e != null) { + handleCumulativeAck(txnID, position); + semaphore.release(); + completableFuture.completeExceptionally(e); + } else { + handleCumulativeAck(txnID, position); + semaphore.release(); + completableFuture.complete(null); + } + })).exceptionally(e -> { + semaphore.release(); + completableFuture.completeExceptionally(e); + return null; + }); + return completableFuture; } @Override - public synchronized CompletableFuture commitTxn(TxnID txnID, Map properties) { + public CompletableFuture commitTxn(TxnID txnID, Map properties) { + if (!checkIfReady()) { + return FutureUtil.failedFuture(new ServiceUnitNotReadyException("PendingAckHandle not replay complete!")); + } CompletableFuture commitFuture = new CompletableFuture<>(); + // It's valid to create transaction then commit without doing any operation, which will cause // pendingAckMessagesMap to be null. + this.semaphore.acquireUninterruptibly(); if (this.cumulativeAckOfTransaction != null) { if (cumulativeAckOfTransaction.getKey().equals(txnID)) { - persistentSubscription.acknowledgeMessage(Collections - .singletonList(this.cumulativeAckOfTransaction.getValue()), AckType.Cumulative, properties); - this.cumulativeAckOfTransaction = null; + pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendCommitMark(txnID, AckType.Cumulative).whenComplete((v, e) -> { + if (e != null) { + semaphore.release(); + commitFuture.completeExceptionally(e); + } else { + persistentSubscription.acknowledgeMessage( + Collections.singletonList(this.cumulativeAckOfTransaction.getValue()), + AckType.Cumulative, properties); + this.cumulativeAckOfTransaction = null; + semaphore.release(); + commitFuture.complete(null); + } + }) + ).exceptionally(e -> { + semaphore.release(); + commitFuture.completeExceptionally(e); + return null; + }); + } else { + semaphore.release(); + commitFuture.complete(null); } } else { if (individualAckOfTransaction != null && individualAckOfTransaction.containsKey(txnID)) { HashMap pendingAckMessageForCurrentTxn = - individualAckOfTransaction.remove(txnID); - if (pendingAckMessageForCurrentTxn != null) { - persistentSubscription.acknowledgeMessage(new ArrayList<>(pendingAckMessageForCurrentTxn.values()), - AckType.Individual, properties); - } + individualAckOfTransaction.get(txnID); + pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendCommitMark(txnID, AckType.Individual).whenComplete((v, e) -> { + if (e != null) { + semaphore.release(); + commitFuture.completeExceptionally(e); + } else { + individualAckCommitCommon(txnID, pendingAckMessageForCurrentTxn, properties); + semaphore.release(); + commitFuture.complete(null); + } + }) + ).exceptionally(e -> { + semaphore.release(); + commitFuture.completeExceptionally(e); + return null; + }); + } else { + semaphore.release(); + commitFuture.complete(null); } } - commitFuture.complete(null); return commitFuture; } @Override - public synchronized CompletableFuture abortTxn(TxnID txnId, Consumer consumer) { + public CompletableFuture abortTxn(TxnID txnId, Consumer consumer) { + if (!checkIfReady()) { + return FutureUtil.failedFuture(new ServiceUnitNotReadyException("PendingAckHandle not replay complete!")); + } CompletableFuture abortFuture = new CompletableFuture<>(); + this.semaphore.acquireUninterruptibly(); if (this.cumulativeAckOfTransaction != null) { - if (this.cumulativeAckOfTransaction.getKey().equals(txnId)) { - this.cumulativeAckOfTransaction = null; - } - this.persistentSubscription.redeliverUnacknowledgedMessages(consumer); + pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendAbortMark(txnId, AckType.Individual).whenComplete((v, e) -> { + if (e != null) { + semaphore.release(); + abortFuture.completeExceptionally(e); + } else { + if (this.cumulativeAckOfTransaction.getKey().equals(txnId)) { + this.cumulativeAckOfTransaction = null; + } + this.persistentSubscription.redeliverUnacknowledgedMessages(consumer); + semaphore.release(); + abortFuture.complete(null); + } + }) + ).exceptionally(e -> { + semaphore.release(); + abortFuture.completeExceptionally(e); + return null; + }); } else if (this.individualAckOfTransaction != null){ HashMap pendingAckMessageForCurrentTxn = - individualAckOfTransaction.remove(txnId); + individualAckOfTransaction.get(txnId); if (pendingAckMessageForCurrentTxn != null) { - for (Entry entry : pendingAckMessageForCurrentTxn.entrySet()) { - if (entry.getValue().hasAckSet() && individualAckPositions.containsKey(entry.getValue())) { - BitSetRecyclable thisBitSet = BitSetRecyclable.valueOf(entry.getValue().getAckSet()); - thisBitSet.flip(0, individualAckPositions.get(entry.getValue()).right); - BitSetRecyclable otherBitSet = - BitSetRecyclable.valueOf(individualAckPositions.get(entry.getValue()).left.getAckSet()); - otherBitSet.or(thisBitSet); - individualAckPositions.get(entry.getKey()).left.setAckSet(otherBitSet.toLongArray()); - otherBitSet.recycle(); - thisBitSet.recycle(); - } else { - individualAckPositions.remove(entry.getValue()); - } - } - this.persistentSubscription.redeliverUnacknowledgedMessages(consumer, - new ArrayList<>(pendingAckMessageForCurrentTxn.values())); + pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendAbortMark(txnId, AckType.Individual).whenComplete((v, e) -> { + if (e != null) { + semaphore.release(); + abortFuture.completeExceptionally(e); + } else { + individualAckAbortCommon(txnId, pendingAckMessageForCurrentTxn); + this.persistentSubscription.redeliverUnacknowledgedMessages(consumer, + new ArrayList<>(pendingAckMessageForCurrentTxn.values())); + semaphore.release(); + abortFuture.complete(null); + } + }) + ).exceptionally(e -> { + semaphore.release(); + abortFuture.completeExceptionally(e); + return null; + }); + } else { + semaphore.release(); + abortFuture.complete(null); } + } else { + semaphore.release(); + abortFuture.complete(null); } - abortFuture.complete(null); return abortFuture; } @Override - public synchronized void syncBatchPositionAckSetForTransaction(PositionImpl position) { - if (individualAckPositions == null) { - individualAckPositions = new HashMap<>(); - } - //sync don't carry the batch size - //when one position is ack by transaction the batch size is for `and` operation. - if (!individualAckPositions.containsKey(position)) { - this.individualAckPositions.put(position, new MutablePair<>(position, 0)); - } else { - andAckSet(this.individualAckPositions.get(position).left, position); + public void syncBatchPositionAckSetForTransaction(PositionImpl position) { + this.semaphore.acquireUninterruptibly(); + try { + if (individualAckPositions == null) { + individualAckPositions = new HashMap<>(); + } + // sync don't carry the batch size + // when one position is ack by transaction the batch size is for `and` operation. + if (!individualAckPositions.containsKey(position)) { + this.individualAckPositions.put(position, new MutablePair<>(position, 0)); + } else { + andAckSet(this.individualAckPositions.get(position).left, position); + } + } finally { + semaphore.release(); } } @Override public boolean checkIsCanDeleteConsumerPendingAck(PositionImpl position) { - if (!individualAckPositions.containsKey(position)) { - return true; + this.semaphore.acquireUninterruptibly(); + try { + if (!individualAckPositions.containsKey(position)) { + return true; + } else { + position = individualAckPositions.get(position).left; + if (position.hasAckSet()) { + BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(position.getAckSet()); + if (bitSetRecyclable.isEmpty()) { + bitSetRecyclable.recycle(); + return true; + } else { + bitSetRecyclable.recycle(); + return false; + } + } else { + return true; + } + } + } finally { + semaphore.release(); + } + } + + protected void handleAbort(TxnID txnID, AckType ackType) { + if (ackType == AckType.Cumulative) { + this.cumulativeAckOfTransaction = null; } else { - position = individualAckPositions.get(position).left; + if (this.individualAckOfTransaction != null) { + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.get(txnID); + individualAckAbortCommon(txnID, pendingAckMessageForCurrentTxn); + } + } + } + + private void individualAckAbortCommon(TxnID txnID, + HashMap pendingAckMessageForCurrentTxn) { + for (Map.Entry entry : + pendingAckMessageForCurrentTxn.entrySet()) { + if (entry.getValue().hasAckSet() && + individualAckPositions.containsKey(entry.getValue())) { + BitSetRecyclable thisBitSet = + BitSetRecyclable.valueOf(entry.getValue().getAckSet()); + thisBitSet.flip(0, individualAckPositions.get(entry.getValue()).right); + BitSetRecyclable otherBitSet = + BitSetRecyclable.valueOf(individualAckPositions + .get(entry.getValue()).left.getAckSet()); + otherBitSet.or(thisBitSet); + individualAckPositions.get(entry.getKey()) + .left.setAckSet(otherBitSet.toLongArray()); + otherBitSet.recycle(); + thisBitSet.recycle(); + } else { + individualAckPositions.remove(entry.getValue()); + } + } + individualAckOfTransaction.remove(txnID); + } + + protected void handleCommit(TxnID txnID, AckType ackType, Map properties) { + if (ackType == AckType.Cumulative) { + persistentSubscription.acknowledgeMessage( + Collections.singletonList(this.cumulativeAckOfTransaction.getValue()), + AckType.Cumulative, properties); + this.cumulativeAckOfTransaction = null; + } else { + if (this.individualAckOfTransaction != null) { + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.get(txnID); + individualAckCommitCommon(txnID, pendingAckMessageForCurrentTxn, null); + } + } + } + + private void individualAckCommitCommon(TxnID txnID, + HashMap pendingAckMessageForCurrentTxn, + Map properties) { + if (pendingAckMessageForCurrentTxn != null) { + persistentSubscription.acknowledgeMessage( + new ArrayList<>(pendingAckMessageForCurrentTxn.values()), + AckType.Individual, properties); + individualAckOfTransaction.remove(txnID); + } + } + + protected void handleIndividualAck(TxnID txnID, List> positions) { + for (int i = 0; i < positions.size(); i++) { + if (log.isDebugEnabled()) { + log.debug("[{}][{}] TxnID:[{}] Individual acks on {}", topicName, + subName, txnID.toString(), positions); + } + if (individualAckOfTransaction == null) { + individualAckOfTransaction = new HashMap<>(); + } + + if (individualAckPositions == null) { + individualAckPositions = new HashMap<>(); + } + + PositionImpl position = positions.get(i).left; + if (position.hasAckSet()) { - BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(position.getAckSet()); - if (bitSetRecyclable.isEmpty()) { - bitSetRecyclable.recycle(); - return true; + + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); + + if (pendingAckMessageForCurrentTxn.containsKey(position)) { + andAckSet(pendingAckMessageForCurrentTxn.get(position), position); + } else { + pendingAckMessageForCurrentTxn.put(position, position); + } + + if (!individualAckPositions.containsKey(position)) { + this.individualAckPositions.put(position, positions.get(i)); } else { - bitSetRecyclable.recycle(); - return false; + MutablePair positionPair = + this.individualAckPositions.get(position); + positionPair.setRight(positions.get(i).right); + andAckSet(positionPair.getLeft(), position); } + } else { - return true; + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); + pendingAckMessageForCurrentTxn.put(position, position); + this.individualAckPositions.putIfAbsent(position, positions.get(i)); } } } + protected void handleCumulativeAck(TxnID txnID, PositionImpl position) { + if (this.cumulativeAckOfTransaction == null) { + this.cumulativeAckOfTransaction = MutablePair.of(txnID, position); + } else if (this.cumulativeAckOfTransaction.getKey().equals(txnID) + && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) > 0) { + this.cumulativeAckOfTransaction.setValue(position); + } + } + + public String getTopicName() { + return topicName; + } + + public String getSubName() { + return subName; + } + @Override public void clearIndividualPosition(Position position) { if (individualAckPositions == null) { @@ -353,4 +579,8 @@ public void clearIndividualPosition(Position position) { } } + @Override + public CompletableFuture close() { + return this.pendingAckStoreFuture.thenAccept(PendingAckStore::closeAsync); + } } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java new file mode 100644 index 0000000000000..352962c33fa84 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; + +/** + * The implement of transaction pending ack store. + */ +public abstract class PendingAckHandleState { + + /** + * The state of the pending ack handle {@link PendingAckHandleState}. + */ + public enum State { + None, + Initializing, + Ready, + Close + } + + private static final AtomicReferenceFieldUpdater STATE_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(PendingAckHandleState.class, State.class, "state"); + + @SuppressWarnings("unused") + private volatile State state = null; + + public PendingAckHandleState(State state) { + STATE_UPDATER.set(this, state); + + } + + protected boolean changeToReadyState() { + return (STATE_UPDATER.compareAndSet(this, State.Initializing, State.Ready)); + } + + protected boolean changeToInitializingState() { + return STATE_UPDATER.compareAndSet(this, State.None, State.Initializing); + } + + protected boolean changeToCloseState() { + return (STATE_UPDATER.compareAndSet(this, State.Ready, State.Close) + || STATE_UPDATER.compareAndSet(this, State.None, State.Close) + || STATE_UPDATER.compareAndSet(this, State.Initializing, State.Close)); + } + + public boolean checkIfReady() { + return STATE_UPDATER.get(this) == State.Ready; + } + + public State getState() { + return STATE_UPDATER.get(this); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java new file mode 100644 index 0000000000000..e2910bbaf08f8 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java @@ -0,0 +1,1254 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: pulsar-broker/src/main/proto/TransactionPendingAck.proto + +package org.apache.pulsar.broker.transaction.proto; + +public final class TransactionPendingAck { + private TransactionPendingAck() {} + public static void registerAllExtensions( + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite registry) { + } + public enum PendingAckOp + implements org.apache.pulsar.shaded.com.google.protobuf.v241.Internal.EnumLite { + ACK(0, 0), + COMMIT(1, 1), + ABORT(2, 2), + ; + + public static final int ACK_VALUE = 0; + public static final int COMMIT_VALUE = 1; + public static final int ABORT_VALUE = 2; + + + public final int getNumber() { return value; } + + public static PendingAckOp valueOf(int value) { + switch (value) { + case 0: return ACK; + case 1: return COMMIT; + case 2: return ABORT; + default: return null; + } + } + + public static org.apache.pulsar.shaded.com.google.protobuf.v241.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static org.apache.pulsar.shaded.com.google.protobuf.v241.Internal.EnumLiteMap + internalValueMap = + new org.apache.pulsar.shaded.com.google.protobuf.v241.Internal.EnumLiteMap() { + public PendingAckOp findValueByNumber(int number) { + return PendingAckOp.valueOf(number); + } + }; + + private final int value; + + private PendingAckOp(int index, int value) { + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:pulsar.transaction.PendingAckOp) + } + + public interface PendingAckMetadataOrBuilder + extends org.apache.pulsar.shaded.com.google.protobuf.v241.MessageLiteOrBuilder { + + // optional int64 ledger_id = 1; + boolean hasLedgerId(); + long getLedgerId(); + + // optional int64 entry_id = 2; + boolean hasEntryId(); + long getEntryId(); + + // repeated int64 ack_set = 3; + java.util.List getAckSetList(); + int getAckSetCount(); + long getAckSet(int index); + + // optional int32 batch_size = 4 [default = 0]; + boolean hasBatchSize(); + int getBatchSize(); + } + public static final class PendingAckMetadata extends + org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite + implements PendingAckMetadataOrBuilder, org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream.ByteBufGeneratedMessage { + // Use PendingAckMetadata.newBuilder() to construct. + private io.netty.util.Recycler.Handle handle; + private PendingAckMetadata(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + } + + private static final io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected PendingAckMetadata newObject(Handle handle) { + return new PendingAckMetadata(handle); + } + }; + + public void recycle() { + this.initFields(); + this.memoizedIsInitialized = -1; + this.bitField0_ = 0; + this.memoizedSerializedSize = -1; + if (handle != null) { RECYCLER.recycle(this, handle); } + } + + private PendingAckMetadata(boolean noInit) {} + + private static final PendingAckMetadata defaultInstance; + public static PendingAckMetadata getDefaultInstance() { + return defaultInstance; + } + + public PendingAckMetadata getDefaultInstanceForType() { + return defaultInstance; + } + + private int bitField0_; + // optional int64 ledger_id = 1; + public static final int LEDGER_ID_FIELD_NUMBER = 1; + private long ledgerId_; + public boolean hasLedgerId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getLedgerId() { + return ledgerId_; + } + + // optional int64 entry_id = 2; + public static final int ENTRY_ID_FIELD_NUMBER = 2; + private long entryId_; + public boolean hasEntryId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getEntryId() { + return entryId_; + } + + // repeated int64 ack_set = 3; + public static final int ACK_SET_FIELD_NUMBER = 3; + private java.util.List ackSet_; + public java.util.List + getAckSetList() { + return ackSet_; + } + public int getAckSetCount() { + return ackSet_.size(); + } + public long getAckSet(int index) { + return ackSet_.get(index); + } + + // optional int32 batch_size = 4 [default = 0]; + public static final int BATCH_SIZE_FIELD_NUMBER = 4; + private int batchSize_; + public boolean hasBatchSize() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public int getBatchSize() { + return batchSize_; + } + + private void initFields() { + ledgerId_ = 0L; + entryId_ = 0L; + ackSet_ = java.util.Collections.emptyList();; + batchSize_ = 0; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream output) + throws java.io.IOException { + throw new RuntimeException("Cannot use CodedOutputStream"); + } + + public void writeTo(org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, ledgerId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeInt64(2, entryId_); + } + for (int i = 0; i < ackSet_.size(); i++) { + output.writeInt64(3, ackSet_.get(i)); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt32(4, batchSize_); + } + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeInt64Size(1, ledgerId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeInt64Size(2, entryId_); + } + { + int dataSize = 0; + for (int i = 0; i < ackSet_.size(); i++) { + dataSize += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeInt64SizeNoTag(ackSet_.get(i)); + } + size += dataSize; + size += 1 * getAckSetList().size(); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeInt32Size(4, batchSize_); + } + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString data) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString data, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom(byte[] data) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( + byte[] data, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( + java.io.InputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseDelimitedFrom( + java.io.InputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + public static final class Builder extends + org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite.Builder< + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata, Builder> + implements org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataOrBuilder, org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream.ByteBufMessageBuilder { + // Construct using org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.newBuilder() + private final io.netty.util.Recycler.Handle handle; + private Builder(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + maybeForceBuilderInitialization(); + } + private final static io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected Builder newObject(io.netty.util.Recycler.Handle handle) { + return new Builder(handle); + } + }; + + public void recycle() { + clear(); + if (handle != null) {RECYCLER.recycle(this, handle);} + } + + private void maybeForceBuilderInitialization() { + } + private static Builder create() { + return RECYCLER.get(); + } + + public Builder clear() { + super.clear(); + ledgerId_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + entryId_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + ackSet_ = java.util.Collections.emptyList();; + bitField0_ = (bitField0_ & ~0x00000004); + batchSize_ = 0; + bitField0_ = (bitField0_ & ~0x00000008); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata getDefaultInstanceForType() { + return org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.getDefaultInstance(); + } + + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata build() { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata buildParsed() + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata buildPartial() { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata result = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.RECYCLER.get(); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.ledgerId_ = ledgerId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.entryId_ = entryId_; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + ackSet_ = java.util.Collections.unmodifiableList(ackSet_); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.ackSet_ = ackSet_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000004; + } + result.batchSize_ = batchSize_; + result.bitField0_ = to_bitField0_; + return result; + } + + public Builder mergeFrom(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata other) { + if (other == org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.getDefaultInstance()) return this; + if (other.hasLedgerId()) { + setLedgerId(other.getLedgerId()); + } + if (other.hasEntryId()) { + setEntryId(other.getEntryId()); + } + if (!other.ackSet_.isEmpty()) { + if (ackSet_.isEmpty()) { + ackSet_ = other.ackSet_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensureAckSetIsMutable(); + ackSet_.addAll(other.ackSet_); + } + + } + if (other.hasBatchSize()) { + setBatchSize(other.getBatchSize()); + } + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom(org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + throw new java.io.IOException("Merge from CodedInputStream is disabled"); + } + public Builder mergeFrom( + org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + + return this; + default: { + if (!input.skipField(tag)) { + + return this; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + ledgerId_ = input.readInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + entryId_ = input.readInt64(); + break; + } + case 24: { + ensureAckSetIsMutable(); + ackSet_.add(input.readInt64()); + break; + } + case 26: { + int length = input.readRawVarint32(); + int limit = input.pushLimit(length); + while (input.getBytesUntilLimit() > 0) { + addAckSet(input.readInt64()); + } + input.popLimit(limit); + break; + } + case 32: { + bitField0_ |= 0x00000008; + batchSize_ = input.readInt32(); + break; + } + } + } + } + + private int bitField0_; + + // optional int64 ledger_id = 1; + private long ledgerId_ ; + public boolean hasLedgerId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getLedgerId() { + return ledgerId_; + } + public Builder setLedgerId(long value) { + bitField0_ |= 0x00000001; + ledgerId_ = value; + + return this; + } + public Builder clearLedgerId() { + bitField0_ = (bitField0_ & ~0x00000001); + ledgerId_ = 0L; + + return this; + } + + // optional int64 entry_id = 2; + private long entryId_ ; + public boolean hasEntryId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getEntryId() { + return entryId_; + } + public Builder setEntryId(long value) { + bitField0_ |= 0x00000002; + entryId_ = value; + + return this; + } + public Builder clearEntryId() { + bitField0_ = (bitField0_ & ~0x00000002); + entryId_ = 0L; + + return this; + } + + // repeated int64 ack_set = 3; + private java.util.List ackSet_ = java.util.Collections.emptyList();; + private void ensureAckSetIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + ackSet_ = new java.util.ArrayList(ackSet_); + bitField0_ |= 0x00000004; + } + } + public java.util.List + getAckSetList() { + return java.util.Collections.unmodifiableList(ackSet_); + } + public int getAckSetCount() { + return ackSet_.size(); + } + public long getAckSet(int index) { + return ackSet_.get(index); + } + public Builder setAckSet( + int index, long value) { + ensureAckSetIsMutable(); + ackSet_.set(index, value); + + return this; + } + public Builder addAckSet(long value) { + ensureAckSetIsMutable(); + ackSet_.add(value); + + return this; + } + public Builder addAllAckSet( + java.lang.Iterable values) { + ensureAckSetIsMutable(); + super.addAll(values, ackSet_); + + return this; + } + public Builder clearAckSet() { + ackSet_ = java.util.Collections.emptyList();; + bitField0_ = (bitField0_ & ~0x00000004); + + return this; + } + + // optional int32 batch_size = 4 [default = 0]; + private int batchSize_ ; + public boolean hasBatchSize() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getBatchSize() { + return batchSize_; + } + public Builder setBatchSize(int value) { + bitField0_ |= 0x00000008; + batchSize_ = value; + + return this; + } + public Builder clearBatchSize() { + bitField0_ = (bitField0_ & ~0x00000008); + batchSize_ = 0; + + return this; + } + + // @@protoc_insertion_point(builder_scope:pulsar.transaction.PendingAckMetadata) + } + + static { + defaultInstance = new PendingAckMetadata(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:pulsar.transaction.PendingAckMetadata) + } + + public interface PendingAckMetadataEntryOrBuilder + extends org.apache.pulsar.shaded.com.google.protobuf.v241.MessageLiteOrBuilder { + + // optional uint64 txnid_least_bits = 1; + boolean hasTxnidLeastBits(); + long getTxnidLeastBits(); + + // optional uint64 txnid_most_bits = 2; + boolean hasTxnidMostBits(); + long getTxnidMostBits(); + + // optional .pulsar.transaction.PendingAckOp pending_ack_op = 3; + boolean hasPendingAckOp(); + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp getPendingAckOp(); + + // optional .pulsar.proto.CommandAck.AckType ack_type = 4; + boolean hasAckType(); + org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType getAckType(); + + // repeated .pulsar.transaction.PendingAckMetadata pendingAckMetadata = 5; + java.util.List + getPendingAckMetadataList(); + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata getPendingAckMetadata(int index); + int getPendingAckMetadataCount(); + } + public static final class PendingAckMetadataEntry extends + org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite + implements PendingAckMetadataEntryOrBuilder, org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream.ByteBufGeneratedMessage { + // Use PendingAckMetadataEntry.newBuilder() to construct. + private io.netty.util.Recycler.Handle handle; + private PendingAckMetadataEntry(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + } + + private static final io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected PendingAckMetadataEntry newObject(Handle handle) { + return new PendingAckMetadataEntry(handle); + } + }; + + public void recycle() { + this.initFields(); + this.memoizedIsInitialized = -1; + this.bitField0_ = 0; + this.memoizedSerializedSize = -1; + if (handle != null) { RECYCLER.recycle(this, handle); } + } + + private PendingAckMetadataEntry(boolean noInit) {} + + private static final PendingAckMetadataEntry defaultInstance; + public static PendingAckMetadataEntry getDefaultInstance() { + return defaultInstance; + } + + public PendingAckMetadataEntry getDefaultInstanceForType() { + return defaultInstance; + } + + private int bitField0_; + // optional uint64 txnid_least_bits = 1; + public static final int TXNID_LEAST_BITS_FIELD_NUMBER = 1; + private long txnidLeastBits_; + public boolean hasTxnidLeastBits() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getTxnidLeastBits() { + return txnidLeastBits_; + } + + // optional uint64 txnid_most_bits = 2; + public static final int TXNID_MOST_BITS_FIELD_NUMBER = 2; + private long txnidMostBits_; + public boolean hasTxnidMostBits() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getTxnidMostBits() { + return txnidMostBits_; + } + + // optional .pulsar.transaction.PendingAckOp pending_ack_op = 3; + public static final int PENDING_ACK_OP_FIELD_NUMBER = 3; + private org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp pendingAckOp_; + public boolean hasPendingAckOp() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp getPendingAckOp() { + return pendingAckOp_; + } + + // optional .pulsar.proto.CommandAck.AckType ack_type = 4; + public static final int ACK_TYPE_FIELD_NUMBER = 4; + private org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType ackType_; + public boolean hasAckType() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType getAckType() { + return ackType_; + } + + // repeated .pulsar.transaction.PendingAckMetadata pendingAckMetadata = 5; + public static final int PENDINGACKMETADATA_FIELD_NUMBER = 5; + private java.util.List pendingAckMetadata_; + public java.util.List getPendingAckMetadataList() { + return pendingAckMetadata_; + } + public java.util.List + getPendingAckMetadataOrBuilderList() { + return pendingAckMetadata_; + } + public int getPendingAckMetadataCount() { + return pendingAckMetadata_.size(); + } + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata getPendingAckMetadata(int index) { + return pendingAckMetadata_.get(index); + } + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataOrBuilder getPendingAckMetadataOrBuilder( + int index) { + return pendingAckMetadata_.get(index); + } + + private void initFields() { + txnidLeastBits_ = 0L; + txnidMostBits_ = 0L; + pendingAckOp_ = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.ACK; + ackType_ = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.Individual; + pendingAckMetadata_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream output) + throws java.io.IOException { + throw new RuntimeException("Cannot use CodedOutputStream"); + } + + public void writeTo(org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeUInt64(1, txnidLeastBits_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeUInt64(2, txnidMostBits_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeEnum(3, pendingAckOp_.getNumber()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeEnum(4, ackType_.getNumber()); + } + for (int i = 0; i < pendingAckMetadata_.size(); i++) { + output.writeMessage(5, pendingAckMetadata_.get(i)); + } + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeUInt64Size(1, txnidLeastBits_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeUInt64Size(2, txnidMostBits_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeEnumSize(3, pendingAckOp_.getNumber()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeEnumSize(4, ackType_.getNumber()); + } + for (int i = 0; i < pendingAckMetadata_.size(); i++) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeMessageSize(5, pendingAckMetadata_.get(i)); + } + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString data) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString data, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom(byte[] data) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( + byte[] data, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( + java.io.InputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseDelimitedFrom( + java.io.InputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + public static final class Builder extends + org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite.Builder< + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry, Builder> + implements org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntryOrBuilder, org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream.ByteBufMessageBuilder { + // Construct using org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry.newBuilder() + private final io.netty.util.Recycler.Handle handle; + private Builder(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + maybeForceBuilderInitialization(); + } + private final static io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected Builder newObject(io.netty.util.Recycler.Handle handle) { + return new Builder(handle); + } + }; + + public void recycle() { + clear(); + if (handle != null) {RECYCLER.recycle(this, handle);} + } + + private void maybeForceBuilderInitialization() { + } + private static Builder create() { + return RECYCLER.get(); + } + + public Builder clear() { + super.clear(); + txnidLeastBits_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + txnidMostBits_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + pendingAckOp_ = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.ACK; + bitField0_ = (bitField0_ & ~0x00000004); + ackType_ = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.Individual; + bitField0_ = (bitField0_ & ~0x00000008); + pendingAckMetadata_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000010); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry getDefaultInstanceForType() { + return org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry.getDefaultInstance(); + } + + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry build() { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry buildParsed() + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry buildPartial() { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry result = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry.RECYCLER.get(); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.txnidLeastBits_ = txnidLeastBits_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.txnidMostBits_ = txnidMostBits_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.pendingAckOp_ = pendingAckOp_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.ackType_ = ackType_; + if (((bitField0_ & 0x00000010) == 0x00000010)) { + pendingAckMetadata_ = java.util.Collections.unmodifiableList(pendingAckMetadata_); + bitField0_ = (bitField0_ & ~0x00000010); + } + result.pendingAckMetadata_ = pendingAckMetadata_; + result.bitField0_ = to_bitField0_; + return result; + } + + public Builder mergeFrom(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry other) { + if (other == org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry.getDefaultInstance()) return this; + if (other.hasTxnidLeastBits()) { + setTxnidLeastBits(other.getTxnidLeastBits()); + } + if (other.hasTxnidMostBits()) { + setTxnidMostBits(other.getTxnidMostBits()); + } + if (other.hasPendingAckOp()) { + setPendingAckOp(other.getPendingAckOp()); + } + if (other.hasAckType()) { + setAckType(other.getAckType()); + } + if (!other.pendingAckMetadata_.isEmpty()) { + if (pendingAckMetadata_.isEmpty()) { + pendingAckMetadata_ = other.pendingAckMetadata_; + bitField0_ = (bitField0_ & ~0x00000010); + } else { + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.addAll(other.pendingAckMetadata_); + } + + } + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom(org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + throw new java.io.IOException("Merge from CodedInputStream is disabled"); + } + public Builder mergeFrom( + org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + + return this; + default: { + if (!input.skipField(tag)) { + + return this; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + txnidLeastBits_ = input.readUInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + txnidMostBits_ = input.readUInt64(); + break; + } + case 24: { + int rawValue = input.readEnum(); + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp value = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.valueOf(rawValue); + if (value != null) { + bitField0_ |= 0x00000004; + pendingAckOp_ = value; + } + break; + } + case 32: { + int rawValue = input.readEnum(); + org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType value = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.valueOf(rawValue); + if (value != null) { + bitField0_ |= 0x00000008; + ackType_ = value; + } + break; + } + case 42: { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.Builder subBuilder = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addPendingAckMetadata(subBuilder.buildPartial()); + break; + } + } + } + } + + private int bitField0_; + + // optional uint64 txnid_least_bits = 1; + private long txnidLeastBits_ ; + public boolean hasTxnidLeastBits() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getTxnidLeastBits() { + return txnidLeastBits_; + } + public Builder setTxnidLeastBits(long value) { + bitField0_ |= 0x00000001; + txnidLeastBits_ = value; + + return this; + } + public Builder clearTxnidLeastBits() { + bitField0_ = (bitField0_ & ~0x00000001); + txnidLeastBits_ = 0L; + + return this; + } + + // optional uint64 txnid_most_bits = 2; + private long txnidMostBits_ ; + public boolean hasTxnidMostBits() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getTxnidMostBits() { + return txnidMostBits_; + } + public Builder setTxnidMostBits(long value) { + bitField0_ |= 0x00000002; + txnidMostBits_ = value; + + return this; + } + public Builder clearTxnidMostBits() { + bitField0_ = (bitField0_ & ~0x00000002); + txnidMostBits_ = 0L; + + return this; + } + + // optional .pulsar.transaction.PendingAckOp pending_ack_op = 3; + private org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp pendingAckOp_ = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.ACK; + public boolean hasPendingAckOp() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp getPendingAckOp() { + return pendingAckOp_; + } + public Builder setPendingAckOp(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + pendingAckOp_ = value; + + return this; + } + public Builder clearPendingAckOp() { + bitField0_ = (bitField0_ & ~0x00000004); + pendingAckOp_ = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.ACK; + + return this; + } + + // optional .pulsar.proto.CommandAck.AckType ack_type = 4; + private org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType ackType_ = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.Individual; + public boolean hasAckType() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType getAckType() { + return ackType_; + } + public Builder setAckType(org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000008; + ackType_ = value; + + return this; + } + public Builder clearAckType() { + bitField0_ = (bitField0_ & ~0x00000008); + ackType_ = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.Individual; + + return this; + } + + // repeated .pulsar.transaction.PendingAckMetadata pendingAckMetadata = 5; + private java.util.List pendingAckMetadata_ = + java.util.Collections.emptyList(); + private void ensurePendingAckMetadataIsMutable() { + if (!((bitField0_ & 0x00000010) == 0x00000010)) { + pendingAckMetadata_ = new java.util.ArrayList(pendingAckMetadata_); + bitField0_ |= 0x00000010; + } + } + + public java.util.List getPendingAckMetadataList() { + return java.util.Collections.unmodifiableList(pendingAckMetadata_); + } + public int getPendingAckMetadataCount() { + return pendingAckMetadata_.size(); + } + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata getPendingAckMetadata(int index) { + return pendingAckMetadata_.get(index); + } + public Builder setPendingAckMetadata( + int index, org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.set(index, value); + + return this; + } + public Builder setPendingAckMetadata( + int index, org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.Builder builderForValue) { + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.set(index, builderForValue.build()); + + return this; + } + public Builder addPendingAckMetadata(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.add(value); + + return this; + } + public Builder addPendingAckMetadata( + int index, org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.add(index, value); + + return this; + } + public Builder addPendingAckMetadata( + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.Builder builderForValue) { + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.add(builderForValue.build()); + + return this; + } + public Builder addPendingAckMetadata( + int index, org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.Builder builderForValue) { + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.add(index, builderForValue.build()); + + return this; + } + public Builder addAllPendingAckMetadata( + java.lang.Iterable values) { + ensurePendingAckMetadataIsMutable(); + super.addAll(values, pendingAckMetadata_); + + return this; + } + public Builder clearPendingAckMetadata() { + pendingAckMetadata_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000010); + + return this; + } + public Builder removePendingAckMetadata(int index) { + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.remove(index); + + return this; + } + + // @@protoc_insertion_point(builder_scope:pulsar.transaction.PendingAckMetadataEntry) + } + + static { + defaultInstance = new PendingAckMetadataEntry(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:pulsar.transaction.PendingAckMetadataEntry) + } + + + static { + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index 8e1a605fa1060..494d7bd03566a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -116,6 +116,8 @@ protected void startBroker() throws Exception { conf.setWebServicePort(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); conf.setTransactionCoordinatorEnabled(true); + conf.setMaxPendingAckTimerTaskIntervalTime(2); + conf.setMinPendingAckTimerTaskIntervalTime(2); serviceConfigurationList.add(conf); PulsarService pulsar = spy(new PulsarService(conf)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java index 285d32741a642..cd6bda8d1ee9b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java @@ -32,6 +32,7 @@ import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.SubscriptionType; @@ -43,6 +44,7 @@ import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -188,8 +190,6 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { .subscriptionName(subscriptionName) .enableBatchIndexAcknowledgment(true) .subscriptionType(SubscriptionType.Shared) - .ackTimeout(2, TimeUnit.SECONDS) - .acknowledgmentGroupTime(0, TimeUnit.MICROSECONDS) .subscribe(); @Cleanup @@ -207,6 +207,7 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { ConcurrentSkipListMap batchDeletedIndexes = null; + MessageId[] messageIds = new MessageId[2]; for (int retryCnt = 0; retryCnt < 2; retryCnt++) { int messageCnt = 1000; @@ -220,15 +221,20 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { Transaction commitTxn = getTxn(); //send 1000 and ack 999, and test the consumer pending ack has already clear 999 messages - for (int i = 0; i < messageCnt - 1; i++) { + for (int i = 0; i < messageCnt; i++) { message = consumer.receive(2, TimeUnit.SECONDS); Assert.assertNotNull(message); - if (i % 2 == 0) { - consumer.acknowledgeAsync(message.getMessageId(), commitTxn).get(); - log.info("txn receive msgId: {}, count: {}", message.getMessageId(), i); + // in order to free up 2 position to judge the consumer pending ack delete + if (i != 500) { + if (i % 2 == 0) { + consumer.acknowledgeAsync(message.getMessageId(), commitTxn).get(); + log.info("txn receive msgId: {}, count: {}", message.getMessageId(), i); + } else { + consumer.acknowledge(message.getMessageId()); + log.info("normal receive msgId: {}, count: {}", message.getMessageId(), i); + } } else { - consumer.acknowledge(message.getMessageId()); - log.info("normal receive msgId: {}, count: {}", message.getMessageId(), i); + messageIds[retryCnt] = message.getMessageId(); } } @@ -244,28 +250,53 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { if (completableFuture != null) { Optional topic = completableFuture.get(); if (topic.isPresent()) { - persistentSubscription = (PersistentSubscription) topic.get().getSubscription(subscriptionName); + final PersistentSubscription testPersistentSubscription = + (PersistentSubscription) topic.get().getSubscription(subscriptionName); field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); field.setAccessible(true); - pendingAckHandle = (PendingAckHandleImpl) field.get(persistentSubscription); + pendingAckHandle = (PendingAckHandleImpl) field.get(testPersistentSubscription); field = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction"); field.setAccessible(true); individualAckOfTransaction = (HashMap>) field.get(pendingAckHandle); assertTrue(individualAckOfTransaction.isEmpty()); - managedCursor = (ManagedCursorImpl) persistentSubscription.getCursor(); + managedCursor = (ManagedCursorImpl) testPersistentSubscription.getCursor(); field = ManagedCursorImpl.class.getDeclaredField("batchDeletedIndexes"); field.setAccessible(true); batchDeletedIndexes = (ConcurrentSkipListMap) field.get(managedCursor); if (retryCnt == 0) { //one message are not ack - assertEquals(batchDeletedIndexes.size(), 1); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1); + Awaitility.await().atMost(2000, TimeUnit.MILLISECONDS).until(() -> { + return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 1; + }); } else { //two message are not ack + Awaitility.await().atMost(2000, TimeUnit.MILLISECONDS).until(() -> { + System.out.println(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size()); + return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 2; + }); + + Transaction commitTwice = getTxn(); + + consumer.acknowledge(messageIds[0]); + assertEquals(batchDeletedIndexes.size(), 2); + Awaitility.await().atMost(2000, TimeUnit.MILLISECONDS).until(() -> { + return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 1; + }); + + // this test is for the last message has been cleared in this consumer pending acks + // and it won't clear the last message in cursor batch index ack set + consumer.acknowledgeAsync(messageIds[1], commitTwice).get(); assertEquals(batchDeletedIndexes.size(), 1); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1); + Awaitility.await().atMost(2000, TimeUnit.MILLISECONDS).until(() -> { + return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 0; + }); + + // the messages has been produced were all acked, the memory in broker for the messages has been cleared. + commitTwice.commit().get(); + assertEquals(batchDeletedIndexes.size(), 0); + assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0); } count++; } @@ -273,26 +304,6 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { } assertEquals(count, 1); } - - Transaction commitTwice = getTxn(); - - //this message is in one batch point - Message message = consumer.receive(); - consumer.acknowledge(message.getMessageId()); - assertEquals(batchDeletedIndexes.size(), 1); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1); - - // this test is for the last message has been cleared in this consumer pending acks - // and it won't clear the last message in cursor batch index ack set - message = consumer.receive(); - consumer.acknowledgeAsync(message.getMessageId(), commitTwice).get(); - assertEquals(batchDeletedIndexes.size(), 1); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0); - - // the messages has been produced were all acked, the memory in broker for the messages has been cleared. - commitTwice.commit().get(); - assertEquals(batchDeletedIndexes.size(), 0); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0); } private Transaction getTxn() throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java new file mode 100644 index 0000000000000..c568f82c59426 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack; + +import com.google.common.collect.Sets; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; +import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +/** + * Test for consuming transaction messages. + */ +@Slf4j +public class PendingAckPersistentTest extends TransactionTestBase { + + private final static String PENDING_ACK_REPLAY_TOPIC = "persistent://public/txn/pending-ack-replay"; + + @BeforeMethod + public void setup() throws Exception { + setBrokerCount(1); + super.internalSetup(); + + String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); + String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1]; + admin.clusters().createCluster(CLUSTER_NAME, new ClusterData("http://localhost:" + webServicePort)); + admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), + new TenantInfo(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); + admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), 16); + admin.tenants().createTenant("public", + new TenantInfo(Sets.newHashSet(), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace("public/txn", 10); + admin.topics().createNonPartitionedTopic(PENDING_ACK_REPLAY_TOPIC); + + pulsarClient = PulsarClient.builder() + .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) + .statsInterval(0, TimeUnit.SECONDS) + .enableTransaction(true) + .build(); + + Thread.sleep(1000 * 3); + } + + @AfterMethod(alwaysRun = true) + protected void cleanup() { + super.internalCleanup(); + } + + @Test + public void pendingAckSharedReplayTest() throws Exception { + int messageCount = 1000; + String subName = "shared-test"; + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(PENDING_ACK_REPLAY_TOPIC) + .enableBatching(true) + .batchingMaxMessages(200) + .create(); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(PENDING_ACK_REPLAY_TOPIC) + .subscriptionName("shared-test") + .subscriptionType(SubscriptionType.Shared) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + Transaction abortTxn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + + List pendingAckMessageIds = new ArrayList<>(); + List normalAckMessageIds = new ArrayList<>(); + for (int i = 0; i < messageCount; i++) { + producer.send("Hello Pulsar!".getBytes()); + Message message = consumer.receive(); + if (i % 2 == 0) { + consumer.acknowledgeAsync(message.getMessageId(), abortTxn).get(); + pendingAckMessageIds.add(message.getMessageId()); + } else { + normalAckMessageIds.add(message.getMessageId()); + } + } + + //in order to test pending ack replay + admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); + Transaction commitTxn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + + // this messageIds are ack by transaction + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + try { + consumer.acknowledgeAsync(pendingAckMessageIds.get(i), txn).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + } + // this messageIds are not ack by transaction + for (int i = 0; i < normalAckMessageIds.size(); i++) { + consumer.acknowledgeAsync(normalAckMessageIds.get(i), commitTxn).get(); + } + + txn.abort().get(); + // commit this txn , normalAckMessageIds are in pending ack state + commitTxn.commit().get(); + // abort this txn, pendingAckMessageIds are delete from pending ack state + abortTxn.abort().get(); + + // replay this pending ack + admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); + Awaitility.await().atMost(4000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + + abortTxn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + + commitTxn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + // normalAckMessageIds are ack and then commit, so ack fail + for (int i = 0; i < normalAckMessageIds.size(); i++) { + try { + consumer.acknowledgeAsync(normalAckMessageIds.get(i), abortTxn).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + } + + // pendingAckMessageIds are all abort, so can ack again + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + consumer.acknowledgeAsync(pendingAckMessageIds.get(i), commitTxn).get(); + } + + abortTxn.abort().get(); + commitTxn.commit().get(); + + PersistentTopic topic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() + .getTopic(TopicName.get(PENDING_ACK_REPLAY_TOPIC).toString(), false).get().get(); + Field field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); + field.setAccessible(true); + PendingAckHandleImpl pendingAckHandle = + (PendingAckHandleImpl) field.get(topic.getSubscription(subName)); + field = PendingAckHandleImpl.class.getDeclaredField("pendingAckStoreFuture"); + field.setAccessible(true); + CompletableFuture pendingAckStoreCompletableFuture = + (CompletableFuture) field.get(pendingAckHandle); + pendingAckStoreCompletableFuture.get(); + + field = MLPendingAckStore.class.getDeclaredField("cursor"); + field.setAccessible(true); + + ManagedCursor managedCursor = (ManagedCursor) field.get(pendingAckStoreCompletableFuture.get()); + + // in order to check out the pending ack cursor is clear whether or not. + Awaitility.await() + .atMost(15000, TimeUnit.MILLISECONDS) + .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition()) + .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == 0); + } +} From e4a11de81ab064b5840eaf9daf74d892c4e7960c Mon Sep 17 00:00:00 2001 From: congbo Date: Wed, 9 Dec 2020 23:45:19 +0800 Subject: [PATCH 02/18] Fix some named --- .../pulsar/broker/ServiceConfiguration.java | 10 +- .../pendingack/PendingAckStore.java | 7 +- .../TransactionPendingAckStoreProvider.java | 3 +- .../impl/InMemoryPendingAckStore.java | 2 +- .../impl/MLPendingAckStoreProvider.java | 4 +- .../pendingack/impl/PendingAckHandleImpl.java | 12 ++- .../pendingack/PendingAckPersistentTest.java | 99 ++++++++++++++++++- 7 files changed, 119 insertions(+), 18 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 62833c71eb9f4..8ddc99d8197a5 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1879,21 +1879,21 @@ public class ServiceConfiguration implements PulsarConfiguration { "org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStoreProvider"; @FieldContext( - category = CATEGORY_SERVER, + category = CATEGORY_TRANSACTION, doc = "Number of threads to use for pulsar transaction replay PendingAckStore or TransactionBuffer." + "Default is 5" ) private int numTransactionReplayThreadPoolSize = 5; @FieldContext( - category = CATEGORY_SERVER, - doc = "Interval time of pending ack timer task, unit is seconds." + category = CATEGORY_TRANSACTION, + doc = "Max interval time of pending ack timer task, unit is seconds." ) private int minPendingAckTimerTaskIntervalTime = 3600; @FieldContext( - category = CATEGORY_SERVER, - doc = "Interval time of pending ack timer task, unit is seconds." + category = CATEGORY_TRANSACTION, + doc = "Min interval time of pending ack timer task, unit is seconds." ) private int maxPendingAckTimerTaskIntervalTime = 36000; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java index 9d20f05f5c685..b9c4c104c4ff8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java @@ -37,11 +37,14 @@ public interface PendingAckStore { * Replay pending ack to recover the pending ack subscription pending ack state. * * @param pendingAckHandle the handle of pending ack + * @param executorService the replay executor service */ - void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService getExecutor); + void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService executorService); /** * Close the transaction pending ack store. + * + * @return a future represents the result of this operation */ CompletableFuture closeAsync(); @@ -49,7 +52,7 @@ public interface PendingAckStore { * Append the individual pending ack operation to the ack persistent store. * * @param txnID {@link TxnID} transaction id. - * @param positions {@link PositionImpl} the pending ack postion. + * @param positions {@link List} the list of position and batch size. * @return a future represents the result of this operation */ CompletableFuture appendIndividualAck(TxnID txnID, List> positions); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java index 4104e36f75b41..61cf63eb3aee3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java @@ -55,12 +55,11 @@ static TransactionPendingAckStoreProvider newProvider(String providerClassName) * Open the pending ack store. * * @param subscription {@link PersistentSubscription} - * @param subName {@link String} * @return a future represents the result of the operation. * an instance of {@link PendingAckStore} is returned * if the operation succeeds. */ - CompletableFuture newPendingAckStore(PersistentSubscription subscription, String subName); + CompletableFuture newPendingAckStore(PersistentSubscription subscription); /** * Close the provider and release all the resources. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java index 39cb094d55cef..3289ca144fcd1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java @@ -29,7 +29,7 @@ import java.util.concurrent.ScheduledExecutorService; /** - * The In memory implementation of {@link PendingAckStore}. + * In memory implementation of {@link PendingAckStore}. */ public class InMemoryPendingAckStore implements PendingAckStore { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java index b39909173ed66..b60de3916c57e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java @@ -48,7 +48,7 @@ public class MLPendingAckStoreProvider implements TransactionPendingAckStoreProv private static final long tickTimeMillis = 1L; @Override - public CompletableFuture newPendingAckStore(PersistentSubscription subscription, String subName) { + public CompletableFuture newPendingAckStore(PersistentSubscription subscription) { CompletableFuture pendingAckStoreFuture = new CompletableFuture<>(); if (subscription == null) { @@ -59,7 +59,7 @@ public CompletableFuture newPendingAckStore(PersistentSubscript PersistentTopic originPersistentTopic = (PersistentTopic) subscription.getTopic(); String pendingAckTopicName = MLPendingAckStore - .getTransactionPendingAckStoreSuffix(originPersistentTopic.getName(), subName); + .getTransactionPendingAckStoreSuffix(originPersistentTopic.getName(), subscription.getName()); originPersistentTopic.getBrokerService().getManagedLedgerFactory() .asyncOpen(TopicName.get(pendingAckTopicName).getPersistenceNamingEncoding(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 91245d8551b24..296dcda6902cf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -117,7 +117,7 @@ public PendingAckHandleImpl(PersistentSubscription persistentSubscription) { ((PersistentTopic) this.persistentSubscription.getTopic()) .getBrokerService().getPulsar().getTransactionPendingAckStoreProvider(); this.pendingAckStoreFuture = - pendingAckStoreProvider.newPendingAckStore(persistentSubscription, subName); + pendingAckStoreProvider.newPendingAckStore(persistentSubscription); this.pendingAckStoreFuture.thenAccept(pendingAckStore -> { if (pendingAckStore instanceof MLPendingAckStore) { @@ -196,6 +196,9 @@ && isAckSetOverlap(individualAckPositions.get(position).getLeft().getAckSet(), a this.pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendIndividualAck(txnID, positions).whenComplete((v, e) -> { if (e != null) { + //we also modify the in memory state when append fail, because we don't know the persistent + // state, when wereplay it, it will produce the wrong operation. so we append fail, we should + // wait tc time out or client abort this transaction. handleIndividualAck(txnID, positions); semaphore.release(); completableFuture.completeExceptionally(e); @@ -247,8 +250,8 @@ public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, log.debug("[{}][{}] TxnID:[{}] Cumulative ack on {}.", topicName, subName, txnID.toString(), position); } CompletableFuture completableFuture = new CompletableFuture<>(); - if (cumulativeAckOfTransaction != null && this.cumulativeAckOfTransaction.getKey().equals(txnID) - && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) <= 0) { + if (cumulativeAckOfTransaction != null && (!this.cumulativeAckOfTransaction.getKey().equals(txnID) + || compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) <= 0)) { String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to cumulative batch ack position: " + position + " within range of current " + "currentPosition: " + this.cumulativeAckOfTransaction.getValue(); @@ -259,6 +262,9 @@ && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) <= this.pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendCumulativeAck(txnID, position).whenComplete((v, e) -> { if (e != null) { + //we also modify the in memory state when append fail, because we don't know the persistent + // state, when wereplay it, it will produce the wrong operation. so we append fail, we should + // wait tc time out or client abort this transaction. handleCumulativeAck(txnID, position); semaphore.release(); completableFuture.completeExceptionally(e); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index c568f82c59426..daaabcf53e8a2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -97,9 +97,9 @@ protected void cleanup() { } @Test - public void pendingAckSharedReplayTest() throws Exception { + public void individualPendingAckReplayTest() throws Exception { int messageCount = 1000; - String subName = "shared-test"; + String subName = "individual-test"; @Cleanup Producer producer = pulsarClient.newProducer() @@ -111,7 +111,7 @@ public void pendingAckSharedReplayTest() throws Exception { @Cleanup Consumer consumer = pulsarClient.newConsumer() .topic(PENDING_ACK_REPLAY_TOPIC) - .subscriptionName("shared-test") + .subscriptionName(subName) .subscriptionType(SubscriptionType.Shared) .enableBatchIndexAcknowledgment(true) .subscribe(); @@ -211,4 +211,97 @@ public void pendingAckSharedReplayTest() throws Exception { .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition()) .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == 0); } + + @Test + public void cumulativePendingAckReplayTest() throws Exception { + int messageCount = 1000; + String subName = "cumulative-test"; + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(PENDING_ACK_REPLAY_TOPIC) + .enableBatching(true) + .batchingMaxMessages(200) + .create(); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(PENDING_ACK_REPLAY_TOPIC) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Failover) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + Transaction abortTxn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + + List pendingAckMessageIds = new ArrayList<>(); + for (int i = 0; i < messageCount; i++) { + producer.send("Hello Pulsar!".getBytes()); + } + + for (int i = 0; i < messageCount; i++) { + Message message = consumer.receive(); + pendingAckMessageIds.add(message.getMessageId()); + consumer.acknowledgeCumulativeAsync(message.getMessageId(), abortTxn).get(); + } + + admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + try { + consumer.acknowledgeCumulativeAsync(pendingAckMessageIds.get(i), txn).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + } + Transaction commitTxn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + abortTxn.abort().get(); + + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + consumer.acknowledgeCumulativeAsync(pendingAckMessageIds.get(i), commitTxn).get(); + } + commitTxn.commit().get(); + + admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); + Awaitility.await().atMost(4000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + try { + consumer.acknowledgeCumulativeAsync(pendingAckMessageIds.get(i), txn).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + } + + PersistentTopic topic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() + .getTopic(TopicName.get(PENDING_ACK_REPLAY_TOPIC).toString(), false).get().get(); + Field field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); + field.setAccessible(true); + PendingAckHandleImpl pendingAckHandle = + (PendingAckHandleImpl) field.get(topic.getSubscription(subName)); + field = PendingAckHandleImpl.class.getDeclaredField("pendingAckStoreFuture"); + field.setAccessible(true); + CompletableFuture pendingAckStoreCompletableFuture = + (CompletableFuture) field.get(pendingAckHandle); + pendingAckStoreCompletableFuture.get(); + + field = MLPendingAckStore.class.getDeclaredField("cursor"); + field.setAccessible(true); + + ManagedCursor managedCursor = (ManagedCursor) field.get(pendingAckStoreCompletableFuture.get()); + + // in order to check out the pending ack cursor is clear whether or not. + Awaitility.await() + .atMost(15000, TimeUnit.MILLISECONDS) + .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition()) + .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == 0); + } } From a17e9596eba589777486cfd0589d90923260ad30 Mon Sep 17 00:00:00 2001 From: congbo Date: Wed, 9 Dec 2020 19:08:38 +0800 Subject: [PATCH 03/18] Transaction pending ack persistent --- .../pulsar/broker/ServiceConfiguration.java | 26 + pulsar-broker/generate_protobuf.sh | 23 + pulsar-broker/generate_protobuf_docker.sh | 42 + .../persistent/PersistentSubscription.java | 14 +- .../pendingack/PendingAckHandle.java | 21 +- .../pendingack/PendingAckReplyCallBack.java | 39 + .../pendingack/PendingAckStore.java | 83 ++ .../TransactionPendingAckStoreProvider.java | 69 + ...ctionPendingAckStoreProviderException.java | 32 + .../impl/InMemoryPendingAckStore.java | 66 + .../impl/MLPendingAckReplyCallBack.java | 107 ++ .../pendingack/impl/MLPendingAckStore.java | 336 +++++ .../impl/MLPendingAckStoreProvider.java | 120 ++ .../impl/MLPendingAckStoreTimerTask.java | 200 +++ .../impl/PendingAckHandleDisabled.java | 8 +- .../pendingack/impl/PendingAckHandleImpl.java | 480 +++++-- .../impl/PendingAckHandleState.java | 70 + .../proto/TransactionPendingAck.java | 1254 +++++++++++++++++ .../PendingAckInMemoryDeleteTest.java | 79 +- .../pendingack/PendingAckPersistentTest.java | 214 +++ 20 files changed, 3111 insertions(+), 172 deletions(-) create mode 100755 pulsar-broker/generate_protobuf.sh create mode 100755 pulsar-broker/generate_protobuf_docker.sh create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTimerTask.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 34746f6cf6123..256223ce97eb1 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1925,6 +1925,32 @@ public class ServiceConfiguration implements PulsarConfiguration { private String transactionBufferProviderClassName = "org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferProvider"; + @FieldContext( + category = CATEGORY_TRANSACTION, + doc = "Class name for transaction pending ack store provider" + ) + private String transactionPendingAckStoreProviderClassName = + "org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStoreProvider"; + + @FieldContext( + category = CATEGORY_SERVER, + doc = "Number of threads to use for pulsar transaction replay PendingAckStore or TransactionBuffer." + + "Default is 5" + ) + private int numTransactionReplayThreadPoolSize = 5; + + @FieldContext( + category = CATEGORY_SERVER, + doc = "Interval time of pending ack timer task, unit is seconds." + ) + private int minPendingAckTimerTaskIntervalTime = 3600; + + @FieldContext( + category = CATEGORY_SERVER, + doc = "Interval time of pending ack timer task, unit is seconds." + ) + private int maxPendingAckTimerTaskIntervalTime = 36000; + /**** --- KeyStore TLS config variables --- ****/ @FieldContext( category = CATEGORY_KEYSTORE_TLS, diff --git a/pulsar-broker/generate_protobuf.sh b/pulsar-broker/generate_protobuf.sh new file mode 100755 index 0000000000000..d13e17db79605 --- /dev/null +++ b/pulsar-broker/generate_protobuf.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +PROTOC=${PROTOC:-protoc} +${PROTOC} --java_out=pulsar-broker/src/main/java pulsar-broker/src/main/proto/TransactionPendingAck.proto diff --git a/pulsar-broker/generate_protobuf_docker.sh b/pulsar-broker/generate_protobuf_docker.sh new file mode 100755 index 0000000000000..c51417f7f0eb4 --- /dev/null +++ b/pulsar-broker/generate_protobuf_docker.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Fail script in case of errors +set -e + +ROOT_DIR=$(git rev-parse --show-toplevel) +COMMON_DIR=$ROOT_DIR/ +cd $COMMON_DIR + +BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}" +BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-16.04}" + +IMAGE="$BUILD_IMAGE_NAME:$BUILD_IMAGE_VERSION" + +echo $IMAGE + +# Force to pull image in case it was updated +docker pull $IMAGE + +WORKDIR=/workdir +docker run -i \ + -v ${COMMON_DIR}:${WORKDIR} $IMAGE \ + bash -c "cd ${WORKDIR}; PROTOC=/pulsar/protobuf/src/protoc ./pulsar-broker/generate_protobuf.sh" + diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 19215f4a3a509..191be91c9395c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -173,6 +173,12 @@ void setReplicated(boolean replicated) { @Override public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { + if (pendingAckHandle instanceof PendingAckHandleImpl) { + if (!((PendingAckHandleImpl) pendingAckHandle).checkIfReady()) { + throw new SubscriptionBusyException("Pending ack handle not init " + + "complete! topicName : " + topicName + " subName : " + subName); + } + } cursor.updateLastActive(); if (IS_FENCED_UPDATER.get(this) == TRUE) { log.warn("Attempting to add consumer {} on a fenced subscription", consumer); @@ -742,11 +748,11 @@ public CompletableFuture close() { if (dispatcher != null && dispatcher.isConsumerConnected()) { return FutureUtil.failedFuture(new SubscriptionBusyException("Subscription has active consumers")); } - IS_FENCED_UPDATER.set(this, TRUE); - log.info("[{}][{}] Successfully closed subscription [{}]", topicName, subName, cursor); + return this.pendingAckHandle.close().thenAccept(v -> { + IS_FENCED_UPDATER.set(this, TRUE); + log.info("[{}][{}] Successfully closed subscription [{}]", topicName, subName, cursor); + }); } - - return CompletableFuture.completedFuture(null); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java index 002cb33e1d2a7..3ef5b2b8807b0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java @@ -27,7 +27,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.common.api.proto.CommandAck.AckType; +import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; import org.apache.pulsar.transaction.common.exception.TransactionConflictException; /** @@ -53,8 +53,7 @@ public interface PendingAckHandle { * @throws NotAllowedException if Use this method incorrectly eg. not use * PositionImpl or cumulative ack with a list of positions. */ - CompletableFuture individualAcknowledgeMessage(TxnID txnID, - List> positions); + CompletableFuture individualAcknowledgeMessage(TxnID txnID, List> positions); /** * Acknowledge message(s) for an ongoing transaction. @@ -83,12 +82,11 @@ CompletableFuture individualAcknowledgeMessage(TxnID txnID, /** * Commit a transaction. * - * @param txnID {@link TxnID} to identify the transaction. - * @param properties Additional user-defined properties that can be - * associated with a particular cursor position. + * @param txnID {@link TxnID} to identify the transaction. + * @param properties Additional user-defined properties that can be associated with a particular cursor position. * @return the future of this operation. */ - CompletableFuture commitTxn(TxnID txnID, Map properties); + CompletableFuture commitTxn(TxnID txnID, Map properties); /** * Abort a transaction. @@ -119,4 +117,11 @@ CompletableFuture individualAcknowledgeMessage(TxnID txnID, * @param position {@link Position} which position need to clear */ void clearIndividualPosition(Position position); -} + + /** + * Close the pending ack handle. + * + * @return the future of this operation. + */ + CompletableFuture close(); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java new file mode 100644 index 0000000000000..b34133974863b --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack; + +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry; + +/** + * Call back for pending ack reply. + */ +public interface PendingAckReplyCallBack { + + /** + * Pending ack replay complete callback for pending ack store. + */ + void replayComplete(); + + /** + * Handle metadata entry. + * + * @param pendingAckMetadataEntry {@link PendingAckMetadataEntry} the metadata entry of pending ack + */ + void handleMetadataEntry(PendingAckMetadataEntry pendingAckMetadataEntry); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java new file mode 100644 index 0000000000000..9d20f05f5c685 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack; + +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; + +/** + * To store transaction pending ack. + */ +public interface PendingAckStore { + /** + * Replay pending ack to recover the pending ack subscription pending ack state. + * + * @param pendingAckHandle the handle of pending ack + */ + void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService getExecutor); + + /** + * Close the transaction pending ack store. + */ + CompletableFuture closeAsync(); + + /** + * Append the individual pending ack operation to the ack persistent store. + * + * @param txnID {@link TxnID} transaction id. + * @param positions {@link PositionImpl} the pending ack postion. + * @return a future represents the result of this operation + */ + CompletableFuture appendIndividualAck(TxnID txnID, List> positions); + + /** + * Append the cumulative pending ack operation to the ack persistent store. + * + * @param txnID {@link TxnID} transaction id. + * @param position {@link PositionImpl} the pending ack position. + * @return a future represents the result of this operation + */ + CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position); + + /** + * Append the pending ack commit mark to the ack persistent store. + * + * @param txnID {@link TxnID} the transaction id for add commit mark. + * @param ackType {@link AckType} the ack type of the commit. + * @return a future represents the result of this operation + */ + CompletableFuture appendCommitMark(TxnID txnID, AckType ackType); + + /** + * Append the pending ack abort mark to the ack persistent store. + * + * @param txnID {@link Position} the txnID + * @param ackType {@link AckType} the ack type of the abort. + * @return a future represents the result of this operation + */ + CompletableFuture appendAbortMark(TxnID txnID, AckType ackType); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java new file mode 100644 index 0000000000000..4104e36f75b41 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack; + +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * Provider of transaction pending ack store. + */ +public interface TransactionPendingAckStoreProvider { + + /** + * Construct a provider from the provided class. + * + * @param providerClassName {@link String} the provider class name + * @return an instance of transaction buffer provider. + */ + static TransactionPendingAckStoreProvider newProvider(String providerClassName) throws IOException { + Class providerClass; + try { + providerClass = Class.forName(providerClassName); + Object obj = providerClass.newInstance(); + checkArgument(obj instanceof TransactionPendingAckStoreProvider, + "The factory has to be an instance of " + + TransactionPendingAckStoreProvider.class.getName()); + + return (TransactionPendingAckStoreProvider) obj; + } catch (Exception e) { + throw new IOException(e); + } + } + + /** + * Open the pending ack store. + * + * @param subscription {@link PersistentSubscription} + * @param subName {@link String} + * @return a future represents the result of the operation. + * an instance of {@link PendingAckStore} is returned + * if the operation succeeds. + */ + CompletableFuture newPendingAckStore(PersistentSubscription subscription, String subName); + + /** + * Close the provider and release all the resources. + */ + void close() throws IOException; +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java new file mode 100644 index 0000000000000..0f41e2cd3c455 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.exceptions; + +import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionBufferException; + +/** + * Transaction pending ack store provider exception. + */ +public class TransactionPendingAckStoreProviderException extends TransactionBufferException { + + public TransactionPendingAckStoreProviderException(String message) { + super(message); + } + +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java new file mode 100644 index 0000000000000..39cb094d55cef --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.api.proto.PulsarApi; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; + +/** + * The In memory implementation of {@link PendingAckStore}. + */ +public class InMemoryPendingAckStore implements PendingAckStore { + + @Override + public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService scheduledExecutorService) { + pendingAckHandle.changeToReadyState(); + } + + @Override + public CompletableFuture closeAsync() { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendIndividualAck(TxnID txnID, List> positions) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendCommitMark(TxnID txnID, PulsarApi.CommandAck.AckType ackType) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendAbortMark(TxnID txnID, PulsarApi.CommandAck.AckType ackType) { + return CompletableFuture.completedFuture(null); + } + +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java new file mode 100644 index 0000000000000..ee8f5f57f2557 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; +import org.apache.pulsar.common.util.SafeCollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * MLPendingAckStore reply call back. + */ +public class MLPendingAckReplyCallBack implements PendingAckReplyCallBack { + + private final MLPendingAckStore mlPendingAckStore; + + private final PendingAckHandleImpl pendingAckHandle; + + public MLPendingAckReplyCallBack(MLPendingAckStore mlPendingAckStore, + PendingAckHandleImpl pendingAckHandle) { + this.mlPendingAckStore = mlPendingAckStore; + this.pendingAckHandle = pendingAckHandle; + } + + @Override + public void replayComplete() { + log.info("Topic name : [{}], SubName : [{}] pending ack state reply success!", + pendingAckHandle.getTopicName(), pendingAckHandle.getSubName()); + + if (pendingAckHandle.changeToReadyState()) { + log.info("Topic name : [{}], SubName : [{}] pending ack state reply success!", + pendingAckHandle.getTopicName(), pendingAckHandle.getSubName()); + mlPendingAckStore.startTimerTask(); + } else { + log.error("Topic name : [{}], SubName : [{}] pending ack state reply fail!", + pendingAckHandle.getTopicName(), pendingAckHandle.getSubName()); + } + } + + @Override + public void handleMetadataEntry(PendingAckMetadataEntry pendingAckMetadataEntry) { + TxnID txnID = new TxnID(pendingAckMetadataEntry.getTxnidMostBits(), + pendingAckMetadataEntry.getTxnidLeastBits()); + AckType ackType = pendingAckMetadataEntry.getAckType(); + switch (pendingAckMetadataEntry.getPendingAckOp()) { + case ABORT: + pendingAckHandle.handleAbort(txnID, ackType); + break; + case COMMIT: + pendingAckHandle.handleCommit(txnID, ackType, null); + break; + case ACK: + if (ackType == AckType.Cumulative) { + PendingAckMetadata pendingAckMetadata = + pendingAckMetadataEntry.getPendingAckMetadataList().get(0); + pendingAckHandle.handleCumulativeAck(txnID, + PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId())); + } else { + List> positions = new ArrayList<>(); + pendingAckMetadataEntry.getPendingAckMetadataList().forEach(pendingAckMetadata -> { + if (pendingAckMetadata.getAckSetCount() == 0) { + positions.add(new MutablePair<>(PositionImpl.get(pendingAckMetadata.getLedgerId(), + pendingAckMetadata.getEntryId()), pendingAckMetadata.getBatchSize())); + } else { + PositionImpl position = + PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId()); + position.setAckSet(SafeCollectionUtils.longListToArray(pendingAckMetadata.getAckSetList())); + positions.add(new MutablePair<>(position, pendingAckMetadata.getBatchSize())); + } + }); + pendingAckHandle.handleIndividualAck(txnID, positions); + } + break; + default: + throw new IllegalStateException("Transaction pending ack replay " + + "error with illegal state : " + pendingAckMetadataEntry.getPendingAckOp()); + + } + } + + private static final Logger log = LoggerFactory.getLogger(MLPendingAckReplyCallBack.class); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java new file mode 100644 index 0000000000000..9afca676e00e2 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -0,0 +1,336 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import io.netty.buffer.ByteBuf; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import io.netty.util.Timer; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.SafeCollectionUtils; +import org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream; +import org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream; +import org.jctools.queues.MessagePassingQueue; +import org.jctools.queues.SpscArrayQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The implement of the pending ack store by manageLedger. + */ +public class MLPendingAckStore implements PendingAckStore { + + + private final ManagedLedger managedLedger; + + private final ManagedCursor cursor; + + private static final String PENDING_ACK_STORE_SUFFIX = "-transaction-pendingack"; + + private static final String PENDING_ACK_STORE_CURSOR_NAME = "pendingack"; + + private final SpscArrayQueue entryQueue; + + //this is for replay + private final PositionImpl lastConfirmedEntry; + + private PositionImpl currentLoadPosition; + + private final Timer timer; + + private final MLPendingAckStoreTimerTask mlPendingAckStoreTimerTask; + + private final int intervalTime; + + public MLPendingAckStore(ManagedLedger managedLedger, ManagedCursor cursor, + Timer timer, ManagedCursor subManagedCursor, int maxIntervalTime, int minIntervalTime) { + this.managedLedger = managedLedger; + this.cursor = cursor; + this.currentLoadPosition = (PositionImpl) this.cursor.getMarkDeletedPosition(); + this.entryQueue = new SpscArrayQueue<>(2000); + this.lastConfirmedEntry = (PositionImpl) managedLedger.getLastConfirmedEntry(); + this.timer = timer; + this.intervalTime = minIntervalTime; + this.mlPendingAckStoreTimerTask = new MLPendingAckStoreTimerTask(cursor, managedLedger, + minIntervalTime, maxIntervalTime, subManagedCursor, this.timer); + } + + @Override + public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService transactionReplayExecutor) { + transactionReplayExecutor + .execute(new PendingAckReplay(new MLPendingAckReplyCallBack(this, pendingAckHandle))); + } + + //TODO can control the number of entry to read + private void readAsync(int numberOfEntriesToRead, + AsyncCallbacks.ReadEntriesCallback readEntriesCallback) { + cursor.asyncReadEntries(numberOfEntriesToRead, readEntriesCallback, System.nanoTime()); + } + + @Override + public CompletableFuture closeAsync() { + CompletableFuture completableFuture = new CompletableFuture<>(); + cursor.asyncClose(new AsyncCallbacks.CloseCallback() { + @Override + public void closeComplete(Object ctx) { + try { + managedLedger.close(); + } catch (Exception e) { + completableFuture.completeExceptionally(e); + } + completableFuture.complete(null); + } + + @Override + public void closeFailed(ManagedLedgerException exception, Object ctx) { + completableFuture.completeExceptionally(exception); + } + }, null); + return completableFuture; + } + + @Override + public CompletableFuture appendIndividualAck(TxnID txnID, List> positions) { + PendingAckMetadataEntry.Builder builder = PendingAckMetadataEntry.newBuilder(); + builder.setPendingAckOp(PendingAckOp.ACK); + builder.setAckType(AckType.Individual); + positions.forEach(positionIntegerMutablePair -> { + PendingAckMetadata.Builder metadataBuilder = PendingAckMetadata.newBuilder(); + PositionImpl position = positionIntegerMutablePair.getLeft(); + int batchSize = positionIntegerMutablePair.getRight(); + if (positionIntegerMutablePair.getLeft().getAckSet() != null) { + metadataBuilder.addAllAckSet(SafeCollectionUtils.longArrayToList(position.getAckSet())); + metadataBuilder.setBatchSize(batchSize); + } + metadataBuilder.setLedgerId(position.getLedgerId()); + metadataBuilder.setEntryId(position.getEntryId()); + PendingAckMetadata pendingAckMetadata = metadataBuilder.build(); + metadataBuilder.recycle(); + builder.addPendingAckMetadata(pendingAckMetadata); + }); + return appendCommon(builder, txnID); + } + + @Override + public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { + PendingAckMetadataEntry.Builder builder = PendingAckMetadataEntry.newBuilder(); + builder.setPendingAckOp(PendingAckOp.ACK); + builder.setAckType(AckType.Cumulative); + PendingAckMetadata.Builder metadataBuilder = PendingAckMetadata.newBuilder(); + if (position.getAckSet() != null) { + metadataBuilder.addAllAckSet(SafeCollectionUtils.longArrayToList(position.getAckSet())); + } + metadataBuilder.setLedgerId(position.getLedgerId()); + metadataBuilder.setEntryId(position.getEntryId()); + PendingAckMetadata pendingAckMetadata = metadataBuilder.build(); + metadataBuilder.recycle(); + builder.addPendingAckMetadata(pendingAckMetadata); + return appendCommon(builder, txnID); + } + + @Override + public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) { + PendingAckMetadataEntry.Builder builder = PendingAckMetadataEntry.newBuilder(); + builder.setPendingAckOp(PendingAckOp.COMMIT); + builder.setAckType(ackType); + return appendCommon(builder, txnID); + } + + @Override + public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) { + PendingAckMetadataEntry.Builder builder = PendingAckMetadataEntry.newBuilder(); + builder.setPendingAckOp(PendingAckOp.ABORT); + builder.setAckType(ackType); + return appendCommon(builder, txnID); + } + + private CompletableFuture appendCommon(PendingAckMetadataEntry.Builder builder, TxnID txnID) { + CompletableFuture completableFuture = new CompletableFuture<>(); + builder.setTxnidLeastBits(txnID.getLeastSigBits()); + builder.setTxnidMostBits(txnID.getMostSigBits()); + PendingAckMetadataEntry pendingAckMetadataEntry = builder.build(); + int transactionMetadataEntrySize = pendingAckMetadataEntry.getSerializedSize(); + ByteBuf buf = PulsarByteBufAllocator.DEFAULT.buffer(transactionMetadataEntrySize, transactionMetadataEntrySize); + ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(buf); + try { + pendingAckMetadataEntry.writeTo(outStream); + managedLedger.asyncAddEntry(buf, new AsyncCallbacks.AddEntryCallback() { + @Override + public void addComplete(Position position, Object ctx) { + if (log.isDebugEnabled()) { + log.debug("[{}][{}] MLPendingAckStore message append success at {}, operation : {}", + managedLedger.getName(), ctx, position, builder.getPendingAckOp()); + } + builder.recycle(); + pendingAckMetadataEntry.recycle(); + buf.release(); + completableFuture.complete(null); + } + + @Override + public void addFailed(ManagedLedgerException exception, Object ctx) { + log.error("[{}][{}] MLPendingAckStore message append fail exception : {}, operation : {}", + managedLedger.getName(), ctx, exception, builder.getPendingAckOp()); + builder.recycle(); + pendingAckMetadataEntry.recycle(); + buf.release(); + completableFuture.completeExceptionally(exception); + } + } , null); + } catch (Exception e) { + log.error("[{}] MLPendingAckStore message append fail exception : {}", + managedLedger.getName(), e); + builder.recycle(); + pendingAckMetadataEntry.recycle(); + buf.release(); + completableFuture.completeExceptionally(e); + } finally { + outStream.recycle(); + } + return completableFuture; + } + + class PendingAckReplay implements Runnable { + + private final FillEntryQueueCallback fillEntryQueueCallback; + private final PendingAckReplyCallBack pendingAckReplyCallBack; + + PendingAckReplay(PendingAckReplyCallBack pendingAckReplyCallBack) { + this.fillEntryQueueCallback = new FillEntryQueueCallback(); + this.pendingAckReplyCallBack = pendingAckReplyCallBack; + } + + @Override + public void run() { + while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0) { + fillEntryQueueCallback.fillQueue(); + Entry entry = entryQueue.poll(); + if (entry != null) { + ByteBuf buffer = entry.getDataBuffer(); + currentLoadPosition = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); + ByteBufCodedInputStream stream = ByteBufCodedInputStream.get(buffer); + PendingAckMetadataEntry.Builder pendingAckMetadataEntryBuilder = + PendingAckMetadataEntry.newBuilder(); + PendingAckMetadataEntry pendingAckMetadataEntry = null; + try { + pendingAckMetadataEntry = + pendingAckMetadataEntryBuilder.mergeFrom(stream, null).build(); + pendingAckReplyCallBack.handleMetadataEntry(pendingAckMetadataEntry); + } catch (Exception e) { + if (pendingAckMetadataEntry != null) { + log.error("TxnId : [{}:{}] MLPendingAckStore reply error!", + pendingAckMetadataEntry.getTxnidMostBits(), + pendingAckMetadataEntry.getTxnidLeastBits(), e); + } else { + log.error("MLPendingAckStore reply error!", e); + } + } + entry.release(); + if (pendingAckMetadataEntry != null) { + pendingAckMetadataEntry.recycle(); + } + pendingAckMetadataEntryBuilder.recycle(); + stream.recycle(); + } else { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + //no-op + } + } + } + pendingAckReplyCallBack.replayComplete(); + } + } + + class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallback { + + private final AtomicLong outstandingReadsRequests = new AtomicLong(0); + + public FillEntryQueueCallback() { + } + + void fillQueue() { + if (entryQueue.size() < entryQueue.capacity() && outstandingReadsRequests.get() == 0) { + if (cursor.hasMoreEntries()) { + outstandingReadsRequests.incrementAndGet(); + readAsync(100, this); + } + } + } + + @Override + public void readEntriesComplete(List entries, Object ctx) { + entryQueue.fill(new MessagePassingQueue.Supplier() { + private int i = 0; + @Override + public Entry get() { + Entry entry = entries.get(i); + i++; + return entry; + } + }, entries.size()); + + outstandingReadsRequests.decrementAndGet(); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + log.error("MLPendingAckStore stat reply fail!", exception); + outstandingReadsRequests.decrementAndGet(); + } + + } + + //this task is for delete the pending ack store + protected void startTimerTask() { + this.timer.newTimeout(this.mlPendingAckStoreTimerTask, intervalTime, TimeUnit.MILLISECONDS); + } + + public static String getTransactionPendingAckStoreSuffix(String originTopicName, String subName) { + return TopicName.get(originTopicName) + "-" + subName + PENDING_ACK_STORE_SUFFIX; + } + + public static String getTransactionPendingAckStoreCursorName() { + return PENDING_ACK_STORE_CURSOR_NAME; + } + + private static final Logger log = LoggerFactory.getLogger(MLPendingAckStore.class); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java new file mode 100644 index 0000000000000..b39909173ed66 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import io.netty.util.HashedWheelTimer; +import io.netty.util.Timer; +import io.netty.util.concurrent.DefaultThreadFactory; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; +import org.apache.pulsar.broker.transaction.pendingack.exceptions.TransactionPendingAckStoreProviderException; +import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition; +import org.apache.pulsar.common.naming.TopicName; + +/** + * Provider is for MLPendingAckStore. + */ +@Slf4j +public class MLPendingAckStoreProvider implements TransactionPendingAckStoreProvider { + + private volatile Timer timer; + + private static final long tickTimeMillis = 1L; + + @Override + public CompletableFuture newPendingAckStore(PersistentSubscription subscription, String subName) { + CompletableFuture pendingAckStoreFuture = new CompletableFuture<>(); + + if (subscription == null) { + pendingAckStoreFuture.completeExceptionally( + new TransactionPendingAckStoreProviderException("The subscription is null.")); + return pendingAckStoreFuture; + } + + PersistentTopic originPersistentTopic = (PersistentTopic) subscription.getTopic(); + String pendingAckTopicName = MLPendingAckStore + .getTransactionPendingAckStoreSuffix(originPersistentTopic.getName(), subName); + + originPersistentTopic.getBrokerService().getManagedLedgerFactory() + .asyncOpen(TopicName.get(pendingAckTopicName).getPersistenceNamingEncoding(), + new AsyncCallbacks.OpenLedgerCallback() { + @Override + public void openLedgerComplete(ManagedLedger ledger, Object ctx) { + ledger.asyncOpenCursor(MLPendingAckStore.getTransactionPendingAckStoreCursorName(), + InitialPosition.Earliest, new AsyncCallbacks.OpenCursorCallback() { + @Override + public void openCursorComplete(ManagedCursor cursor, Object ctx) { + if (timer == null) { + synchronized (MLPendingAckStoreProvider.this) { + if (timer == null) { + initialize(); + } + } + } + int maxIntervalTime = + originPersistentTopic.getBrokerService().getPulsar() + .getConfig().getMaxPendingAckTimerTaskIntervalTime(); + int minIntervalTime = + originPersistentTopic.getBrokerService().getPulsar() + .getConfig().getMaxPendingAckTimerTaskIntervalTime(); + pendingAckStoreFuture + .complete(new MLPendingAckStore(ledger, cursor, timer, + subscription.getCursor(), + maxIntervalTime, minIntervalTime)); + } + + @Override + public void openCursorFailed(ManagedLedgerException exception, Object ctx) { + log.error("Open MLPendingAckStore cursor failed.", exception); + pendingAckStoreFuture.completeExceptionally(exception); + } + }, null); + } + + @Override + public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { + log.error("Open MLPendingAckStore managedLedger failed.", exception); + pendingAckStoreFuture.completeExceptionally(exception); + } + }, null); + return pendingAckStoreFuture; + } + + public void initialize() { + this.timer = new HashedWheelTimer(new DefaultThreadFactory("pending-ack-provider-timer"), + tickTimeMillis, TimeUnit.MILLISECONDS); + } + + @Override + public void close() { + if (timer != null) { + timer.stop(); + } + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTimerTask.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTimerTask.java new file mode 100644 index 0000000000000..2a1deaf721f62 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTimerTask.java @@ -0,0 +1,200 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import io.netty.buffer.ByteBuf; +import io.netty.util.Timeout; +import io.netty.util.Timer; +import io.netty.util.TimerTask; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata; +import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry; +import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; +import org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +/** + * Pending ack timer task. + */ +@Slf4j +public class MLPendingAckStoreTimerTask implements TimerTask { + + private int intervalTime; + + private final int maxIntervalTime; + + private final int minIntervalTime; + + private final ManagedCursorImpl subManagedCursor; + + private final ManagedLedgerImpl storeManagedLedger; + + private final ManagedCursorImpl managedCursor; + + private final Timer timer; + + private volatile PositionImpl markDeletePosition; + + public MLPendingAckStoreTimerTask(ManagedCursor managedCursor, ManagedLedger storeManagedLedger, + int maxIntervalTime, int minIntervalTime, + ManagedCursor subManageCursor, Timer timer) { + this.intervalTime = minIntervalTime; + this.maxIntervalTime = maxIntervalTime; + this.minIntervalTime = minIntervalTime; + this.managedCursor = (ManagedCursorImpl) managedCursor; + this.subManagedCursor = (ManagedCursorImpl) subManageCursor; + this.storeManagedLedger = (ManagedLedgerImpl) storeManagedLedger; + this.markDeletePosition = (PositionImpl) managedCursor.getMarkDeletedPosition(); + this.timer = timer; + } + + @Override + public void run(Timeout timeout) { + if (this.managedCursor.getState().equals("Closed")) { + return; + } + try { + // when no transaction ack operation in this pending ack store, it will increase the interval time + if (markDeletePosition.compareTo((PositionImpl) storeManagedLedger.getLastConfirmedEntry()) == 0) { + int time = intervalTime + minIntervalTime; + if (time > maxIntervalTime) { + intervalTime = maxIntervalTime; + } else { + intervalTime = time; + } + managedCursor.markDelete(markDeletePosition); + timer.newTimeout(MLPendingAckStoreTimerTask.this, intervalTime, TimeUnit.SECONDS); + return; + } else { + int time = intervalTime - minIntervalTime; + if (time < minIntervalTime) { + intervalTime = minIntervalTime; + } else { + intervalTime = time; + } + } + // this while in order to find the last position witch can mark delete + while (true) { + PositionImpl nextPosition = storeManagedLedger.getNextValidPosition(markDeletePosition); + Entry entry = getEntry(nextPosition).get(); + ByteBuf buffer = entry.getDataBuffer(); + ByteBufCodedInputStream stream = ByteBufCodedInputStream.get(buffer); + PendingAckMetadataEntry.Builder pendingAckMetadataEntryBuilder = + PendingAckMetadataEntry.newBuilder(); + PendingAckMetadataEntry pendingAckMetadataEntry = null; + try { + pendingAckMetadataEntry = + pendingAckMetadataEntryBuilder.mergeFrom(stream, null).build(); + switch (pendingAckMetadataEntry.getPendingAckOp()) { + case ACK: + if (pendingAckMetadataEntry.getAckType() == AckType.Cumulative) { + PendingAckMetadata pendingAckMetadata = + pendingAckMetadataEntry.getPendingAckMetadata(0); + handleAckCommon(PositionImpl.get(pendingAckMetadata.getLedgerId(), + pendingAckMetadata.getEntryId()), nextPosition); + } else { + //this judge the pendingAckMetadataEntry is can delete + PositionImpl largestPosition = null; + List metadataList = + pendingAckMetadataEntry.getPendingAckMetadataList(); + for (int i = 0; i < metadataList.size(); i++) { + PendingAckMetadata pendingAckMetadata = metadataList.get(0); + if (largestPosition == null) { + largestPosition = PositionImpl.get(pendingAckMetadata.getLedgerId(), + pendingAckMetadata.getEntryId()); + } else { + PositionImpl comparePosition = PositionImpl + .get(pendingAckMetadata.getLedgerId(), + pendingAckMetadata.getEntryId()); + if (largestPosition.compareTo(comparePosition) < 0) { + largestPosition = comparePosition; + } + + } + } + if (largestPosition != null) { + handleAckCommon(largestPosition, nextPosition); + } + } + break; + case ABORT: + case COMMIT: + markDeletePosition = nextPosition; + break; + default: + log.error("PendingAck timer task read illegal metadata state! {}", + pendingAckMetadataEntry.getPendingAckOp()); + } + } finally { + entry.release(); + if (pendingAckMetadataEntry != null) { + pendingAckMetadataEntry.recycle(); + } + pendingAckMetadataEntryBuilder.recycle(); + stream.recycle(); + } + // when markDeletePosition is not nextPosition, before markDeletePosition can delete + if (markDeletePosition != nextPosition) { + this.managedCursor.markDelete(markDeletePosition); + break; + } + } + this.timer.newTimeout(this, intervalTime, TimeUnit.MILLISECONDS); + } catch (Exception e) { + log.error("PendingAck timer task error!", e); + if ("Cursor was already closed".equals(e.getCause().getMessage())) { + return; + } + this.timer.newTimeout(this, intervalTime, TimeUnit.MILLISECONDS); + } + } + + private CompletableFuture getEntry(PositionImpl position) { + CompletableFuture completableFuture = new CompletableFuture<>(); + this.storeManagedLedger.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + completableFuture.complete(entry); + } + + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + completableFuture.completeExceptionally(exception); + } + }, null); + return completableFuture; + } + + private void handleAckCommon(PositionImpl readPosition, PositionImpl storePosition) { + if (readPosition.compareTo((PositionImpl) subManagedCursor.getMarkDeletedPosition()) <= 0) { + markDeletePosition = storePosition; + } + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java index f315f8b5eb854..39c78148377e4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java @@ -68,6 +68,12 @@ public boolean checkIsCanDeleteConsumerPendingAck(PositionImpl position) { @Override public void clearIndividualPosition(Position position) { - //no operation + //no-op } + + @Override + public CompletableFuture close() { + return CompletableFuture.completedFuture(null); + } + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index ade375514120c..91245d8551b24 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -26,8 +26,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Semaphore; + import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; @@ -35,11 +36,15 @@ import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; +import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.transaction.pendingack.PendingAckHandle; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.common.api.proto.CommandAck.AckType; +import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.transaction.common.exception.TransactionConflictException; @@ -48,7 +53,7 @@ * The default implementation of {@link PendingAckHandle}. */ @Slf4j -public class PendingAckHandleImpl implements PendingAckHandle { +public class PendingAckHandleImpl extends PendingAckHandleState implements PendingAckHandle { /** * The map is for transaction with position witch was individual acked by this transaction. @@ -93,14 +98,42 @@ public class PendingAckHandleImpl implements PendingAckHandle { private final PersistentSubscription persistentSubscription; + private final CompletableFuture pendingAckStoreFuture; + + /** + * This is for multi thread handle transaction pending ack, + * we should think a better method to handle the multi thread problem. + */ + private final Semaphore semaphore; + public PendingAckHandleImpl(PersistentSubscription persistentSubscription) { + super(State.None); this.topicName = persistentSubscription.getTopicName(); this.subName = persistentSubscription.getName(); this.persistentSubscription = persistentSubscription; + this.semaphore = new Semaphore(1); + + TransactionPendingAckStoreProvider pendingAckStoreProvider = + ((PersistentTopic) this.persistentSubscription.getTopic()) + .getBrokerService().getPulsar().getTransactionPendingAckStoreProvider(); + this.pendingAckStoreFuture = + pendingAckStoreProvider.newPendingAckStore(persistentSubscription, subName); + + this.pendingAckStoreFuture.thenAccept(pendingAckStore -> { + if (pendingAckStore instanceof MLPendingAckStore) { + changeToInitializingState(); + pendingAckStore.replayAsync(this, + ((PersistentTopic) persistentSubscription.getTopic()).getBrokerService() + .getPulsar().getTransactionReplayExecutor()); + } + }).exceptionally(e -> { + log.error("PendingAckHandleImpl init fail! TopicName : {}, SubName: {}", topicName, subName, e); + return null; + }); } @Override - public synchronized CompletableFuture individualAcknowledgeMessage(TxnID txnID, + public CompletableFuture individualAcknowledgeMessage(TxnID txnID, List> positions) { if (txnID == null) { return FutureUtil.failedFuture(new NotAllowedException("TransactionID can not be null.")); @@ -108,6 +141,7 @@ public synchronized CompletableFuture individualAcknowledgeMessage(TxnID t if (positions == null) { return FutureUtil.failedFuture(new NotAllowedException("Positions can not be null.")); } + this.semaphore.acquireUninterruptibly(); CompletableFuture completableFuture = new CompletableFuture<>(); for (MutablePair positionIntegerMutablePair : positions) { PositionImpl position = positionIntegerMutablePair.left; @@ -115,9 +149,10 @@ public synchronized CompletableFuture individualAcknowledgeMessage(TxnID t // If try to ack message already acked by committed transaction or normal acknowledge, throw exception. if (((ManagedCursorImpl) persistentSubscription.getCursor()) .isMessageDeleted(position)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to ack message:" + position + " already acked before."; + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to ack message:" + position + " already acked before."; log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); } @@ -133,74 +168,52 @@ public synchronized CompletableFuture individualAcknowledgeMessage(TxnID t bitSetRecyclable.recycle(); if (isAckSetOverlap(ackSetOverlap, ((ManagedCursorImpl) persistentSubscription.getCursor()).getBatchPositionAckSet(position))) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to ack message:" + position + " already acked before."; + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to ack message:" + position + " already acked before."; log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); } if (this.individualAckPositions != null && individualAckPositions.containsKey(position) && isAckSetOverlap(individualAckPositions.get(position).getLeft().getAckSet(), ackSetOverlap)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to ack batch message:" + position + " in pending ack status."; + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to ack batch message:" + position + " in pending ack status."; log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); } } else { if (this.individualAckPositions != null && this.individualAckPositions.containsKey(position)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to ack message:" + position + " in pending ack status."; + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to ack message:" + position + " in pending ack status."; log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); } } } - for (int i = 0; i < positions.size(); i++) { - if (log.isDebugEnabled()) { - log.debug("[{}][{}] TxnID:[{}] Individual acks on {}", topicName, subName, txnID.toString(), positions); - } - if (individualAckOfTransaction == null) { - individualAckOfTransaction = new HashMap<>(); - } - - if (individualAckPositions == null) { - individualAckPositions = new HashMap<>(); - } - - PositionImpl position = positions.get(i).left; - - if (position.hasAckSet()) { - - HashMap pendingAckMessageForCurrentTxn = - individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); - - if (pendingAckMessageForCurrentTxn.containsKey(position)) { - andAckSet(pendingAckMessageForCurrentTxn.get(position), position); - } else { - pendingAckMessageForCurrentTxn.put(position, position); - } - - if (!individualAckPositions.containsKey(position)) { - this.individualAckPositions.put(position, positions.get(i)); - } else { - MutablePair positionPair = this.individualAckPositions.get(position); - positionPair.setRight(positions.get(i).right); - andAckSet(positionPair.getLeft(), position); - } - - } else { - HashMap pendingAckMessageForCurrentTxn = - individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); - pendingAckMessageForCurrentTxn.put(position, position); - this.individualAckPositions.putIfAbsent(position, positions.get(i)); - } - } - completableFuture.complete(null); + this.pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendIndividualAck(txnID, positions).whenComplete((v, e) -> { + if (e != null) { + handleIndividualAck(txnID, positions); + semaphore.release(); + completableFuture.completeExceptionally(e); + } else { + handleIndividualAck(txnID, positions); + semaphore.release(); + completableFuture.complete(null); + } + })).exceptionally(e -> { + semaphore.release(); + completableFuture.completeExceptionally(e); + return null; + }); return completableFuture; } @Override - public synchronized CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, + public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, List positions) { if (txnID == null) { @@ -209,139 +222,352 @@ public synchronized CompletableFuture cumulativeAcknowledgeMessage(TxnID t if (positions == null) { return FutureUtil.failedFuture(new NotAllowedException("Positions can not be null.")); } + this.semaphore.acquireUninterruptibly(); if (positions.size() != 1) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " invalid cumulative ack received with multiple message ids."; + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " invalid cumulative ack received with multiple message ids."; log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new NotAllowedException(errorMsg)); } PositionImpl position = positions.get(0); if (position.compareTo((PositionImpl) persistentSubscription.getCursor().getMarkDeletedPosition()) <= 0) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to cumulative ack position: " + position + " within range of cursor's " - + "markDeletePosition: " + persistentSubscription.getCursor().getMarkDeletedPosition(); + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to cumulative ack position: " + position + " within range of cursor's " + + "markDeletePosition: " + persistentSubscription.getCursor().getMarkDeletedPosition(); log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); } if (log.isDebugEnabled()) { log.debug("[{}][{}] TxnID:[{}] Cumulative ack on {}.", topicName, subName, txnID.toString(), position); } - - if (this.cumulativeAckOfTransaction == null) { - this.cumulativeAckOfTransaction = MutablePair.of(txnID, position); - } else if (this.cumulativeAckOfTransaction.getKey().equals(txnID) - && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) > 0) { - this.cumulativeAckOfTransaction.setValue(position); - - } else { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to cumulative batch ack position: " + position + " within range of current " - + "currentPosition: " + this.cumulativeAckOfTransaction.getValue(); + CompletableFuture completableFuture = new CompletableFuture<>(); + if (cumulativeAckOfTransaction != null && this.cumulativeAckOfTransaction.getKey().equals(txnID) + && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) <= 0) { + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to cumulative batch ack position: " + position + " within range of current " + + "currentPosition: " + this.cumulativeAckOfTransaction.getValue(); log.error(errorMsg); + semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); } - return CompletableFuture.completedFuture(null); + this.pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendCumulativeAck(txnID, position).whenComplete((v, e) -> { + if (e != null) { + handleCumulativeAck(txnID, position); + semaphore.release(); + completableFuture.completeExceptionally(e); + } else { + handleCumulativeAck(txnID, position); + semaphore.release(); + completableFuture.complete(null); + } + })).exceptionally(e -> { + semaphore.release(); + completableFuture.completeExceptionally(e); + return null; + }); + return completableFuture; } @Override - public synchronized CompletableFuture commitTxn(TxnID txnID, Map properties) { + public CompletableFuture commitTxn(TxnID txnID, Map properties) { + if (!checkIfReady()) { + return FutureUtil.failedFuture(new ServiceUnitNotReadyException("PendingAckHandle not replay complete!")); + } CompletableFuture commitFuture = new CompletableFuture<>(); + // It's valid to create transaction then commit without doing any operation, which will cause // pendingAckMessagesMap to be null. + this.semaphore.acquireUninterruptibly(); if (this.cumulativeAckOfTransaction != null) { if (cumulativeAckOfTransaction.getKey().equals(txnID)) { - persistentSubscription.acknowledgeMessage(Collections - .singletonList(this.cumulativeAckOfTransaction.getValue()), AckType.Cumulative, properties); - this.cumulativeAckOfTransaction = null; + pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendCommitMark(txnID, AckType.Cumulative).whenComplete((v, e) -> { + if (e != null) { + semaphore.release(); + commitFuture.completeExceptionally(e); + } else { + persistentSubscription.acknowledgeMessage( + Collections.singletonList(this.cumulativeAckOfTransaction.getValue()), + AckType.Cumulative, properties); + this.cumulativeAckOfTransaction = null; + semaphore.release(); + commitFuture.complete(null); + } + }) + ).exceptionally(e -> { + semaphore.release(); + commitFuture.completeExceptionally(e); + return null; + }); + } else { + semaphore.release(); + commitFuture.complete(null); } } else { if (individualAckOfTransaction != null && individualAckOfTransaction.containsKey(txnID)) { HashMap pendingAckMessageForCurrentTxn = - individualAckOfTransaction.remove(txnID); - if (pendingAckMessageForCurrentTxn != null) { - persistentSubscription.acknowledgeMessage(new ArrayList<>(pendingAckMessageForCurrentTxn.values()), - AckType.Individual, properties); - } + individualAckOfTransaction.get(txnID); + pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendCommitMark(txnID, AckType.Individual).whenComplete((v, e) -> { + if (e != null) { + semaphore.release(); + commitFuture.completeExceptionally(e); + } else { + individualAckCommitCommon(txnID, pendingAckMessageForCurrentTxn, properties); + semaphore.release(); + commitFuture.complete(null); + } + }) + ).exceptionally(e -> { + semaphore.release(); + commitFuture.completeExceptionally(e); + return null; + }); + } else { + semaphore.release(); + commitFuture.complete(null); } } - commitFuture.complete(null); return commitFuture; } @Override - public synchronized CompletableFuture abortTxn(TxnID txnId, Consumer consumer) { + public CompletableFuture abortTxn(TxnID txnId, Consumer consumer) { + if (!checkIfReady()) { + return FutureUtil.failedFuture(new ServiceUnitNotReadyException("PendingAckHandle not replay complete!")); + } CompletableFuture abortFuture = new CompletableFuture<>(); + this.semaphore.acquireUninterruptibly(); if (this.cumulativeAckOfTransaction != null) { - if (this.cumulativeAckOfTransaction.getKey().equals(txnId)) { - this.cumulativeAckOfTransaction = null; - } - this.persistentSubscription.redeliverUnacknowledgedMessages(consumer); + pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendAbortMark(txnId, AckType.Individual).whenComplete((v, e) -> { + if (e != null) { + semaphore.release(); + abortFuture.completeExceptionally(e); + } else { + if (this.cumulativeAckOfTransaction.getKey().equals(txnId)) { + this.cumulativeAckOfTransaction = null; + } + this.persistentSubscription.redeliverUnacknowledgedMessages(consumer); + semaphore.release(); + abortFuture.complete(null); + } + }) + ).exceptionally(e -> { + semaphore.release(); + abortFuture.completeExceptionally(e); + return null; + }); } else if (this.individualAckOfTransaction != null){ HashMap pendingAckMessageForCurrentTxn = - individualAckOfTransaction.remove(txnId); + individualAckOfTransaction.get(txnId); if (pendingAckMessageForCurrentTxn != null) { - for (Entry entry : pendingAckMessageForCurrentTxn.entrySet()) { - if (entry.getValue().hasAckSet() && individualAckPositions.containsKey(entry.getValue())) { - BitSetRecyclable thisBitSet = BitSetRecyclable.valueOf(entry.getValue().getAckSet()); - thisBitSet.flip(0, individualAckPositions.get(entry.getValue()).right); - BitSetRecyclable otherBitSet = - BitSetRecyclable.valueOf(individualAckPositions.get(entry.getValue()).left.getAckSet()); - otherBitSet.or(thisBitSet); - individualAckPositions.get(entry.getKey()).left.setAckSet(otherBitSet.toLongArray()); - otherBitSet.recycle(); - thisBitSet.recycle(); - } else { - individualAckPositions.remove(entry.getValue()); - } - } - this.persistentSubscription.redeliverUnacknowledgedMessages(consumer, - new ArrayList<>(pendingAckMessageForCurrentTxn.values())); + pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendAbortMark(txnId, AckType.Individual).whenComplete((v, e) -> { + if (e != null) { + semaphore.release(); + abortFuture.completeExceptionally(e); + } else { + individualAckAbortCommon(txnId, pendingAckMessageForCurrentTxn); + this.persistentSubscription.redeliverUnacknowledgedMessages(consumer, + new ArrayList<>(pendingAckMessageForCurrentTxn.values())); + semaphore.release(); + abortFuture.complete(null); + } + }) + ).exceptionally(e -> { + semaphore.release(); + abortFuture.completeExceptionally(e); + return null; + }); + } else { + semaphore.release(); + abortFuture.complete(null); } + } else { + semaphore.release(); + abortFuture.complete(null); } - abortFuture.complete(null); return abortFuture; } @Override - public synchronized void syncBatchPositionAckSetForTransaction(PositionImpl position) { - if (individualAckPositions == null) { - individualAckPositions = new HashMap<>(); - } - //sync don't carry the batch size - //when one position is ack by transaction the batch size is for `and` operation. - if (!individualAckPositions.containsKey(position)) { - this.individualAckPositions.put(position, new MutablePair<>(position, 0)); - } else { - andAckSet(this.individualAckPositions.get(position).left, position); + public void syncBatchPositionAckSetForTransaction(PositionImpl position) { + this.semaphore.acquireUninterruptibly(); + try { + if (individualAckPositions == null) { + individualAckPositions = new HashMap<>(); + } + // sync don't carry the batch size + // when one position is ack by transaction the batch size is for `and` operation. + if (!individualAckPositions.containsKey(position)) { + this.individualAckPositions.put(position, new MutablePair<>(position, 0)); + } else { + andAckSet(this.individualAckPositions.get(position).left, position); + } + } finally { + semaphore.release(); } } @Override public boolean checkIsCanDeleteConsumerPendingAck(PositionImpl position) { - if (!individualAckPositions.containsKey(position)) { - return true; + this.semaphore.acquireUninterruptibly(); + try { + if (!individualAckPositions.containsKey(position)) { + return true; + } else { + position = individualAckPositions.get(position).left; + if (position.hasAckSet()) { + BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(position.getAckSet()); + if (bitSetRecyclable.isEmpty()) { + bitSetRecyclable.recycle(); + return true; + } else { + bitSetRecyclable.recycle(); + return false; + } + } else { + return true; + } + } + } finally { + semaphore.release(); + } + } + + protected void handleAbort(TxnID txnID, AckType ackType) { + if (ackType == AckType.Cumulative) { + this.cumulativeAckOfTransaction = null; } else { - position = individualAckPositions.get(position).left; + if (this.individualAckOfTransaction != null) { + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.get(txnID); + individualAckAbortCommon(txnID, pendingAckMessageForCurrentTxn); + } + } + } + + private void individualAckAbortCommon(TxnID txnID, + HashMap pendingAckMessageForCurrentTxn) { + for (Map.Entry entry : + pendingAckMessageForCurrentTxn.entrySet()) { + if (entry.getValue().hasAckSet() && + individualAckPositions.containsKey(entry.getValue())) { + BitSetRecyclable thisBitSet = + BitSetRecyclable.valueOf(entry.getValue().getAckSet()); + thisBitSet.flip(0, individualAckPositions.get(entry.getValue()).right); + BitSetRecyclable otherBitSet = + BitSetRecyclable.valueOf(individualAckPositions + .get(entry.getValue()).left.getAckSet()); + otherBitSet.or(thisBitSet); + individualAckPositions.get(entry.getKey()) + .left.setAckSet(otherBitSet.toLongArray()); + otherBitSet.recycle(); + thisBitSet.recycle(); + } else { + individualAckPositions.remove(entry.getValue()); + } + } + individualAckOfTransaction.remove(txnID); + } + + protected void handleCommit(TxnID txnID, AckType ackType, Map properties) { + if (ackType == AckType.Cumulative) { + persistentSubscription.acknowledgeMessage( + Collections.singletonList(this.cumulativeAckOfTransaction.getValue()), + AckType.Cumulative, properties); + this.cumulativeAckOfTransaction = null; + } else { + if (this.individualAckOfTransaction != null) { + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.get(txnID); + individualAckCommitCommon(txnID, pendingAckMessageForCurrentTxn, null); + } + } + } + + private void individualAckCommitCommon(TxnID txnID, + HashMap pendingAckMessageForCurrentTxn, + Map properties) { + if (pendingAckMessageForCurrentTxn != null) { + persistentSubscription.acknowledgeMessage( + new ArrayList<>(pendingAckMessageForCurrentTxn.values()), + AckType.Individual, properties); + individualAckOfTransaction.remove(txnID); + } + } + + protected void handleIndividualAck(TxnID txnID, List> positions) { + for (int i = 0; i < positions.size(); i++) { + if (log.isDebugEnabled()) { + log.debug("[{}][{}] TxnID:[{}] Individual acks on {}", topicName, + subName, txnID.toString(), positions); + } + if (individualAckOfTransaction == null) { + individualAckOfTransaction = new HashMap<>(); + } + + if (individualAckPositions == null) { + individualAckPositions = new HashMap<>(); + } + + PositionImpl position = positions.get(i).left; + if (position.hasAckSet()) { - BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(position.getAckSet()); - if (bitSetRecyclable.isEmpty()) { - bitSetRecyclable.recycle(); - return true; + + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); + + if (pendingAckMessageForCurrentTxn.containsKey(position)) { + andAckSet(pendingAckMessageForCurrentTxn.get(position), position); + } else { + pendingAckMessageForCurrentTxn.put(position, position); + } + + if (!individualAckPositions.containsKey(position)) { + this.individualAckPositions.put(position, positions.get(i)); } else { - bitSetRecyclable.recycle(); - return false; + MutablePair positionPair = + this.individualAckPositions.get(position); + positionPair.setRight(positions.get(i).right); + andAckSet(positionPair.getLeft(), position); } + } else { - return true; + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); + pendingAckMessageForCurrentTxn.put(position, position); + this.individualAckPositions.putIfAbsent(position, positions.get(i)); } } } + protected void handleCumulativeAck(TxnID txnID, PositionImpl position) { + if (this.cumulativeAckOfTransaction == null) { + this.cumulativeAckOfTransaction = MutablePair.of(txnID, position); + } else if (this.cumulativeAckOfTransaction.getKey().equals(txnID) + && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) > 0) { + this.cumulativeAckOfTransaction.setValue(position); + } + } + + public String getTopicName() { + return topicName; + } + + public String getSubName() { + return subName; + } + @Override public void clearIndividualPosition(Position position) { if (individualAckPositions == null) { @@ -353,4 +579,8 @@ public void clearIndividualPosition(Position position) { } } -} + @Override + public CompletableFuture close() { + return this.pendingAckStoreFuture.thenAccept(PendingAckStore::closeAsync); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java new file mode 100644 index 0000000000000..352962c33fa84 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; + +/** + * The implement of transaction pending ack store. + */ +public abstract class PendingAckHandleState { + + /** + * The state of the pending ack handle {@link PendingAckHandleState}. + */ + public enum State { + None, + Initializing, + Ready, + Close + } + + private static final AtomicReferenceFieldUpdater STATE_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(PendingAckHandleState.class, State.class, "state"); + + @SuppressWarnings("unused") + private volatile State state = null; + + public PendingAckHandleState(State state) { + STATE_UPDATER.set(this, state); + + } + + protected boolean changeToReadyState() { + return (STATE_UPDATER.compareAndSet(this, State.Initializing, State.Ready)); + } + + protected boolean changeToInitializingState() { + return STATE_UPDATER.compareAndSet(this, State.None, State.Initializing); + } + + protected boolean changeToCloseState() { + return (STATE_UPDATER.compareAndSet(this, State.Ready, State.Close) + || STATE_UPDATER.compareAndSet(this, State.None, State.Close) + || STATE_UPDATER.compareAndSet(this, State.Initializing, State.Close)); + } + + public boolean checkIfReady() { + return STATE_UPDATER.get(this) == State.Ready; + } + + public State getState() { + return STATE_UPDATER.get(this); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java new file mode 100644 index 0000000000000..e2910bbaf08f8 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java @@ -0,0 +1,1254 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: pulsar-broker/src/main/proto/TransactionPendingAck.proto + +package org.apache.pulsar.broker.transaction.proto; + +public final class TransactionPendingAck { + private TransactionPendingAck() {} + public static void registerAllExtensions( + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite registry) { + } + public enum PendingAckOp + implements org.apache.pulsar.shaded.com.google.protobuf.v241.Internal.EnumLite { + ACK(0, 0), + COMMIT(1, 1), + ABORT(2, 2), + ; + + public static final int ACK_VALUE = 0; + public static final int COMMIT_VALUE = 1; + public static final int ABORT_VALUE = 2; + + + public final int getNumber() { return value; } + + public static PendingAckOp valueOf(int value) { + switch (value) { + case 0: return ACK; + case 1: return COMMIT; + case 2: return ABORT; + default: return null; + } + } + + public static org.apache.pulsar.shaded.com.google.protobuf.v241.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static org.apache.pulsar.shaded.com.google.protobuf.v241.Internal.EnumLiteMap + internalValueMap = + new org.apache.pulsar.shaded.com.google.protobuf.v241.Internal.EnumLiteMap() { + public PendingAckOp findValueByNumber(int number) { + return PendingAckOp.valueOf(number); + } + }; + + private final int value; + + private PendingAckOp(int index, int value) { + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:pulsar.transaction.PendingAckOp) + } + + public interface PendingAckMetadataOrBuilder + extends org.apache.pulsar.shaded.com.google.protobuf.v241.MessageLiteOrBuilder { + + // optional int64 ledger_id = 1; + boolean hasLedgerId(); + long getLedgerId(); + + // optional int64 entry_id = 2; + boolean hasEntryId(); + long getEntryId(); + + // repeated int64 ack_set = 3; + java.util.List getAckSetList(); + int getAckSetCount(); + long getAckSet(int index); + + // optional int32 batch_size = 4 [default = 0]; + boolean hasBatchSize(); + int getBatchSize(); + } + public static final class PendingAckMetadata extends + org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite + implements PendingAckMetadataOrBuilder, org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream.ByteBufGeneratedMessage { + // Use PendingAckMetadata.newBuilder() to construct. + private io.netty.util.Recycler.Handle handle; + private PendingAckMetadata(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + } + + private static final io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected PendingAckMetadata newObject(Handle handle) { + return new PendingAckMetadata(handle); + } + }; + + public void recycle() { + this.initFields(); + this.memoizedIsInitialized = -1; + this.bitField0_ = 0; + this.memoizedSerializedSize = -1; + if (handle != null) { RECYCLER.recycle(this, handle); } + } + + private PendingAckMetadata(boolean noInit) {} + + private static final PendingAckMetadata defaultInstance; + public static PendingAckMetadata getDefaultInstance() { + return defaultInstance; + } + + public PendingAckMetadata getDefaultInstanceForType() { + return defaultInstance; + } + + private int bitField0_; + // optional int64 ledger_id = 1; + public static final int LEDGER_ID_FIELD_NUMBER = 1; + private long ledgerId_; + public boolean hasLedgerId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getLedgerId() { + return ledgerId_; + } + + // optional int64 entry_id = 2; + public static final int ENTRY_ID_FIELD_NUMBER = 2; + private long entryId_; + public boolean hasEntryId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getEntryId() { + return entryId_; + } + + // repeated int64 ack_set = 3; + public static final int ACK_SET_FIELD_NUMBER = 3; + private java.util.List ackSet_; + public java.util.List + getAckSetList() { + return ackSet_; + } + public int getAckSetCount() { + return ackSet_.size(); + } + public long getAckSet(int index) { + return ackSet_.get(index); + } + + // optional int32 batch_size = 4 [default = 0]; + public static final int BATCH_SIZE_FIELD_NUMBER = 4; + private int batchSize_; + public boolean hasBatchSize() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public int getBatchSize() { + return batchSize_; + } + + private void initFields() { + ledgerId_ = 0L; + entryId_ = 0L; + ackSet_ = java.util.Collections.emptyList();; + batchSize_ = 0; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream output) + throws java.io.IOException { + throw new RuntimeException("Cannot use CodedOutputStream"); + } + + public void writeTo(org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, ledgerId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeInt64(2, entryId_); + } + for (int i = 0; i < ackSet_.size(); i++) { + output.writeInt64(3, ackSet_.get(i)); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt32(4, batchSize_); + } + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeInt64Size(1, ledgerId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeInt64Size(2, entryId_); + } + { + int dataSize = 0; + for (int i = 0; i < ackSet_.size(); i++) { + dataSize += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeInt64SizeNoTag(ackSet_.get(i)); + } + size += dataSize; + size += 1 * getAckSetList().size(); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeInt32Size(4, batchSize_); + } + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString data) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString data, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom(byte[] data) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( + byte[] data, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( + java.io.InputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseDelimitedFrom( + java.io.InputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + public static final class Builder extends + org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite.Builder< + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata, Builder> + implements org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataOrBuilder, org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream.ByteBufMessageBuilder { + // Construct using org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.newBuilder() + private final io.netty.util.Recycler.Handle handle; + private Builder(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + maybeForceBuilderInitialization(); + } + private final static io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected Builder newObject(io.netty.util.Recycler.Handle handle) { + return new Builder(handle); + } + }; + + public void recycle() { + clear(); + if (handle != null) {RECYCLER.recycle(this, handle);} + } + + private void maybeForceBuilderInitialization() { + } + private static Builder create() { + return RECYCLER.get(); + } + + public Builder clear() { + super.clear(); + ledgerId_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + entryId_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + ackSet_ = java.util.Collections.emptyList();; + bitField0_ = (bitField0_ & ~0x00000004); + batchSize_ = 0; + bitField0_ = (bitField0_ & ~0x00000008); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata getDefaultInstanceForType() { + return org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.getDefaultInstance(); + } + + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata build() { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata buildParsed() + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata buildPartial() { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata result = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.RECYCLER.get(); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.ledgerId_ = ledgerId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.entryId_ = entryId_; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + ackSet_ = java.util.Collections.unmodifiableList(ackSet_); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.ackSet_ = ackSet_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000004; + } + result.batchSize_ = batchSize_; + result.bitField0_ = to_bitField0_; + return result; + } + + public Builder mergeFrom(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata other) { + if (other == org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.getDefaultInstance()) return this; + if (other.hasLedgerId()) { + setLedgerId(other.getLedgerId()); + } + if (other.hasEntryId()) { + setEntryId(other.getEntryId()); + } + if (!other.ackSet_.isEmpty()) { + if (ackSet_.isEmpty()) { + ackSet_ = other.ackSet_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensureAckSetIsMutable(); + ackSet_.addAll(other.ackSet_); + } + + } + if (other.hasBatchSize()) { + setBatchSize(other.getBatchSize()); + } + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom(org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + throw new java.io.IOException("Merge from CodedInputStream is disabled"); + } + public Builder mergeFrom( + org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + + return this; + default: { + if (!input.skipField(tag)) { + + return this; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + ledgerId_ = input.readInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + entryId_ = input.readInt64(); + break; + } + case 24: { + ensureAckSetIsMutable(); + ackSet_.add(input.readInt64()); + break; + } + case 26: { + int length = input.readRawVarint32(); + int limit = input.pushLimit(length); + while (input.getBytesUntilLimit() > 0) { + addAckSet(input.readInt64()); + } + input.popLimit(limit); + break; + } + case 32: { + bitField0_ |= 0x00000008; + batchSize_ = input.readInt32(); + break; + } + } + } + } + + private int bitField0_; + + // optional int64 ledger_id = 1; + private long ledgerId_ ; + public boolean hasLedgerId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getLedgerId() { + return ledgerId_; + } + public Builder setLedgerId(long value) { + bitField0_ |= 0x00000001; + ledgerId_ = value; + + return this; + } + public Builder clearLedgerId() { + bitField0_ = (bitField0_ & ~0x00000001); + ledgerId_ = 0L; + + return this; + } + + // optional int64 entry_id = 2; + private long entryId_ ; + public boolean hasEntryId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getEntryId() { + return entryId_; + } + public Builder setEntryId(long value) { + bitField0_ |= 0x00000002; + entryId_ = value; + + return this; + } + public Builder clearEntryId() { + bitField0_ = (bitField0_ & ~0x00000002); + entryId_ = 0L; + + return this; + } + + // repeated int64 ack_set = 3; + private java.util.List ackSet_ = java.util.Collections.emptyList();; + private void ensureAckSetIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + ackSet_ = new java.util.ArrayList(ackSet_); + bitField0_ |= 0x00000004; + } + } + public java.util.List + getAckSetList() { + return java.util.Collections.unmodifiableList(ackSet_); + } + public int getAckSetCount() { + return ackSet_.size(); + } + public long getAckSet(int index) { + return ackSet_.get(index); + } + public Builder setAckSet( + int index, long value) { + ensureAckSetIsMutable(); + ackSet_.set(index, value); + + return this; + } + public Builder addAckSet(long value) { + ensureAckSetIsMutable(); + ackSet_.add(value); + + return this; + } + public Builder addAllAckSet( + java.lang.Iterable values) { + ensureAckSetIsMutable(); + super.addAll(values, ackSet_); + + return this; + } + public Builder clearAckSet() { + ackSet_ = java.util.Collections.emptyList();; + bitField0_ = (bitField0_ & ~0x00000004); + + return this; + } + + // optional int32 batch_size = 4 [default = 0]; + private int batchSize_ ; + public boolean hasBatchSize() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public int getBatchSize() { + return batchSize_; + } + public Builder setBatchSize(int value) { + bitField0_ |= 0x00000008; + batchSize_ = value; + + return this; + } + public Builder clearBatchSize() { + bitField0_ = (bitField0_ & ~0x00000008); + batchSize_ = 0; + + return this; + } + + // @@protoc_insertion_point(builder_scope:pulsar.transaction.PendingAckMetadata) + } + + static { + defaultInstance = new PendingAckMetadata(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:pulsar.transaction.PendingAckMetadata) + } + + public interface PendingAckMetadataEntryOrBuilder + extends org.apache.pulsar.shaded.com.google.protobuf.v241.MessageLiteOrBuilder { + + // optional uint64 txnid_least_bits = 1; + boolean hasTxnidLeastBits(); + long getTxnidLeastBits(); + + // optional uint64 txnid_most_bits = 2; + boolean hasTxnidMostBits(); + long getTxnidMostBits(); + + // optional .pulsar.transaction.PendingAckOp pending_ack_op = 3; + boolean hasPendingAckOp(); + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp getPendingAckOp(); + + // optional .pulsar.proto.CommandAck.AckType ack_type = 4; + boolean hasAckType(); + org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType getAckType(); + + // repeated .pulsar.transaction.PendingAckMetadata pendingAckMetadata = 5; + java.util.List + getPendingAckMetadataList(); + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata getPendingAckMetadata(int index); + int getPendingAckMetadataCount(); + } + public static final class PendingAckMetadataEntry extends + org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite + implements PendingAckMetadataEntryOrBuilder, org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream.ByteBufGeneratedMessage { + // Use PendingAckMetadataEntry.newBuilder() to construct. + private io.netty.util.Recycler.Handle handle; + private PendingAckMetadataEntry(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + } + + private static final io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected PendingAckMetadataEntry newObject(Handle handle) { + return new PendingAckMetadataEntry(handle); + } + }; + + public void recycle() { + this.initFields(); + this.memoizedIsInitialized = -1; + this.bitField0_ = 0; + this.memoizedSerializedSize = -1; + if (handle != null) { RECYCLER.recycle(this, handle); } + } + + private PendingAckMetadataEntry(boolean noInit) {} + + private static final PendingAckMetadataEntry defaultInstance; + public static PendingAckMetadataEntry getDefaultInstance() { + return defaultInstance; + } + + public PendingAckMetadataEntry getDefaultInstanceForType() { + return defaultInstance; + } + + private int bitField0_; + // optional uint64 txnid_least_bits = 1; + public static final int TXNID_LEAST_BITS_FIELD_NUMBER = 1; + private long txnidLeastBits_; + public boolean hasTxnidLeastBits() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getTxnidLeastBits() { + return txnidLeastBits_; + } + + // optional uint64 txnid_most_bits = 2; + public static final int TXNID_MOST_BITS_FIELD_NUMBER = 2; + private long txnidMostBits_; + public boolean hasTxnidMostBits() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getTxnidMostBits() { + return txnidMostBits_; + } + + // optional .pulsar.transaction.PendingAckOp pending_ack_op = 3; + public static final int PENDING_ACK_OP_FIELD_NUMBER = 3; + private org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp pendingAckOp_; + public boolean hasPendingAckOp() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp getPendingAckOp() { + return pendingAckOp_; + } + + // optional .pulsar.proto.CommandAck.AckType ack_type = 4; + public static final int ACK_TYPE_FIELD_NUMBER = 4; + private org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType ackType_; + public boolean hasAckType() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType getAckType() { + return ackType_; + } + + // repeated .pulsar.transaction.PendingAckMetadata pendingAckMetadata = 5; + public static final int PENDINGACKMETADATA_FIELD_NUMBER = 5; + private java.util.List pendingAckMetadata_; + public java.util.List getPendingAckMetadataList() { + return pendingAckMetadata_; + } + public java.util.List + getPendingAckMetadataOrBuilderList() { + return pendingAckMetadata_; + } + public int getPendingAckMetadataCount() { + return pendingAckMetadata_.size(); + } + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata getPendingAckMetadata(int index) { + return pendingAckMetadata_.get(index); + } + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataOrBuilder getPendingAckMetadataOrBuilder( + int index) { + return pendingAckMetadata_.get(index); + } + + private void initFields() { + txnidLeastBits_ = 0L; + txnidMostBits_ = 0L; + pendingAckOp_ = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.ACK; + ackType_ = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.Individual; + pendingAckMetadata_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream output) + throws java.io.IOException { + throw new RuntimeException("Cannot use CodedOutputStream"); + } + + public void writeTo(org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeUInt64(1, txnidLeastBits_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeUInt64(2, txnidMostBits_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeEnum(3, pendingAckOp_.getNumber()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeEnum(4, ackType_.getNumber()); + } + for (int i = 0; i < pendingAckMetadata_.size(); i++) { + output.writeMessage(5, pendingAckMetadata_.get(i)); + } + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeUInt64Size(1, txnidLeastBits_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeUInt64Size(2, txnidMostBits_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeEnumSize(3, pendingAckOp_.getNumber()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeEnumSize(4, ackType_.getNumber()); + } + for (int i = 0; i < pendingAckMetadata_.size(); i++) { + size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream + .computeMessageSize(5, pendingAckMetadata_.get(i)); + } + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString data) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString data, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + throw new RuntimeException("Disabled"); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom(byte[] data) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data).buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( + byte[] data, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + return newBuilder().mergeFrom(data, extensionRegistry) + .buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom(java.io.InputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( + java.io.InputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseDelimitedFrom( + java.io.InputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input, extensionRegistry)) { + return builder.buildParsed(); + } else { + return null; + } + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input) + throws java.io.IOException { + return newBuilder().mergeFrom(input).buildParsed(); + } + public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( + org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return newBuilder().mergeFrom(input, extensionRegistry) + .buildParsed(); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + public static final class Builder extends + org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite.Builder< + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry, Builder> + implements org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntryOrBuilder, org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream.ByteBufMessageBuilder { + // Construct using org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry.newBuilder() + private final io.netty.util.Recycler.Handle handle; + private Builder(io.netty.util.Recycler.Handle handle) { + this.handle = handle; + maybeForceBuilderInitialization(); + } + private final static io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { + protected Builder newObject(io.netty.util.Recycler.Handle handle) { + return new Builder(handle); + } + }; + + public void recycle() { + clear(); + if (handle != null) {RECYCLER.recycle(this, handle);} + } + + private void maybeForceBuilderInitialization() { + } + private static Builder create() { + return RECYCLER.get(); + } + + public Builder clear() { + super.clear(); + txnidLeastBits_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + txnidMostBits_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + pendingAckOp_ = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.ACK; + bitField0_ = (bitField0_ & ~0x00000004); + ackType_ = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.Individual; + bitField0_ = (bitField0_ & ~0x00000008); + pendingAckMetadata_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000010); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry getDefaultInstanceForType() { + return org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry.getDefaultInstance(); + } + + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry build() { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + private org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry buildParsed() + throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException( + result).asInvalidProtocolBufferException(); + } + return result; + } + + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry buildPartial() { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry result = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry.RECYCLER.get(); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.txnidLeastBits_ = txnidLeastBits_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.txnidMostBits_ = txnidMostBits_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.pendingAckOp_ = pendingAckOp_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.ackType_ = ackType_; + if (((bitField0_ & 0x00000010) == 0x00000010)) { + pendingAckMetadata_ = java.util.Collections.unmodifiableList(pendingAckMetadata_); + bitField0_ = (bitField0_ & ~0x00000010); + } + result.pendingAckMetadata_ = pendingAckMetadata_; + result.bitField0_ = to_bitField0_; + return result; + } + + public Builder mergeFrom(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry other) { + if (other == org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry.getDefaultInstance()) return this; + if (other.hasTxnidLeastBits()) { + setTxnidLeastBits(other.getTxnidLeastBits()); + } + if (other.hasTxnidMostBits()) { + setTxnidMostBits(other.getTxnidMostBits()); + } + if (other.hasPendingAckOp()) { + setPendingAckOp(other.getPendingAckOp()); + } + if (other.hasAckType()) { + setAckType(other.getAckType()); + } + if (!other.pendingAckMetadata_.isEmpty()) { + if (pendingAckMetadata_.isEmpty()) { + pendingAckMetadata_ = other.pendingAckMetadata_; + bitField0_ = (bitField0_ & ~0x00000010); + } else { + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.addAll(other.pendingAckMetadata_); + } + + } + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom(org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + throw new java.io.IOException("Merge from CodedInputStream is disabled"); + } + public Builder mergeFrom( + org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream input, + org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + while (true) { + int tag = input.readTag(); + switch (tag) { + case 0: + + return this; + default: { + if (!input.skipField(tag)) { + + return this; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + txnidLeastBits_ = input.readUInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + txnidMostBits_ = input.readUInt64(); + break; + } + case 24: { + int rawValue = input.readEnum(); + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp value = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.valueOf(rawValue); + if (value != null) { + bitField0_ |= 0x00000004; + pendingAckOp_ = value; + } + break; + } + case 32: { + int rawValue = input.readEnum(); + org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType value = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.valueOf(rawValue); + if (value != null) { + bitField0_ |= 0x00000008; + ackType_ = value; + } + break; + } + case 42: { + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.Builder subBuilder = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.newBuilder(); + input.readMessage(subBuilder, extensionRegistry); + addPendingAckMetadata(subBuilder.buildPartial()); + break; + } + } + } + } + + private int bitField0_; + + // optional uint64 txnid_least_bits = 1; + private long txnidLeastBits_ ; + public boolean hasTxnidLeastBits() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + public long getTxnidLeastBits() { + return txnidLeastBits_; + } + public Builder setTxnidLeastBits(long value) { + bitField0_ |= 0x00000001; + txnidLeastBits_ = value; + + return this; + } + public Builder clearTxnidLeastBits() { + bitField0_ = (bitField0_ & ~0x00000001); + txnidLeastBits_ = 0L; + + return this; + } + + // optional uint64 txnid_most_bits = 2; + private long txnidMostBits_ ; + public boolean hasTxnidMostBits() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + public long getTxnidMostBits() { + return txnidMostBits_; + } + public Builder setTxnidMostBits(long value) { + bitField0_ |= 0x00000002; + txnidMostBits_ = value; + + return this; + } + public Builder clearTxnidMostBits() { + bitField0_ = (bitField0_ & ~0x00000002); + txnidMostBits_ = 0L; + + return this; + } + + // optional .pulsar.transaction.PendingAckOp pending_ack_op = 3; + private org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp pendingAckOp_ = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.ACK; + public boolean hasPendingAckOp() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp getPendingAckOp() { + return pendingAckOp_; + } + public Builder setPendingAckOp(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + pendingAckOp_ = value; + + return this; + } + public Builder clearPendingAckOp() { + bitField0_ = (bitField0_ & ~0x00000004); + pendingAckOp_ = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.ACK; + + return this; + } + + // optional .pulsar.proto.CommandAck.AckType ack_type = 4; + private org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType ackType_ = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.Individual; + public boolean hasAckType() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + public org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType getAckType() { + return ackType_; + } + public Builder setAckType(org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000008; + ackType_ = value; + + return this; + } + public Builder clearAckType() { + bitField0_ = (bitField0_ & ~0x00000008); + ackType_ = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.Individual; + + return this; + } + + // repeated .pulsar.transaction.PendingAckMetadata pendingAckMetadata = 5; + private java.util.List pendingAckMetadata_ = + java.util.Collections.emptyList(); + private void ensurePendingAckMetadataIsMutable() { + if (!((bitField0_ & 0x00000010) == 0x00000010)) { + pendingAckMetadata_ = new java.util.ArrayList(pendingAckMetadata_); + bitField0_ |= 0x00000010; + } + } + + public java.util.List getPendingAckMetadataList() { + return java.util.Collections.unmodifiableList(pendingAckMetadata_); + } + public int getPendingAckMetadataCount() { + return pendingAckMetadata_.size(); + } + public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata getPendingAckMetadata(int index) { + return pendingAckMetadata_.get(index); + } + public Builder setPendingAckMetadata( + int index, org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.set(index, value); + + return this; + } + public Builder setPendingAckMetadata( + int index, org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.Builder builderForValue) { + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.set(index, builderForValue.build()); + + return this; + } + public Builder addPendingAckMetadata(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.add(value); + + return this; + } + public Builder addPendingAckMetadata( + int index, org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.add(index, value); + + return this; + } + public Builder addPendingAckMetadata( + org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.Builder builderForValue) { + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.add(builderForValue.build()); + + return this; + } + public Builder addPendingAckMetadata( + int index, org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.Builder builderForValue) { + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.add(index, builderForValue.build()); + + return this; + } + public Builder addAllPendingAckMetadata( + java.lang.Iterable values) { + ensurePendingAckMetadataIsMutable(); + super.addAll(values, pendingAckMetadata_); + + return this; + } + public Builder clearPendingAckMetadata() { + pendingAckMetadata_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000010); + + return this; + } + public Builder removePendingAckMetadata(int index) { + ensurePendingAckMetadataIsMutable(); + pendingAckMetadata_.remove(index); + + return this; + } + + // @@protoc_insertion_point(builder_scope:pulsar.transaction.PendingAckMetadataEntry) + } + + static { + defaultInstance = new PendingAckMetadataEntry(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:pulsar.transaction.PendingAckMetadataEntry) + } + + + static { + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java index 285d32741a642..cd6bda8d1ee9b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java @@ -32,6 +32,7 @@ import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.SubscriptionType; @@ -43,6 +44,7 @@ import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -188,8 +190,6 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { .subscriptionName(subscriptionName) .enableBatchIndexAcknowledgment(true) .subscriptionType(SubscriptionType.Shared) - .ackTimeout(2, TimeUnit.SECONDS) - .acknowledgmentGroupTime(0, TimeUnit.MICROSECONDS) .subscribe(); @Cleanup @@ -207,6 +207,7 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { ConcurrentSkipListMap batchDeletedIndexes = null; + MessageId[] messageIds = new MessageId[2]; for (int retryCnt = 0; retryCnt < 2; retryCnt++) { int messageCnt = 1000; @@ -220,15 +221,20 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { Transaction commitTxn = getTxn(); //send 1000 and ack 999, and test the consumer pending ack has already clear 999 messages - for (int i = 0; i < messageCnt - 1; i++) { + for (int i = 0; i < messageCnt; i++) { message = consumer.receive(2, TimeUnit.SECONDS); Assert.assertNotNull(message); - if (i % 2 == 0) { - consumer.acknowledgeAsync(message.getMessageId(), commitTxn).get(); - log.info("txn receive msgId: {}, count: {}", message.getMessageId(), i); + // in order to free up 2 position to judge the consumer pending ack delete + if (i != 500) { + if (i % 2 == 0) { + consumer.acknowledgeAsync(message.getMessageId(), commitTxn).get(); + log.info("txn receive msgId: {}, count: {}", message.getMessageId(), i); + } else { + consumer.acknowledge(message.getMessageId()); + log.info("normal receive msgId: {}, count: {}", message.getMessageId(), i); + } } else { - consumer.acknowledge(message.getMessageId()); - log.info("normal receive msgId: {}, count: {}", message.getMessageId(), i); + messageIds[retryCnt] = message.getMessageId(); } } @@ -244,28 +250,53 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { if (completableFuture != null) { Optional topic = completableFuture.get(); if (topic.isPresent()) { - persistentSubscription = (PersistentSubscription) topic.get().getSubscription(subscriptionName); + final PersistentSubscription testPersistentSubscription = + (PersistentSubscription) topic.get().getSubscription(subscriptionName); field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); field.setAccessible(true); - pendingAckHandle = (PendingAckHandleImpl) field.get(persistentSubscription); + pendingAckHandle = (PendingAckHandleImpl) field.get(testPersistentSubscription); field = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction"); field.setAccessible(true); individualAckOfTransaction = (HashMap>) field.get(pendingAckHandle); assertTrue(individualAckOfTransaction.isEmpty()); - managedCursor = (ManagedCursorImpl) persistentSubscription.getCursor(); + managedCursor = (ManagedCursorImpl) testPersistentSubscription.getCursor(); field = ManagedCursorImpl.class.getDeclaredField("batchDeletedIndexes"); field.setAccessible(true); batchDeletedIndexes = (ConcurrentSkipListMap) field.get(managedCursor); if (retryCnt == 0) { //one message are not ack - assertEquals(batchDeletedIndexes.size(), 1); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1); + Awaitility.await().atMost(2000, TimeUnit.MILLISECONDS).until(() -> { + return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 1; + }); } else { //two message are not ack + Awaitility.await().atMost(2000, TimeUnit.MILLISECONDS).until(() -> { + System.out.println(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size()); + return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 2; + }); + + Transaction commitTwice = getTxn(); + + consumer.acknowledge(messageIds[0]); + assertEquals(batchDeletedIndexes.size(), 2); + Awaitility.await().atMost(2000, TimeUnit.MILLISECONDS).until(() -> { + return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 1; + }); + + // this test is for the last message has been cleared in this consumer pending acks + // and it won't clear the last message in cursor batch index ack set + consumer.acknowledgeAsync(messageIds[1], commitTwice).get(); assertEquals(batchDeletedIndexes.size(), 1); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1); + Awaitility.await().atMost(2000, TimeUnit.MILLISECONDS).until(() -> { + return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 0; + }); + + // the messages has been produced were all acked, the memory in broker for the messages has been cleared. + commitTwice.commit().get(); + assertEquals(batchDeletedIndexes.size(), 0); + assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0); } count++; } @@ -273,26 +304,6 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { } assertEquals(count, 1); } - - Transaction commitTwice = getTxn(); - - //this message is in one batch point - Message message = consumer.receive(); - consumer.acknowledge(message.getMessageId()); - assertEquals(batchDeletedIndexes.size(), 1); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1); - - // this test is for the last message has been cleared in this consumer pending acks - // and it won't clear the last message in cursor batch index ack set - message = consumer.receive(); - consumer.acknowledgeAsync(message.getMessageId(), commitTwice).get(); - assertEquals(batchDeletedIndexes.size(), 1); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0); - - // the messages has been produced were all acked, the memory in broker for the messages has been cleared. - commitTwice.commit().get(); - assertEquals(batchDeletedIndexes.size(), 0); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0); } private Transaction getTxn() throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java new file mode 100644 index 0000000000000..c568f82c59426 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack; + +import com.google.common.collect.Sets; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; +import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +/** + * Test for consuming transaction messages. + */ +@Slf4j +public class PendingAckPersistentTest extends TransactionTestBase { + + private final static String PENDING_ACK_REPLAY_TOPIC = "persistent://public/txn/pending-ack-replay"; + + @BeforeMethod + public void setup() throws Exception { + setBrokerCount(1); + super.internalSetup(); + + String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); + String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1]; + admin.clusters().createCluster(CLUSTER_NAME, new ClusterData("http://localhost:" + webServicePort)); + admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), + new TenantInfo(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); + admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), 16); + admin.tenants().createTenant("public", + new TenantInfo(Sets.newHashSet(), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace("public/txn", 10); + admin.topics().createNonPartitionedTopic(PENDING_ACK_REPLAY_TOPIC); + + pulsarClient = PulsarClient.builder() + .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) + .statsInterval(0, TimeUnit.SECONDS) + .enableTransaction(true) + .build(); + + Thread.sleep(1000 * 3); + } + + @AfterMethod(alwaysRun = true) + protected void cleanup() { + super.internalCleanup(); + } + + @Test + public void pendingAckSharedReplayTest() throws Exception { + int messageCount = 1000; + String subName = "shared-test"; + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(PENDING_ACK_REPLAY_TOPIC) + .enableBatching(true) + .batchingMaxMessages(200) + .create(); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(PENDING_ACK_REPLAY_TOPIC) + .subscriptionName("shared-test") + .subscriptionType(SubscriptionType.Shared) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + Transaction abortTxn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + + List pendingAckMessageIds = new ArrayList<>(); + List normalAckMessageIds = new ArrayList<>(); + for (int i = 0; i < messageCount; i++) { + producer.send("Hello Pulsar!".getBytes()); + Message message = consumer.receive(); + if (i % 2 == 0) { + consumer.acknowledgeAsync(message.getMessageId(), abortTxn).get(); + pendingAckMessageIds.add(message.getMessageId()); + } else { + normalAckMessageIds.add(message.getMessageId()); + } + } + + //in order to test pending ack replay + admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); + Transaction commitTxn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + + // this messageIds are ack by transaction + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + try { + consumer.acknowledgeAsync(pendingAckMessageIds.get(i), txn).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + } + // this messageIds are not ack by transaction + for (int i = 0; i < normalAckMessageIds.size(); i++) { + consumer.acknowledgeAsync(normalAckMessageIds.get(i), commitTxn).get(); + } + + txn.abort().get(); + // commit this txn , normalAckMessageIds are in pending ack state + commitTxn.commit().get(); + // abort this txn, pendingAckMessageIds are delete from pending ack state + abortTxn.abort().get(); + + // replay this pending ack + admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); + Awaitility.await().atMost(4000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + + abortTxn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + + commitTxn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + // normalAckMessageIds are ack and then commit, so ack fail + for (int i = 0; i < normalAckMessageIds.size(); i++) { + try { + consumer.acknowledgeAsync(normalAckMessageIds.get(i), abortTxn).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + } + + // pendingAckMessageIds are all abort, so can ack again + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + consumer.acknowledgeAsync(pendingAckMessageIds.get(i), commitTxn).get(); + } + + abortTxn.abort().get(); + commitTxn.commit().get(); + + PersistentTopic topic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() + .getTopic(TopicName.get(PENDING_ACK_REPLAY_TOPIC).toString(), false).get().get(); + Field field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); + field.setAccessible(true); + PendingAckHandleImpl pendingAckHandle = + (PendingAckHandleImpl) field.get(topic.getSubscription(subName)); + field = PendingAckHandleImpl.class.getDeclaredField("pendingAckStoreFuture"); + field.setAccessible(true); + CompletableFuture pendingAckStoreCompletableFuture = + (CompletableFuture) field.get(pendingAckHandle); + pendingAckStoreCompletableFuture.get(); + + field = MLPendingAckStore.class.getDeclaredField("cursor"); + field.setAccessible(true); + + ManagedCursor managedCursor = (ManagedCursor) field.get(pendingAckStoreCompletableFuture.get()); + + // in order to check out the pending ack cursor is clear whether or not. + Awaitility.await() + .atMost(15000, TimeUnit.MILLISECONDS) + .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition()) + .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == 0); + } +} From 45588c6d5c755db7995ab6d43e8289aee99ca074 Mon Sep 17 00:00:00 2001 From: congbo Date: Wed, 9 Dec 2020 23:45:19 +0800 Subject: [PATCH 04/18] Fix some named --- .../pulsar/broker/ServiceConfiguration.java | 10 +- .../pendingack/PendingAckStore.java | 7 +- .../TransactionPendingAckStoreProvider.java | 3 +- .../impl/InMemoryPendingAckStore.java | 2 +- .../impl/MLPendingAckStoreProvider.java | 4 +- .../pendingack/impl/PendingAckHandleImpl.java | 12 ++- .../pendingack/PendingAckPersistentTest.java | 99 ++++++++++++++++++- 7 files changed, 119 insertions(+), 18 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 256223ce97eb1..3ed7b94e46c79 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1933,21 +1933,21 @@ public class ServiceConfiguration implements PulsarConfiguration { "org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStoreProvider"; @FieldContext( - category = CATEGORY_SERVER, + category = CATEGORY_TRANSACTION, doc = "Number of threads to use for pulsar transaction replay PendingAckStore or TransactionBuffer." + "Default is 5" ) private int numTransactionReplayThreadPoolSize = 5; @FieldContext( - category = CATEGORY_SERVER, - doc = "Interval time of pending ack timer task, unit is seconds." + category = CATEGORY_TRANSACTION, + doc = "Max interval time of pending ack timer task, unit is seconds." ) private int minPendingAckTimerTaskIntervalTime = 3600; @FieldContext( - category = CATEGORY_SERVER, - doc = "Interval time of pending ack timer task, unit is seconds." + category = CATEGORY_TRANSACTION, + doc = "Min interval time of pending ack timer task, unit is seconds." ) private int maxPendingAckTimerTaskIntervalTime = 36000; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java index 9d20f05f5c685..b9c4c104c4ff8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java @@ -37,11 +37,14 @@ public interface PendingAckStore { * Replay pending ack to recover the pending ack subscription pending ack state. * * @param pendingAckHandle the handle of pending ack + * @param executorService the replay executor service */ - void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService getExecutor); + void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService executorService); /** * Close the transaction pending ack store. + * + * @return a future represents the result of this operation */ CompletableFuture closeAsync(); @@ -49,7 +52,7 @@ public interface PendingAckStore { * Append the individual pending ack operation to the ack persistent store. * * @param txnID {@link TxnID} transaction id. - * @param positions {@link PositionImpl} the pending ack postion. + * @param positions {@link List} the list of position and batch size. * @return a future represents the result of this operation */ CompletableFuture appendIndividualAck(TxnID txnID, List> positions); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java index 4104e36f75b41..61cf63eb3aee3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java @@ -55,12 +55,11 @@ static TransactionPendingAckStoreProvider newProvider(String providerClassName) * Open the pending ack store. * * @param subscription {@link PersistentSubscription} - * @param subName {@link String} * @return a future represents the result of the operation. * an instance of {@link PendingAckStore} is returned * if the operation succeeds. */ - CompletableFuture newPendingAckStore(PersistentSubscription subscription, String subName); + CompletableFuture newPendingAckStore(PersistentSubscription subscription); /** * Close the provider and release all the resources. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java index 39cb094d55cef..3289ca144fcd1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java @@ -29,7 +29,7 @@ import java.util.concurrent.ScheduledExecutorService; /** - * The In memory implementation of {@link PendingAckStore}. + * In memory implementation of {@link PendingAckStore}. */ public class InMemoryPendingAckStore implements PendingAckStore { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java index b39909173ed66..b60de3916c57e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java @@ -48,7 +48,7 @@ public class MLPendingAckStoreProvider implements TransactionPendingAckStoreProv private static final long tickTimeMillis = 1L; @Override - public CompletableFuture newPendingAckStore(PersistentSubscription subscription, String subName) { + public CompletableFuture newPendingAckStore(PersistentSubscription subscription) { CompletableFuture pendingAckStoreFuture = new CompletableFuture<>(); if (subscription == null) { @@ -59,7 +59,7 @@ public CompletableFuture newPendingAckStore(PersistentSubscript PersistentTopic originPersistentTopic = (PersistentTopic) subscription.getTopic(); String pendingAckTopicName = MLPendingAckStore - .getTransactionPendingAckStoreSuffix(originPersistentTopic.getName(), subName); + .getTransactionPendingAckStoreSuffix(originPersistentTopic.getName(), subscription.getName()); originPersistentTopic.getBrokerService().getManagedLedgerFactory() .asyncOpen(TopicName.get(pendingAckTopicName).getPersistenceNamingEncoding(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 91245d8551b24..296dcda6902cf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -117,7 +117,7 @@ public PendingAckHandleImpl(PersistentSubscription persistentSubscription) { ((PersistentTopic) this.persistentSubscription.getTopic()) .getBrokerService().getPulsar().getTransactionPendingAckStoreProvider(); this.pendingAckStoreFuture = - pendingAckStoreProvider.newPendingAckStore(persistentSubscription, subName); + pendingAckStoreProvider.newPendingAckStore(persistentSubscription); this.pendingAckStoreFuture.thenAccept(pendingAckStore -> { if (pendingAckStore instanceof MLPendingAckStore) { @@ -196,6 +196,9 @@ && isAckSetOverlap(individualAckPositions.get(position).getLeft().getAckSet(), a this.pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendIndividualAck(txnID, positions).whenComplete((v, e) -> { if (e != null) { + //we also modify the in memory state when append fail, because we don't know the persistent + // state, when wereplay it, it will produce the wrong operation. so we append fail, we should + // wait tc time out or client abort this transaction. handleIndividualAck(txnID, positions); semaphore.release(); completableFuture.completeExceptionally(e); @@ -247,8 +250,8 @@ public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, log.debug("[{}][{}] TxnID:[{}] Cumulative ack on {}.", topicName, subName, txnID.toString(), position); } CompletableFuture completableFuture = new CompletableFuture<>(); - if (cumulativeAckOfTransaction != null && this.cumulativeAckOfTransaction.getKey().equals(txnID) - && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) <= 0) { + if (cumulativeAckOfTransaction != null && (!this.cumulativeAckOfTransaction.getKey().equals(txnID) + || compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) <= 0)) { String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + " try to cumulative batch ack position: " + position + " within range of current " + "currentPosition: " + this.cumulativeAckOfTransaction.getValue(); @@ -259,6 +262,9 @@ && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) <= this.pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendCumulativeAck(txnID, position).whenComplete((v, e) -> { if (e != null) { + //we also modify the in memory state when append fail, because we don't know the persistent + // state, when wereplay it, it will produce the wrong operation. so we append fail, we should + // wait tc time out or client abort this transaction. handleCumulativeAck(txnID, position); semaphore.release(); completableFuture.completeExceptionally(e); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index c568f82c59426..daaabcf53e8a2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -97,9 +97,9 @@ protected void cleanup() { } @Test - public void pendingAckSharedReplayTest() throws Exception { + public void individualPendingAckReplayTest() throws Exception { int messageCount = 1000; - String subName = "shared-test"; + String subName = "individual-test"; @Cleanup Producer producer = pulsarClient.newProducer() @@ -111,7 +111,7 @@ public void pendingAckSharedReplayTest() throws Exception { @Cleanup Consumer consumer = pulsarClient.newConsumer() .topic(PENDING_ACK_REPLAY_TOPIC) - .subscriptionName("shared-test") + .subscriptionName(subName) .subscriptionType(SubscriptionType.Shared) .enableBatchIndexAcknowledgment(true) .subscribe(); @@ -211,4 +211,97 @@ public void pendingAckSharedReplayTest() throws Exception { .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition()) .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == 0); } + + @Test + public void cumulativePendingAckReplayTest() throws Exception { + int messageCount = 1000; + String subName = "cumulative-test"; + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(PENDING_ACK_REPLAY_TOPIC) + .enableBatching(true) + .batchingMaxMessages(200) + .create(); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(PENDING_ACK_REPLAY_TOPIC) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Failover) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + Transaction abortTxn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + + List pendingAckMessageIds = new ArrayList<>(); + for (int i = 0; i < messageCount; i++) { + producer.send("Hello Pulsar!".getBytes()); + } + + for (int i = 0; i < messageCount; i++) { + Message message = consumer.receive(); + pendingAckMessageIds.add(message.getMessageId()); + consumer.acknowledgeCumulativeAsync(message.getMessageId(), abortTxn).get(); + } + + admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + try { + consumer.acknowledgeCumulativeAsync(pendingAckMessageIds.get(i), txn).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + } + Transaction commitTxn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); + abortTxn.abort().get(); + + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + consumer.acknowledgeCumulativeAsync(pendingAckMessageIds.get(i), commitTxn).get(); + } + commitTxn.commit().get(); + + admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); + Awaitility.await().atMost(4000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + try { + consumer.acknowledgeCumulativeAsync(pendingAckMessageIds.get(i), txn).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + } + + PersistentTopic topic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() + .getTopic(TopicName.get(PENDING_ACK_REPLAY_TOPIC).toString(), false).get().get(); + Field field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); + field.setAccessible(true); + PendingAckHandleImpl pendingAckHandle = + (PendingAckHandleImpl) field.get(topic.getSubscription(subName)); + field = PendingAckHandleImpl.class.getDeclaredField("pendingAckStoreFuture"); + field.setAccessible(true); + CompletableFuture pendingAckStoreCompletableFuture = + (CompletableFuture) field.get(pendingAckHandle); + pendingAckStoreCompletableFuture.get(); + + field = MLPendingAckStore.class.getDeclaredField("cursor"); + field.setAccessible(true); + + ManagedCursor managedCursor = (ManagedCursor) field.get(pendingAckStoreCompletableFuture.get()); + + // in order to check out the pending ack cursor is clear whether or not. + Awaitility.await() + .atMost(15000, TimeUnit.MILLISECONDS) + .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition()) + .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == 0); + } } From 516d7d224bedccef2054ef19ffc9a301f91cb469 Mon Sep 17 00:00:00 2001 From: congbo Date: Sun, 7 Feb 2021 17:42:49 +0800 Subject: [PATCH 05/18] Merge remote-tracking branch 'apache/master' into congbobo184_transaction_pendingack_persistent # Conflicts: # pom.xml # pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java # pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java # pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java # pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java --- .../mledger/impl/ManagedCursorImpl.java | 15 +- .../impl/ManagedCursorConcurrencyTest.java | 2 +- pulsar-broker/generate_protobuf.sh | 23 - pulsar-broker/generate_protobuf_docker.sh | 42 - pulsar-broker/pom.xml | 22 + .../apache/pulsar/broker/PulsarService.java | 21 + .../persistent/PersistentSubscription.java | 8 +- .../pendingack/PendingAckHandle.java | 14 +- .../pendingack/PendingAckReplyCallBack.java | 2 +- .../pendingack/PendingAckStore.java | 9 +- .../TransactionPendingAckStoreProvider.java | 10 +- .../pendingack/exceptions/package-info.java | 23 + .../impl/InMemoryPendingAckStore.java | 16 +- .../impl/MLPendingAckReplyCallBack.java | 33 +- .../pendingack/impl/MLPendingAckStore.java | 157 +-- .../impl/MLPendingAckStoreProvider.java | 35 +- .../impl/MLPendingAckStoreTimerTask.java | 63 +- .../impl/PendingAckHandleDisabled.java | 5 + .../pendingack/impl/PendingAckHandleImpl.java | 46 +- .../proto/TransactionPendingAck.java | 1254 ----------------- .../main/proto/TransactionPendingAck.proto | 45 + .../transaction/TransactionTestBase.java | 2 + .../pendingack/PendingAckPersistentTest.java | 4 +- 23 files changed, 303 insertions(+), 1548 deletions(-) delete mode 100755 pulsar-broker/generate_protobuf.sh delete mode 100755 pulsar-broker/generate_protobuf_docker.sh create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/package-info.java delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java create mode 100644 pulsar-broker/src/main/proto/TransactionPendingAck.proto diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index a195d7a1c592c..26769d6143050 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -563,7 +563,8 @@ public void asyncReadEntries(final int numberOfEntriesToRead, final ReadEntriesC final Object ctx, PositionImpl maxPosition) { checkArgument(numberOfEntriesToRead > 0); if (isClosed()) { - callback.readEntriesFailed(new ManagedLedgerException("Cursor was already closed"), ctx); + callback.readEntriesFailed(new ManagedLedgerException + .CursorAlreadyClosedException("Cursor was already closed"), ctx); return; } @@ -614,7 +615,8 @@ public void asyncGetNthEntry(int n, IndividualDeletedEntries deletedEntries, Rea Object ctx) { checkArgument(n > 0); if (isClosed()) { - callback.readEntryFailed(new ManagedLedgerException("Cursor was already closed"), ctx); + callback.readEntryFailed(new ManagedLedgerException + .CursorAlreadyClosedException("Cursor was already closed"), ctx); return; } @@ -1583,7 +1585,8 @@ public void asyncMarkDelete(final Position position, Map propertie checkArgument(position instanceof PositionImpl); if (isClosed()) { - callback.markDeleteFailed(new ManagedLedgerException("Cursor was already closed"), ctx); + callback.markDeleteFailed(new ManagedLedgerException + .CursorAlreadyClosedException("Cursor was already closed"), ctx); return; } @@ -1658,7 +1661,8 @@ protected void internalAsyncMarkDelete(final PositionImpl newPosition, Map positions, AsyncCallbacks.DeleteCallback callback, Object ctx) { if (isClosed()) { - callback.deleteFailed(new ManagedLedgerException("Cursor was already closed"), ctx); + callback.deleteFailed(new ManagedLedgerException + .CursorAlreadyClosedException("Cursor was already closed"), ctx); return; } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java index 856a7e8b1248d..0f6386d1fe1db 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java @@ -161,7 +161,7 @@ public void run() { Thread.sleep(1); } } catch (ManagedLedgerException e) { - if (!e.getMessage().equals("Cursor was already closed")) { + if (!(e instanceof ManagedLedgerException.CursorAlreadyClosedException)) { gotException.set(true); } diff --git a/pulsar-broker/generate_protobuf.sh b/pulsar-broker/generate_protobuf.sh deleted file mode 100755 index d13e17db79605..0000000000000 --- a/pulsar-broker/generate_protobuf.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - - -PROTOC=${PROTOC:-protoc} -${PROTOC} --java_out=pulsar-broker/src/main/java pulsar-broker/src/main/proto/TransactionPendingAck.proto diff --git a/pulsar-broker/generate_protobuf_docker.sh b/pulsar-broker/generate_protobuf_docker.sh deleted file mode 100755 index c51417f7f0eb4..0000000000000 --- a/pulsar-broker/generate_protobuf_docker.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Fail script in case of errors -set -e - -ROOT_DIR=$(git rev-parse --show-toplevel) -COMMON_DIR=$ROOT_DIR/ -cd $COMMON_DIR - -BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}" -BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-16.04}" - -IMAGE="$BUILD_IMAGE_NAME:$BUILD_IMAGE_VERSION" - -echo $IMAGE - -# Force to pull image in case it was updated -docker pull $IMAGE - -WORKDIR=/workdir -docker run -i \ - -v ${COMMON_DIR}:${WORKDIR} $IMAGE \ - bash -c "cd ${WORKDIR}; PROTOC=/pulsar/protobuf/src/protoc ./pulsar-broker/generate_protobuf.sh" - diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index db6ab77f51f82..f6755a60fd0cc 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -387,6 +387,9 @@ com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} true + + TransactionPendingAck.proto + @@ -399,6 +402,25 @@ + + + com.github.splunk.lightproto + lightproto-maven-plugin + ${lightproto-maven-plugin.version} + + + + generate + + + + + + src/main/proto/TransactionPendingAck.proto + + + + org.apache.maven.plugins maven-antrun-plugin diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index c8207417279ac..4caf2b03e3223 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -95,6 +95,7 @@ import org.apache.pulsar.broker.storage.ManagedLedgerStorage; import org.apache.pulsar.broker.transaction.buffer.TransactionBufferProvider; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferClientImpl; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; import org.apache.pulsar.broker.validator.MultipleListenerValidator; import org.apache.pulsar.broker.web.WebService; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -225,6 +226,9 @@ public class PulsarService implements AutoCloseable { private MetadataStoreExtended configurationMetadataStore; private PulsarResources pulsarResources; + private TransactionPendingAckStoreProvider transactionPendingAckStoreProvider; + private final ScheduledExecutorService transactionReplayExecutor; + public enum State { Init, Started, Closed } @@ -282,6 +286,13 @@ public PulsarService(ServiceConfiguration config, new DefaultThreadFactory("pulsar")); this.cacheExecutor = Executors.newScheduledThreadPool(config.getNumCacheExecutorThreadPoolSize(), new DefaultThreadFactory("zk-cache-callback")); + + if (config.isTransactionCoordinatorEnabled()) { + this.transactionReplayExecutor = Executors.newScheduledThreadPool(config.getNumExecutorThreadPoolSize(), + new DefaultThreadFactory("transaction-replay")); + } else { + this.transactionReplayExecutor = null; + } } public MetadataStoreExtended createConfigurationMetadataStore() throws MetadataStoreException { @@ -412,6 +423,10 @@ public void close() throws PulsarServerException { configurationMetadataStore.close(); } + if (transactionReplayExecutor != null) { + transactionReplayExecutor.shutdown(); + } + state = State.Closed; isClosedCondition.signalAll(); } catch (Exception e) { @@ -651,6 +666,8 @@ public Boolean get() { transactionBufferProvider = TransactionBufferProvider .newProvider(config.getTransactionBufferProviderClassName()); + transactionPendingAckStoreProvider = TransactionPendingAckStoreProvider + .newProvider(config.getTransactionPendingAckStoreProviderClassName()); } this.metricsGenerator = new MetricsGenerator(this); @@ -1068,6 +1085,10 @@ public ScheduledExecutorService getCacheExecutor() { return cacheExecutor; } + public ScheduledExecutorService getTransactionReplayExecutor() { + return transactionReplayExecutor; + } + public ScheduledExecutorService getLoadManagerExecutor() { return loadManagerExecutor; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 191be91c9395c..a9f960e61e6bc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -173,11 +173,9 @@ void setReplicated(boolean replicated) { @Override public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { - if (pendingAckHandle instanceof PendingAckHandleImpl) { - if (!((PendingAckHandleImpl) pendingAckHandle).checkIfReady()) { - throw new SubscriptionBusyException("Pending ack handle not init " + - "complete! topicName : " + topicName + " subName : " + subName); - } + if (!pendingAckHandle.checkIfReady()) { + throw new BrokerServiceException.ServiceUnitNotReadyException("Pending ack handle not init " + + "complete! topicName : " + topicName + " subName : " + subName); } cursor.updateLastActive(); if (IS_FENCED_UPDATER.get(this) == TRUE) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java index 3ef5b2b8807b0..69f1f852ab4b4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java @@ -27,7 +27,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; +import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.transaction.common.exception.TransactionConflictException; /** @@ -53,7 +53,8 @@ public interface PendingAckHandle { * @throws NotAllowedException if Use this method incorrectly eg. not use * PositionImpl or cumulative ack with a list of positions. */ - CompletableFuture individualAcknowledgeMessage(TxnID txnID, List> positions); + CompletableFuture individualAcknowledgeMessage(TxnID txnID, List> positions); /** * Acknowledge message(s) for an ongoing transaction. @@ -86,7 +87,7 @@ public interface PendingAckHandle { * @param properties Additional user-defined properties that can be associated with a particular cursor position. * @return the future of this operation. */ - CompletableFuture commitTxn(TxnID txnID, Map properties); + CompletableFuture commitTxn(TxnID txnID, Map properties); /** * Abort a transaction. @@ -118,6 +119,13 @@ public interface PendingAckHandle { */ void clearIndividualPosition(Position position); + /** + * Check the pending ack handle is ready. + * + * @return the ready or not. + */ + boolean checkIfReady(); + /** * Close the pending ack handle. * diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java index b34133974863b..3f2cc51f8ecb6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.broker.transaction.pendingack; -import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry; +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadataEntry; /** * Call back for pending ack reply. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java index b9c4c104c4ff8..3da676eb827d0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java @@ -18,16 +18,15 @@ */ package org.apache.pulsar.broker.transaction.pendingack; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; - -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; +import org.apache.pulsar.common.api.proto.CommandAck.AckType; /** * To store transaction pending ack. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java index 61cf63eb3aee3..e1a14d766d52b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java @@ -18,12 +18,10 @@ */ package org.apache.pulsar.broker.transaction.pendingack; -import org.apache.pulsar.broker.service.persistent.PersistentSubscription; - +import static com.google.common.base.Preconditions.checkArgument; import java.io.IOException; import java.util.concurrent.CompletableFuture; - -import static com.google.common.base.Preconditions.checkArgument; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; /** * Provider of transaction pending ack store. @@ -61,8 +59,4 @@ static TransactionPendingAckStoreProvider newProvider(String providerClassName) */ CompletableFuture newPendingAckStore(PersistentSubscription subscription); - /** - * Close the provider and release all the resources. - */ - void close() throws IOException; } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/package-info.java new file mode 100644 index 0000000000000..314cceec797fd --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/package-info.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * The implementation for pending ack exceptions. + */ +package org.apache.pulsar.broker.transaction.pendingack.exceptions; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java index 3289ca144fcd1..d882c80c47863 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java @@ -18,15 +18,14 @@ */ package org.apache.pulsar.broker.transaction.pendingack.impl; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.common.api.proto.PulsarApi; - -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; +import org.apache.pulsar.common.api.proto.CommandAck.AckType; /** * In memory implementation of {@link PendingAckStore}. @@ -44,7 +43,8 @@ public CompletableFuture closeAsync() { } @Override - public CompletableFuture appendIndividualAck(TxnID txnID, List> positions) { + public CompletableFuture appendIndividualAck(TxnID txnID, + List> positions) { return CompletableFuture.completedFuture(null); } @@ -54,12 +54,12 @@ public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl pos } @Override - public CompletableFuture appendCommitMark(TxnID txnID, PulsarApi.CommandAck.AckType ackType) { + public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) { return CompletableFuture.completedFuture(null); } @Override - public CompletableFuture appendAbortMark(TxnID txnID, PulsarApi.CommandAck.AckType ackType) { + public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) { return CompletableFuture.completedFuture(null); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java index ee8f5f57f2557..5378e3929e538 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java @@ -18,20 +18,19 @@ */ package org.apache.pulsar.broker.transaction.pendingack.impl; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack; -import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata; -import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry; +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadata; +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadataEntry; import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; -import org.apache.pulsar.common.util.SafeCollectionUtils; +import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; - /** * MLPendingAckStore reply call back. */ @@ -72,24 +71,30 @@ public void handleMetadataEntry(PendingAckMetadataEntry pendingAckMetadataEntry) pendingAckHandle.handleAbort(txnID, ackType); break; case COMMIT: - pendingAckHandle.handleCommit(txnID, ackType, null); + pendingAckHandle.handleCommit(txnID, ackType, Collections.emptyMap()); break; case ACK: if (ackType == AckType.Cumulative) { PendingAckMetadata pendingAckMetadata = - pendingAckMetadataEntry.getPendingAckMetadataList().get(0); + pendingAckMetadataEntry.getPendingAckMetadatasList().get(0); pendingAckHandle.handleCumulativeAck(txnID, PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId())); } else { List> positions = new ArrayList<>(); - pendingAckMetadataEntry.getPendingAckMetadataList().forEach(pendingAckMetadata -> { - if (pendingAckMetadata.getAckSetCount() == 0) { + pendingAckMetadataEntry.getPendingAckMetadatasList().forEach(pendingAckMetadata -> { + if (pendingAckMetadata.getAckSetsCount() == 0) { positions.add(new MutablePair<>(PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId()), pendingAckMetadata.getBatchSize())); } else { PositionImpl position = PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId()); - position.setAckSet(SafeCollectionUtils.longListToArray(pendingAckMetadata.getAckSetList())); + if (pendingAckMetadata.getAckSetsCount() > 0) { + long[] ackSets = new long[pendingAckMetadata.getAckSetsCount()]; + for (int i = 0; i < pendingAckMetadata.getAckSetsCount(); i++) { + ackSets[i] = pendingAckMetadata.getAckSetAt(i); + } + position.setAckSet(ackSets); + } positions.add(new MutablePair<>(position, pendingAckMetadata.getBatchSize())); } }); @@ -97,8 +102,8 @@ public void handleMetadataEntry(PendingAckMetadataEntry pendingAckMetadataEntry) } break; default: - throw new IllegalStateException("Transaction pending ack replay " + - "error with illegal state : " + pendingAckMetadataEntry.getPendingAckOp()); + throw new IllegalStateException("Transaction pending ack replay " + + "error with illegal state : " + pendingAckMetadataEntry.getPendingAckOp()); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index 9afca676e00e2..ea26d9cae646c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -19,14 +19,14 @@ package org.apache.pulsar.broker.transaction.pendingack.impl; import io.netty.buffer.ByteBuf; - +import io.netty.util.Timer; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; - -import io.netty.util.Timer; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -37,16 +37,13 @@ import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; -import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata; -import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry; -import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp; +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadata; +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadataEntry; +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckOp; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; -import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; +import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.util.SafeCollectionUtils; -import org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream; -import org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream; import org.jctools.queues.MessagePassingQueue; import org.jctools.queues.SpscArrayQueue; import org.slf4j.Logger; @@ -101,7 +98,7 @@ public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutor //TODO can control the number of entry to read private void readAsync(int numberOfEntriesToRead, AsyncCallbacks.ReadEntriesCallback readEntriesCallback) { - cursor.asyncReadEntries(numberOfEntriesToRead, readEntriesCallback, System.nanoTime()); + cursor.asyncReadEntries(numberOfEntriesToRead, readEntriesCallback, System.nanoTime(), PositionImpl.latest); } @Override @@ -127,79 +124,79 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) { } @Override - public CompletableFuture appendIndividualAck(TxnID txnID, List> positions) { - PendingAckMetadataEntry.Builder builder = PendingAckMetadataEntry.newBuilder(); - builder.setPendingAckOp(PendingAckOp.ACK); - builder.setAckType(AckType.Individual); + public CompletableFuture appendIndividualAck(TxnID txnID, + List> positions) { + PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); + pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ACK); + pendingAckMetadataEntry.setAckType(AckType.Individual); + List pendingAckMetadataList = new ArrayList<>(); positions.forEach(positionIntegerMutablePair -> { - PendingAckMetadata.Builder metadataBuilder = PendingAckMetadata.newBuilder(); + PendingAckMetadata pendingAckMetadata = new PendingAckMetadata(); PositionImpl position = positionIntegerMutablePair.getLeft(); int batchSize = positionIntegerMutablePair.getRight(); if (positionIntegerMutablePair.getLeft().getAckSet() != null) { - metadataBuilder.addAllAckSet(SafeCollectionUtils.longArrayToList(position.getAckSet())); - metadataBuilder.setBatchSize(batchSize); + for (long l : position.getAckSet()) { + pendingAckMetadata.addAckSet(l); + } } - metadataBuilder.setLedgerId(position.getLedgerId()); - metadataBuilder.setEntryId(position.getEntryId()); - PendingAckMetadata pendingAckMetadata = metadataBuilder.build(); - metadataBuilder.recycle(); - builder.addPendingAckMetadata(pendingAckMetadata); + pendingAckMetadata.setLedgerId(position.getLedgerId()); + pendingAckMetadata.setEntryId(position.getEntryId()); + pendingAckMetadata.setBatchSize(batchSize); + pendingAckMetadataList.add(pendingAckMetadata); }); - return appendCommon(builder, txnID); + pendingAckMetadataEntry.addAllPendingAckMetadatas(pendingAckMetadataList); + return appendCommon(pendingAckMetadataEntry, txnID); } @Override public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { - PendingAckMetadataEntry.Builder builder = PendingAckMetadataEntry.newBuilder(); - builder.setPendingAckOp(PendingAckOp.ACK); - builder.setAckType(AckType.Cumulative); - PendingAckMetadata.Builder metadataBuilder = PendingAckMetadata.newBuilder(); + PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); + pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ACK); + pendingAckMetadataEntry.setAckType(AckType.Cumulative); + PendingAckMetadata pendingAckMetadata = new PendingAckMetadata(); if (position.getAckSet() != null) { - metadataBuilder.addAllAckSet(SafeCollectionUtils.longArrayToList(position.getAckSet())); + for (long l : position.getAckSet()) { + pendingAckMetadata.addAckSet(l); + } } - metadataBuilder.setLedgerId(position.getLedgerId()); - metadataBuilder.setEntryId(position.getEntryId()); - PendingAckMetadata pendingAckMetadata = metadataBuilder.build(); - metadataBuilder.recycle(); - builder.addPendingAckMetadata(pendingAckMetadata); - return appendCommon(builder, txnID); + pendingAckMetadata.setLedgerId(position.getLedgerId()); + pendingAckMetadata.setEntryId(position.getEntryId()); + pendingAckMetadataEntry.addAllPendingAckMetadatas(Collections.singleton(pendingAckMetadata)); + return appendCommon(pendingAckMetadataEntry, txnID); } @Override public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) { - PendingAckMetadataEntry.Builder builder = PendingAckMetadataEntry.newBuilder(); - builder.setPendingAckOp(PendingAckOp.COMMIT); - builder.setAckType(ackType); - return appendCommon(builder, txnID); + PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); + pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.COMMIT); + pendingAckMetadataEntry.setAckType(ackType); + return appendCommon(pendingAckMetadataEntry, txnID); } @Override public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) { - PendingAckMetadataEntry.Builder builder = PendingAckMetadataEntry.newBuilder(); - builder.setPendingAckOp(PendingAckOp.ABORT); - builder.setAckType(ackType); - return appendCommon(builder, txnID); + PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); + pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ABORT); + pendingAckMetadataEntry.setAckType(ackType); + return appendCommon(pendingAckMetadataEntry, txnID); } - private CompletableFuture appendCommon(PendingAckMetadataEntry.Builder builder, TxnID txnID) { + private CompletableFuture appendCommon(PendingAckMetadataEntry pendingAckMetadataEntry, TxnID txnID) { CompletableFuture completableFuture = new CompletableFuture<>(); - builder.setTxnidLeastBits(txnID.getLeastSigBits()); - builder.setTxnidMostBits(txnID.getMostSigBits()); - PendingAckMetadataEntry pendingAckMetadataEntry = builder.build(); + pendingAckMetadataEntry.setTxnidLeastBits(txnID.getLeastSigBits()); + pendingAckMetadataEntry.setTxnidMostBits(txnID.getMostSigBits()); int transactionMetadataEntrySize = pendingAckMetadataEntry.getSerializedSize(); ByteBuf buf = PulsarByteBufAllocator.DEFAULT.buffer(transactionMetadataEntrySize, transactionMetadataEntrySize); - ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(buf); + pendingAckMetadataEntry.writeTo(buf); try { - pendingAckMetadataEntry.writeTo(outStream); managedLedger.asyncAddEntry(buf, new AsyncCallbacks.AddEntryCallback() { + @Override - public void addComplete(Position position, Object ctx) { + public void addComplete(Position position, ByteBuf entryData, Object ctx) { if (log.isDebugEnabled()) { log.debug("[{}][{}] MLPendingAckStore message append success at {}, operation : {}", - managedLedger.getName(), ctx, position, builder.getPendingAckOp()); + managedLedger.getName(), ctx, position, pendingAckMetadataEntry.getPendingAckOp()); } - builder.recycle(); - pendingAckMetadataEntry.recycle(); buf.release(); completableFuture.complete(null); } @@ -207,9 +204,7 @@ public void addComplete(Position position, Object ctx) { @Override public void addFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}][{}] MLPendingAckStore message append fail exception : {}, operation : {}", - managedLedger.getName(), ctx, exception, builder.getPendingAckOp()); - builder.recycle(); - pendingAckMetadataEntry.recycle(); + managedLedger.getName(), ctx, exception, pendingAckMetadataEntry.getPendingAckOp()); buf.release(); completableFuture.completeExceptionally(exception); } @@ -217,12 +212,8 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { } catch (Exception e) { log.error("[{}] MLPendingAckStore message append fail exception : {}", managedLedger.getName(), e); - builder.recycle(); - pendingAckMetadataEntry.recycle(); buf.release(); completableFuture.completeExceptionally(e); - } finally { - outStream.recycle(); } return completableFuture; } @@ -231,9 +222,12 @@ class PendingAckReplay implements Runnable { private final FillEntryQueueCallback fillEntryQueueCallback; private final PendingAckReplyCallBack pendingAckReplyCallBack; + private final AtomicLong exceptionNumber = new AtomicLong(); + // TODO: MAX_EXCEPTION_NUMBER can config + private static final int MAX_EXCEPTION_NUMBER = 500; PendingAckReplay(PendingAckReplyCallBack pendingAckReplyCallBack) { - this.fillEntryQueueCallback = new FillEntryQueueCallback(); + this.fillEntryQueueCallback = new FillEntryQueueCallback(exceptionNumber); this.pendingAckReplyCallBack = pendingAckReplyCallBack; } @@ -245,34 +239,23 @@ public void run() { if (entry != null) { ByteBuf buffer = entry.getDataBuffer(); currentLoadPosition = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); - ByteBufCodedInputStream stream = ByteBufCodedInputStream.get(buffer); - PendingAckMetadataEntry.Builder pendingAckMetadataEntryBuilder = - PendingAckMetadataEntry.newBuilder(); - PendingAckMetadataEntry pendingAckMetadataEntry = null; - try { - pendingAckMetadataEntry = - pendingAckMetadataEntryBuilder.mergeFrom(stream, null).build(); - pendingAckReplyCallBack.handleMetadataEntry(pendingAckMetadataEntry); - } catch (Exception e) { - if (pendingAckMetadataEntry != null) { - log.error("TxnId : [{}:{}] MLPendingAckStore reply error!", - pendingAckMetadataEntry.getTxnidMostBits(), - pendingAckMetadataEntry.getTxnidLeastBits(), e); - } else { - log.error("MLPendingAckStore reply error!", e); - } - } + PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); + pendingAckMetadataEntry.parseFrom(buffer, buffer.readableBytes()); + pendingAckReplyCallBack.handleMetadataEntry(pendingAckMetadataEntry); entry.release(); - if (pendingAckMetadataEntry != null) { - pendingAckMetadataEntry.recycle(); - } - pendingAckMetadataEntryBuilder.recycle(); - stream.recycle(); } else { + if (exceptionNumber.get() > MAX_EXCEPTION_NUMBER) { + log.error("[{}]Transaction pending ack recover fail when " + + "replay message error number > {}!", managedLedger.getName(), MAX_EXCEPTION_NUMBER); + return; + } try { Thread.sleep(1); } catch (InterruptedException e) { - //no-op + if (Thread.interrupted()) { + log.error("[{}]Transaction pending " + + "replay thread interrupt!", managedLedger.getName(), e); + } } } } @@ -283,8 +266,9 @@ public void run() { class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallback { private final AtomicLong outstandingReadsRequests = new AtomicLong(0); - - public FillEntryQueueCallback() { + private final AtomicLong exceptionNumber; + public FillEntryQueueCallback(AtomicLong exceptionNumber) { + this.exceptionNumber = exceptionNumber; } void fillQueue() { @@ -315,6 +299,7 @@ public Entry get() { public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { log.error("MLPendingAckStore stat reply fail!", exception); outstandingReadsRequests.decrementAndGet(); + exceptionNumber.getAndIncrement(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java index b60de3916c57e..0332f3607e178 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java @@ -19,11 +19,6 @@ package org.apache.pulsar.broker.transaction.pendingack.impl; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; - -import io.netty.util.HashedWheelTimer; -import io.netty.util.Timer; -import io.netty.util.concurrent.DefaultThreadFactory; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -34,7 +29,7 @@ import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; import org.apache.pulsar.broker.transaction.pendingack.exceptions.TransactionPendingAckStoreProviderException; -import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition; +import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; import org.apache.pulsar.common.naming.TopicName; /** @@ -43,10 +38,6 @@ @Slf4j public class MLPendingAckStoreProvider implements TransactionPendingAckStoreProvider { - private volatile Timer timer; - - private static final long tickTimeMillis = 1L; - @Override public CompletableFuture newPendingAckStore(PersistentSubscription subscription) { CompletableFuture pendingAckStoreFuture = new CompletableFuture<>(); @@ -70,13 +61,6 @@ public void openLedgerComplete(ManagedLedger ledger, Object ctx) { InitialPosition.Earliest, new AsyncCallbacks.OpenCursorCallback() { @Override public void openCursorComplete(ManagedCursor cursor, Object ctx) { - if (timer == null) { - synchronized (MLPendingAckStoreProvider.this) { - if (timer == null) { - initialize(); - } - } - } int maxIntervalTime = originPersistentTopic.getBrokerService().getPulsar() .getConfig().getMaxPendingAckTimerTaskIntervalTime(); @@ -84,7 +68,10 @@ public void openCursorComplete(ManagedCursor cursor, Object ctx) { originPersistentTopic.getBrokerService().getPulsar() .getConfig().getMaxPendingAckTimerTaskIntervalTime(); pendingAckStoreFuture - .complete(new MLPendingAckStore(ledger, cursor, timer, + .complete(new MLPendingAckStore(ledger, cursor, + ((PersistentTopic) subscription.getTopic()) + .getBrokerService().getPulsar() + .getTransactionTimer(), subscription.getCursor(), maxIntervalTime, minIntervalTime)); } @@ -105,16 +92,4 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { }, null); return pendingAckStoreFuture; } - - public void initialize() { - this.timer = new HashedWheelTimer(new DefaultThreadFactory("pending-ack-provider-timer"), - tickTimeMillis, TimeUnit.MILLISECONDS); - } - - @Override - public void close() { - if (timer != null) { - timer.stop(); - } - } } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTimerTask.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTimerTask.java index 2a1deaf721f62..b8bdece6a2882 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTimerTask.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreTimerTask.java @@ -22,6 +22,9 @@ import io.netty.util.Timeout; import io.netty.util.Timer; import io.netty.util.TimerTask; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; @@ -31,14 +34,9 @@ import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata; -import org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry; -import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; -import org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream; - -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadata; +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadataEntry; +import org.apache.pulsar.common.api.proto.CommandAck.AckType; /** * Pending ack timer task. @@ -84,46 +82,39 @@ public void run(Timeout timeout) { // when no transaction ack operation in this pending ack store, it will increase the interval time if (markDeletePosition.compareTo((PositionImpl) storeManagedLedger.getLastConfirmedEntry()) == 0) { int time = intervalTime + minIntervalTime; - if (time > maxIntervalTime) { - intervalTime = maxIntervalTime; - } else { - intervalTime = time; - } + intervalTime = Math.min(time, maxIntervalTime); managedCursor.markDelete(markDeletePosition); timer.newTimeout(MLPendingAckStoreTimerTask.this, intervalTime, TimeUnit.SECONDS); return; } else { int time = intervalTime - minIntervalTime; - if (time < minIntervalTime) { - intervalTime = minIntervalTime; - } else { - intervalTime = time; - } + intervalTime = Math.max(time, minIntervalTime); } // this while in order to find the last position witch can mark delete while (true) { PositionImpl nextPosition = storeManagedLedger.getNextValidPosition(markDeletePosition); + if (nextPosition.compareTo((PositionImpl) storeManagedLedger.getLastConfirmedEntry()) > 0) { + timer.newTimeout(MLPendingAckStoreTimerTask.this, intervalTime, TimeUnit.SECONDS); + return; + } Entry entry = getEntry(nextPosition).get(); ByteBuf buffer = entry.getDataBuffer(); - ByteBufCodedInputStream stream = ByteBufCodedInputStream.get(buffer); - PendingAckMetadataEntry.Builder pendingAckMetadataEntryBuilder = - PendingAckMetadataEntry.newBuilder(); - PendingAckMetadataEntry pendingAckMetadataEntry = null; + PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); + pendingAckMetadataEntry.parseFrom(buffer, buffer.readableBytes()); + try { - pendingAckMetadataEntry = - pendingAckMetadataEntryBuilder.mergeFrom(stream, null).build(); switch (pendingAckMetadataEntry.getPendingAckOp()) { case ACK: if (pendingAckMetadataEntry.getAckType() == AckType.Cumulative) { PendingAckMetadata pendingAckMetadata = - pendingAckMetadataEntry.getPendingAckMetadata(0); - handleAckCommon(PositionImpl.get(pendingAckMetadata.getLedgerId(), + pendingAckMetadataEntry.getPendingAckMetadatasList().get(0); + handleAckMarkDeletePosition(PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId()), nextPosition); } else { //this judge the pendingAckMetadataEntry is can delete PositionImpl largestPosition = null; List metadataList = - pendingAckMetadataEntry.getPendingAckMetadataList(); + pendingAckMetadataEntry.getPendingAckMetadatasList(); for (int i = 0; i < metadataList.size(); i++) { PendingAckMetadata pendingAckMetadata = metadataList.get(0); if (largestPosition == null) { @@ -140,7 +131,7 @@ public void run(Timeout timeout) { } } if (largestPosition != null) { - handleAckCommon(largestPosition, nextPosition); + handleAckMarkDeletePosition(largestPosition, nextPosition); } } break; @@ -154,11 +145,6 @@ public void run(Timeout timeout) { } } finally { entry.release(); - if (pendingAckMetadataEntry != null) { - pendingAckMetadataEntry.recycle(); - } - pendingAckMetadataEntryBuilder.recycle(); - stream.recycle(); } // when markDeletePosition is not nextPosition, before markDeletePosition can delete if (markDeletePosition != nextPosition) { @@ -168,8 +154,13 @@ public void run(Timeout timeout) { } this.timer.newTimeout(this, intervalTime, TimeUnit.MILLISECONDS); } catch (Exception e) { - log.error("PendingAck timer task error!", e); - if ("Cursor was already closed".equals(e.getCause().getMessage())) { + if (log.isDebugEnabled()) { + log.debug("PendingAck timer task error!", e); + } + if (e instanceof ManagedLedgerException.CursorAlreadyClosedException) { + if (log.isDebugEnabled()) { + log.debug("[{}]MLPendingAckStoreTimerTask will close!", this.storeManagedLedger.getName(), e); + } return; } this.timer.newTimeout(this, intervalTime, TimeUnit.MILLISECONDS); @@ -192,7 +183,7 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { return completableFuture; } - private void handleAckCommon(PositionImpl readPosition, PositionImpl storePosition) { + private void handleAckMarkDeletePosition(PositionImpl readPosition, PositionImpl storePosition) { if (readPosition.compareTo((PositionImpl) subManagedCursor.getMarkDeletedPosition()) <= 0) { markDeletePosition = storePosition; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java index 39c78148377e4..3dc207015531c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java @@ -71,6 +71,11 @@ public void clearIndividualPosition(Position position) { //no-op } + @Override + public boolean checkIfReady() { + return true; + } + @Override public CompletableFuture close() { return CompletableFuture.completedFuture(null); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 296dcda6902cf..1554cef5c2a57 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -28,7 +28,6 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Semaphore; - import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; @@ -44,7 +43,7 @@ import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; +import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.transaction.common.exception.TransactionConflictException; @@ -149,8 +148,8 @@ public CompletableFuture individualAcknowledgeMessage(TxnID txnID, // If try to ack message already acked by committed transaction or normal acknowledge, throw exception. if (((ManagedCursorImpl) persistentSubscription.getCursor()) .isMessageDeleted(position)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + - " try to ack message:" + position + " already acked before."; + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to ack message:" + position + " already acked before."; log.error(errorMsg); semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); @@ -168,8 +167,8 @@ public CompletableFuture individualAcknowledgeMessage(TxnID txnID, bitSetRecyclable.recycle(); if (isAckSetOverlap(ackSetOverlap, ((ManagedCursorImpl) persistentSubscription.getCursor()).getBatchPositionAckSet(position))) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + - " try to ack message:" + position + " already acked before."; + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to ack message:" + position + " already acked before."; log.error(errorMsg); semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); @@ -177,16 +176,16 @@ public CompletableFuture individualAcknowledgeMessage(TxnID txnID, if (this.individualAckPositions != null && individualAckPositions.containsKey(position) && isAckSetOverlap(individualAckPositions.get(position).getLeft().getAckSet(), ackSetOverlap)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + - " try to ack batch message:" + position + " in pending ack status."; + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to ack batch message:" + position + " in pending ack status."; log.error(errorMsg); semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); } } else { if (this.individualAckPositions != null && this.individualAckPositions.containsKey(position)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + - " try to ack message:" + position + " in pending ack status."; + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to ack message:" + position + " in pending ack status."; log.error(errorMsg); semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); @@ -217,7 +216,7 @@ && isAckSetOverlap(individualAckPositions.get(position).getLeft().getAckSet(), a @Override public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, - List positions) { + List positions) { if (txnID == null) { return FutureUtil.failedFuture(new NotAllowedException("TransactionID can not be null.")); @@ -228,8 +227,8 @@ public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, this.semaphore.acquireUninterruptibly(); if (positions.size() != 1) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + - " invalid cumulative ack received with multiple message ids."; + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " invalid cumulative ack received with multiple message ids."; log.error(errorMsg); semaphore.release(); return FutureUtil.failedFuture(new NotAllowedException(errorMsg)); @@ -238,9 +237,9 @@ public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, PositionImpl position = positions.get(0); if (position.compareTo((PositionImpl) persistentSubscription.getCursor().getMarkDeletedPosition()) <= 0) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + - " try to cumulative ack position: " + position + " within range of cursor's " + - "markDeletePosition: " + persistentSubscription.getCursor().getMarkDeletedPosition(); + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to cumulative ack position: " + position + " within range of cursor's " + + "markDeletePosition: " + persistentSubscription.getCursor().getMarkDeletedPosition(); log.error(errorMsg); semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); @@ -252,9 +251,9 @@ public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, CompletableFuture completableFuture = new CompletableFuture<>(); if (cumulativeAckOfTransaction != null && (!this.cumulativeAckOfTransaction.getKey().equals(txnID) || compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) <= 0)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + - " try to cumulative batch ack position: " + position + " within range of current " + - "currentPosition: " + this.cumulativeAckOfTransaction.getValue(); + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to cumulative batch ack position: " + position + " within range of current " + + "currentPosition: " + this.cumulativeAckOfTransaction.getValue(); log.error(errorMsg); semaphore.release(); return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); @@ -466,8 +465,8 @@ private void individualAckAbortCommon(TxnID txnID, HashMap pendingAckMessageForCurrentTxn) { for (Map.Entry entry : pendingAckMessageForCurrentTxn.entrySet()) { - if (entry.getValue().hasAckSet() && - individualAckPositions.containsKey(entry.getValue())) { + if (entry.getValue().hasAckSet() + && individualAckPositions.containsKey(entry.getValue())) { BitSetRecyclable thisBitSet = BitSetRecyclable.valueOf(entry.getValue().getAckSet()); thisBitSet.flip(0, individualAckPositions.get(entry.getValue()).right); @@ -503,10 +502,9 @@ protected void handleCommit(TxnID txnID, AckType ackType, Map prop private void individualAckCommitCommon(TxnID txnID, HashMap pendingAckMessageForCurrentTxn, - Map properties) { + Map properties) { if (pendingAckMessageForCurrentTxn != null) { - persistentSubscription.acknowledgeMessage( - new ArrayList<>(pendingAckMessageForCurrentTxn.values()), + persistentSubscription.acknowledgeMessage(new ArrayList<>(pendingAckMessageForCurrentTxn.values()), AckType.Individual, properties); individualAckOfTransaction.remove(txnID); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java deleted file mode 100644 index e2910bbaf08f8..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/proto/TransactionPendingAck.java +++ /dev/null @@ -1,1254 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: pulsar-broker/src/main/proto/TransactionPendingAck.proto - -package org.apache.pulsar.broker.transaction.proto; - -public final class TransactionPendingAck { - private TransactionPendingAck() {} - public static void registerAllExtensions( - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite registry) { - } - public enum PendingAckOp - implements org.apache.pulsar.shaded.com.google.protobuf.v241.Internal.EnumLite { - ACK(0, 0), - COMMIT(1, 1), - ABORT(2, 2), - ; - - public static final int ACK_VALUE = 0; - public static final int COMMIT_VALUE = 1; - public static final int ABORT_VALUE = 2; - - - public final int getNumber() { return value; } - - public static PendingAckOp valueOf(int value) { - switch (value) { - case 0: return ACK; - case 1: return COMMIT; - case 2: return ABORT; - default: return null; - } - } - - public static org.apache.pulsar.shaded.com.google.protobuf.v241.Internal.EnumLiteMap - internalGetValueMap() { - return internalValueMap; - } - private static org.apache.pulsar.shaded.com.google.protobuf.v241.Internal.EnumLiteMap - internalValueMap = - new org.apache.pulsar.shaded.com.google.protobuf.v241.Internal.EnumLiteMap() { - public PendingAckOp findValueByNumber(int number) { - return PendingAckOp.valueOf(number); - } - }; - - private final int value; - - private PendingAckOp(int index, int value) { - this.value = value; - } - - // @@protoc_insertion_point(enum_scope:pulsar.transaction.PendingAckOp) - } - - public interface PendingAckMetadataOrBuilder - extends org.apache.pulsar.shaded.com.google.protobuf.v241.MessageLiteOrBuilder { - - // optional int64 ledger_id = 1; - boolean hasLedgerId(); - long getLedgerId(); - - // optional int64 entry_id = 2; - boolean hasEntryId(); - long getEntryId(); - - // repeated int64 ack_set = 3; - java.util.List getAckSetList(); - int getAckSetCount(); - long getAckSet(int index); - - // optional int32 batch_size = 4 [default = 0]; - boolean hasBatchSize(); - int getBatchSize(); - } - public static final class PendingAckMetadata extends - org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite - implements PendingAckMetadataOrBuilder, org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream.ByteBufGeneratedMessage { - // Use PendingAckMetadata.newBuilder() to construct. - private io.netty.util.Recycler.Handle handle; - private PendingAckMetadata(io.netty.util.Recycler.Handle handle) { - this.handle = handle; - } - - private static final io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { - protected PendingAckMetadata newObject(Handle handle) { - return new PendingAckMetadata(handle); - } - }; - - public void recycle() { - this.initFields(); - this.memoizedIsInitialized = -1; - this.bitField0_ = 0; - this.memoizedSerializedSize = -1; - if (handle != null) { RECYCLER.recycle(this, handle); } - } - - private PendingAckMetadata(boolean noInit) {} - - private static final PendingAckMetadata defaultInstance; - public static PendingAckMetadata getDefaultInstance() { - return defaultInstance; - } - - public PendingAckMetadata getDefaultInstanceForType() { - return defaultInstance; - } - - private int bitField0_; - // optional int64 ledger_id = 1; - public static final int LEDGER_ID_FIELD_NUMBER = 1; - private long ledgerId_; - public boolean hasLedgerId() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - public long getLedgerId() { - return ledgerId_; - } - - // optional int64 entry_id = 2; - public static final int ENTRY_ID_FIELD_NUMBER = 2; - private long entryId_; - public boolean hasEntryId() { - return ((bitField0_ & 0x00000002) == 0x00000002); - } - public long getEntryId() { - return entryId_; - } - - // repeated int64 ack_set = 3; - public static final int ACK_SET_FIELD_NUMBER = 3; - private java.util.List ackSet_; - public java.util.List - getAckSetList() { - return ackSet_; - } - public int getAckSetCount() { - return ackSet_.size(); - } - public long getAckSet(int index) { - return ackSet_.get(index); - } - - // optional int32 batch_size = 4 [default = 0]; - public static final int BATCH_SIZE_FIELD_NUMBER = 4; - private int batchSize_; - public boolean hasBatchSize() { - return ((bitField0_ & 0x00000004) == 0x00000004); - } - public int getBatchSize() { - return batchSize_; - } - - private void initFields() { - ledgerId_ = 0L; - entryId_ = 0L; - ackSet_ = java.util.Collections.emptyList();; - batchSize_ = 0; - } - private byte memoizedIsInitialized = -1; - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized != -1) return isInitialized == 1; - - memoizedIsInitialized = 1; - return true; - } - - public void writeTo(org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream output) - throws java.io.IOException { - throw new RuntimeException("Cannot use CodedOutputStream"); - } - - public void writeTo(org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream output) - throws java.io.IOException { - getSerializedSize(); - if (((bitField0_ & 0x00000001) == 0x00000001)) { - output.writeInt64(1, ledgerId_); - } - if (((bitField0_ & 0x00000002) == 0x00000002)) { - output.writeInt64(2, entryId_); - } - for (int i = 0; i < ackSet_.size(); i++) { - output.writeInt64(3, ackSet_.get(i)); - } - if (((bitField0_ & 0x00000004) == 0x00000004)) { - output.writeInt32(4, batchSize_); - } - } - - private int memoizedSerializedSize = -1; - public int getSerializedSize() { - int size = memoizedSerializedSize; - if (size != -1) return size; - - size = 0; - if (((bitField0_ & 0x00000001) == 0x00000001)) { - size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream - .computeInt64Size(1, ledgerId_); - } - if (((bitField0_ & 0x00000002) == 0x00000002)) { - size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream - .computeInt64Size(2, entryId_); - } - { - int dataSize = 0; - for (int i = 0; i < ackSet_.size(); i++) { - dataSize += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream - .computeInt64SizeNoTag(ackSet_.get(i)); - } - size += dataSize; - size += 1 * getAckSetList().size(); - } - if (((bitField0_ & 0x00000004) == 0x00000004)) { - size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream - .computeInt32Size(4, batchSize_); - } - memoizedSerializedSize = size; - return size; - } - - private static final long serialVersionUID = 0L; - @java.lang.Override - protected java.lang.Object writeReplace() - throws java.io.ObjectStreamException { - return super.writeReplace(); - } - - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( - org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString data) - throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { - throw new RuntimeException("Disabled"); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( - org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString data, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { - throw new RuntimeException("Disabled"); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom(byte[] data) - throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { - return newBuilder().mergeFrom(data).buildParsed(); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( - byte[] data, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { - return newBuilder().mergeFrom(data, extensionRegistry) - .buildParsed(); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom(java.io.InputStream input) - throws java.io.IOException { - return newBuilder().mergeFrom(input).buildParsed(); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( - java.io.InputStream input, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return newBuilder().mergeFrom(input, extensionRegistry) - .buildParsed(); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - Builder builder = newBuilder(); - if (builder.mergeDelimitedFrom(input)) { - return builder.buildParsed(); - } else { - return null; - } - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseDelimitedFrom( - java.io.InputStream input, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - Builder builder = newBuilder(); - if (builder.mergeDelimitedFrom(input, extensionRegistry)) { - return builder.buildParsed(); - } else { - return null; - } - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( - org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input) - throws java.io.IOException { - return newBuilder().mergeFrom(input).buildParsed(); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata parseFrom( - org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return newBuilder().mergeFrom(input, extensionRegistry) - .buildParsed(); - } - - public static Builder newBuilder() { return Builder.create(); } - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata prototype) { - return newBuilder().mergeFrom(prototype); - } - public Builder toBuilder() { return newBuilder(this); } - - public static final class Builder extends - org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite.Builder< - org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata, Builder> - implements org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataOrBuilder, org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream.ByteBufMessageBuilder { - // Construct using org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.newBuilder() - private final io.netty.util.Recycler.Handle handle; - private Builder(io.netty.util.Recycler.Handle handle) { - this.handle = handle; - maybeForceBuilderInitialization(); - } - private final static io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { - protected Builder newObject(io.netty.util.Recycler.Handle handle) { - return new Builder(handle); - } - }; - - public void recycle() { - clear(); - if (handle != null) {RECYCLER.recycle(this, handle);} - } - - private void maybeForceBuilderInitialization() { - } - private static Builder create() { - return RECYCLER.get(); - } - - public Builder clear() { - super.clear(); - ledgerId_ = 0L; - bitField0_ = (bitField0_ & ~0x00000001); - entryId_ = 0L; - bitField0_ = (bitField0_ & ~0x00000002); - ackSet_ = java.util.Collections.emptyList();; - bitField0_ = (bitField0_ & ~0x00000004); - batchSize_ = 0; - bitField0_ = (bitField0_ & ~0x00000008); - return this; - } - - public Builder clone() { - return create().mergeFrom(buildPartial()); - } - - public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata getDefaultInstanceForType() { - return org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.getDefaultInstance(); - } - - public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata build() { - org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - private org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata buildParsed() - throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { - org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException( - result).asInvalidProtocolBufferException(); - } - return result; - } - - public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata buildPartial() { - org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata result = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.RECYCLER.get(); - int from_bitField0_ = bitField0_; - int to_bitField0_ = 0; - if (((from_bitField0_ & 0x00000001) == 0x00000001)) { - to_bitField0_ |= 0x00000001; - } - result.ledgerId_ = ledgerId_; - if (((from_bitField0_ & 0x00000002) == 0x00000002)) { - to_bitField0_ |= 0x00000002; - } - result.entryId_ = entryId_; - if (((bitField0_ & 0x00000004) == 0x00000004)) { - ackSet_ = java.util.Collections.unmodifiableList(ackSet_); - bitField0_ = (bitField0_ & ~0x00000004); - } - result.ackSet_ = ackSet_; - if (((from_bitField0_ & 0x00000008) == 0x00000008)) { - to_bitField0_ |= 0x00000004; - } - result.batchSize_ = batchSize_; - result.bitField0_ = to_bitField0_; - return result; - } - - public Builder mergeFrom(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata other) { - if (other == org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.getDefaultInstance()) return this; - if (other.hasLedgerId()) { - setLedgerId(other.getLedgerId()); - } - if (other.hasEntryId()) { - setEntryId(other.getEntryId()); - } - if (!other.ackSet_.isEmpty()) { - if (ackSet_.isEmpty()) { - ackSet_ = other.ackSet_; - bitField0_ = (bitField0_ & ~0x00000004); - } else { - ensureAckSetIsMutable(); - ackSet_.addAll(other.ackSet_); - } - - } - if (other.hasBatchSize()) { - setBatchSize(other.getBatchSize()); - } - return this; - } - - public final boolean isInitialized() { - return true; - } - - public Builder mergeFrom(org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - throw new java.io.IOException("Merge from CodedInputStream is disabled"); - } - public Builder mergeFrom( - org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream input, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - while (true) { - int tag = input.readTag(); - switch (tag) { - case 0: - - return this; - default: { - if (!input.skipField(tag)) { - - return this; - } - break; - } - case 8: { - bitField0_ |= 0x00000001; - ledgerId_ = input.readInt64(); - break; - } - case 16: { - bitField0_ |= 0x00000002; - entryId_ = input.readInt64(); - break; - } - case 24: { - ensureAckSetIsMutable(); - ackSet_.add(input.readInt64()); - break; - } - case 26: { - int length = input.readRawVarint32(); - int limit = input.pushLimit(length); - while (input.getBytesUntilLimit() > 0) { - addAckSet(input.readInt64()); - } - input.popLimit(limit); - break; - } - case 32: { - bitField0_ |= 0x00000008; - batchSize_ = input.readInt32(); - break; - } - } - } - } - - private int bitField0_; - - // optional int64 ledger_id = 1; - private long ledgerId_ ; - public boolean hasLedgerId() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - public long getLedgerId() { - return ledgerId_; - } - public Builder setLedgerId(long value) { - bitField0_ |= 0x00000001; - ledgerId_ = value; - - return this; - } - public Builder clearLedgerId() { - bitField0_ = (bitField0_ & ~0x00000001); - ledgerId_ = 0L; - - return this; - } - - // optional int64 entry_id = 2; - private long entryId_ ; - public boolean hasEntryId() { - return ((bitField0_ & 0x00000002) == 0x00000002); - } - public long getEntryId() { - return entryId_; - } - public Builder setEntryId(long value) { - bitField0_ |= 0x00000002; - entryId_ = value; - - return this; - } - public Builder clearEntryId() { - bitField0_ = (bitField0_ & ~0x00000002); - entryId_ = 0L; - - return this; - } - - // repeated int64 ack_set = 3; - private java.util.List ackSet_ = java.util.Collections.emptyList();; - private void ensureAckSetIsMutable() { - if (!((bitField0_ & 0x00000004) == 0x00000004)) { - ackSet_ = new java.util.ArrayList(ackSet_); - bitField0_ |= 0x00000004; - } - } - public java.util.List - getAckSetList() { - return java.util.Collections.unmodifiableList(ackSet_); - } - public int getAckSetCount() { - return ackSet_.size(); - } - public long getAckSet(int index) { - return ackSet_.get(index); - } - public Builder setAckSet( - int index, long value) { - ensureAckSetIsMutable(); - ackSet_.set(index, value); - - return this; - } - public Builder addAckSet(long value) { - ensureAckSetIsMutable(); - ackSet_.add(value); - - return this; - } - public Builder addAllAckSet( - java.lang.Iterable values) { - ensureAckSetIsMutable(); - super.addAll(values, ackSet_); - - return this; - } - public Builder clearAckSet() { - ackSet_ = java.util.Collections.emptyList();; - bitField0_ = (bitField0_ & ~0x00000004); - - return this; - } - - // optional int32 batch_size = 4 [default = 0]; - private int batchSize_ ; - public boolean hasBatchSize() { - return ((bitField0_ & 0x00000008) == 0x00000008); - } - public int getBatchSize() { - return batchSize_; - } - public Builder setBatchSize(int value) { - bitField0_ |= 0x00000008; - batchSize_ = value; - - return this; - } - public Builder clearBatchSize() { - bitField0_ = (bitField0_ & ~0x00000008); - batchSize_ = 0; - - return this; - } - - // @@protoc_insertion_point(builder_scope:pulsar.transaction.PendingAckMetadata) - } - - static { - defaultInstance = new PendingAckMetadata(true); - defaultInstance.initFields(); - } - - // @@protoc_insertion_point(class_scope:pulsar.transaction.PendingAckMetadata) - } - - public interface PendingAckMetadataEntryOrBuilder - extends org.apache.pulsar.shaded.com.google.protobuf.v241.MessageLiteOrBuilder { - - // optional uint64 txnid_least_bits = 1; - boolean hasTxnidLeastBits(); - long getTxnidLeastBits(); - - // optional uint64 txnid_most_bits = 2; - boolean hasTxnidMostBits(); - long getTxnidMostBits(); - - // optional .pulsar.transaction.PendingAckOp pending_ack_op = 3; - boolean hasPendingAckOp(); - org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp getPendingAckOp(); - - // optional .pulsar.proto.CommandAck.AckType ack_type = 4; - boolean hasAckType(); - org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType getAckType(); - - // repeated .pulsar.transaction.PendingAckMetadata pendingAckMetadata = 5; - java.util.List - getPendingAckMetadataList(); - org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata getPendingAckMetadata(int index); - int getPendingAckMetadataCount(); - } - public static final class PendingAckMetadataEntry extends - org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite - implements PendingAckMetadataEntryOrBuilder, org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream.ByteBufGeneratedMessage { - // Use PendingAckMetadataEntry.newBuilder() to construct. - private io.netty.util.Recycler.Handle handle; - private PendingAckMetadataEntry(io.netty.util.Recycler.Handle handle) { - this.handle = handle; - } - - private static final io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { - protected PendingAckMetadataEntry newObject(Handle handle) { - return new PendingAckMetadataEntry(handle); - } - }; - - public void recycle() { - this.initFields(); - this.memoizedIsInitialized = -1; - this.bitField0_ = 0; - this.memoizedSerializedSize = -1; - if (handle != null) { RECYCLER.recycle(this, handle); } - } - - private PendingAckMetadataEntry(boolean noInit) {} - - private static final PendingAckMetadataEntry defaultInstance; - public static PendingAckMetadataEntry getDefaultInstance() { - return defaultInstance; - } - - public PendingAckMetadataEntry getDefaultInstanceForType() { - return defaultInstance; - } - - private int bitField0_; - // optional uint64 txnid_least_bits = 1; - public static final int TXNID_LEAST_BITS_FIELD_NUMBER = 1; - private long txnidLeastBits_; - public boolean hasTxnidLeastBits() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - public long getTxnidLeastBits() { - return txnidLeastBits_; - } - - // optional uint64 txnid_most_bits = 2; - public static final int TXNID_MOST_BITS_FIELD_NUMBER = 2; - private long txnidMostBits_; - public boolean hasTxnidMostBits() { - return ((bitField0_ & 0x00000002) == 0x00000002); - } - public long getTxnidMostBits() { - return txnidMostBits_; - } - - // optional .pulsar.transaction.PendingAckOp pending_ack_op = 3; - public static final int PENDING_ACK_OP_FIELD_NUMBER = 3; - private org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp pendingAckOp_; - public boolean hasPendingAckOp() { - return ((bitField0_ & 0x00000004) == 0x00000004); - } - public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp getPendingAckOp() { - return pendingAckOp_; - } - - // optional .pulsar.proto.CommandAck.AckType ack_type = 4; - public static final int ACK_TYPE_FIELD_NUMBER = 4; - private org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType ackType_; - public boolean hasAckType() { - return ((bitField0_ & 0x00000008) == 0x00000008); - } - public org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType getAckType() { - return ackType_; - } - - // repeated .pulsar.transaction.PendingAckMetadata pendingAckMetadata = 5; - public static final int PENDINGACKMETADATA_FIELD_NUMBER = 5; - private java.util.List pendingAckMetadata_; - public java.util.List getPendingAckMetadataList() { - return pendingAckMetadata_; - } - public java.util.List - getPendingAckMetadataOrBuilderList() { - return pendingAckMetadata_; - } - public int getPendingAckMetadataCount() { - return pendingAckMetadata_.size(); - } - public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata getPendingAckMetadata(int index) { - return pendingAckMetadata_.get(index); - } - public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataOrBuilder getPendingAckMetadataOrBuilder( - int index) { - return pendingAckMetadata_.get(index); - } - - private void initFields() { - txnidLeastBits_ = 0L; - txnidMostBits_ = 0L; - pendingAckOp_ = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.ACK; - ackType_ = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.Individual; - pendingAckMetadata_ = java.util.Collections.emptyList(); - } - private byte memoizedIsInitialized = -1; - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized != -1) return isInitialized == 1; - - memoizedIsInitialized = 1; - return true; - } - - public void writeTo(org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream output) - throws java.io.IOException { - throw new RuntimeException("Cannot use CodedOutputStream"); - } - - public void writeTo(org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream output) - throws java.io.IOException { - getSerializedSize(); - if (((bitField0_ & 0x00000001) == 0x00000001)) { - output.writeUInt64(1, txnidLeastBits_); - } - if (((bitField0_ & 0x00000002) == 0x00000002)) { - output.writeUInt64(2, txnidMostBits_); - } - if (((bitField0_ & 0x00000004) == 0x00000004)) { - output.writeEnum(3, pendingAckOp_.getNumber()); - } - if (((bitField0_ & 0x00000008) == 0x00000008)) { - output.writeEnum(4, ackType_.getNumber()); - } - for (int i = 0; i < pendingAckMetadata_.size(); i++) { - output.writeMessage(5, pendingAckMetadata_.get(i)); - } - } - - private int memoizedSerializedSize = -1; - public int getSerializedSize() { - int size = memoizedSerializedSize; - if (size != -1) return size; - - size = 0; - if (((bitField0_ & 0x00000001) == 0x00000001)) { - size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream - .computeUInt64Size(1, txnidLeastBits_); - } - if (((bitField0_ & 0x00000002) == 0x00000002)) { - size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream - .computeUInt64Size(2, txnidMostBits_); - } - if (((bitField0_ & 0x00000004) == 0x00000004)) { - size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream - .computeEnumSize(3, pendingAckOp_.getNumber()); - } - if (((bitField0_ & 0x00000008) == 0x00000008)) { - size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream - .computeEnumSize(4, ackType_.getNumber()); - } - for (int i = 0; i < pendingAckMetadata_.size(); i++) { - size += org.apache.pulsar.shaded.com.google.protobuf.v241.CodedOutputStream - .computeMessageSize(5, pendingAckMetadata_.get(i)); - } - memoizedSerializedSize = size; - return size; - } - - private static final long serialVersionUID = 0L; - @java.lang.Override - protected java.lang.Object writeReplace() - throws java.io.ObjectStreamException { - return super.writeReplace(); - } - - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( - org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString data) - throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { - throw new RuntimeException("Disabled"); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( - org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString data, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { - throw new RuntimeException("Disabled"); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom(byte[] data) - throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { - return newBuilder().mergeFrom(data).buildParsed(); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( - byte[] data, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { - return newBuilder().mergeFrom(data, extensionRegistry) - .buildParsed(); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom(java.io.InputStream input) - throws java.io.IOException { - return newBuilder().mergeFrom(input).buildParsed(); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( - java.io.InputStream input, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return newBuilder().mergeFrom(input, extensionRegistry) - .buildParsed(); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - Builder builder = newBuilder(); - if (builder.mergeDelimitedFrom(input)) { - return builder.buildParsed(); - } else { - return null; - } - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseDelimitedFrom( - java.io.InputStream input, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - Builder builder = newBuilder(); - if (builder.mergeDelimitedFrom(input, extensionRegistry)) { - return builder.buildParsed(); - } else { - return null; - } - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( - org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input) - throws java.io.IOException { - return newBuilder().mergeFrom(input).buildParsed(); - } - public static org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry parseFrom( - org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return newBuilder().mergeFrom(input, extensionRegistry) - .buildParsed(); - } - - public static Builder newBuilder() { return Builder.create(); } - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry prototype) { - return newBuilder().mergeFrom(prototype); - } - public Builder toBuilder() { return newBuilder(this); } - - public static final class Builder extends - org.apache.pulsar.shaded.com.google.protobuf.v241.GeneratedMessageLite.Builder< - org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry, Builder> - implements org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntryOrBuilder, org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream.ByteBufMessageBuilder { - // Construct using org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry.newBuilder() - private final io.netty.util.Recycler.Handle handle; - private Builder(io.netty.util.Recycler.Handle handle) { - this.handle = handle; - maybeForceBuilderInitialization(); - } - private final static io.netty.util.Recycler RECYCLER = new io.netty.util.Recycler() { - protected Builder newObject(io.netty.util.Recycler.Handle handle) { - return new Builder(handle); - } - }; - - public void recycle() { - clear(); - if (handle != null) {RECYCLER.recycle(this, handle);} - } - - private void maybeForceBuilderInitialization() { - } - private static Builder create() { - return RECYCLER.get(); - } - - public Builder clear() { - super.clear(); - txnidLeastBits_ = 0L; - bitField0_ = (bitField0_ & ~0x00000001); - txnidMostBits_ = 0L; - bitField0_ = (bitField0_ & ~0x00000002); - pendingAckOp_ = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.ACK; - bitField0_ = (bitField0_ & ~0x00000004); - ackType_ = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.Individual; - bitField0_ = (bitField0_ & ~0x00000008); - pendingAckMetadata_ = java.util.Collections.emptyList(); - bitField0_ = (bitField0_ & ~0x00000010); - return this; - } - - public Builder clone() { - return create().mergeFrom(buildPartial()); - } - - public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry getDefaultInstanceForType() { - return org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry.getDefaultInstance(); - } - - public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry build() { - org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - private org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry buildParsed() - throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { - org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException( - result).asInvalidProtocolBufferException(); - } - return result; - } - - public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry buildPartial() { - org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry result = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry.RECYCLER.get(); - int from_bitField0_ = bitField0_; - int to_bitField0_ = 0; - if (((from_bitField0_ & 0x00000001) == 0x00000001)) { - to_bitField0_ |= 0x00000001; - } - result.txnidLeastBits_ = txnidLeastBits_; - if (((from_bitField0_ & 0x00000002) == 0x00000002)) { - to_bitField0_ |= 0x00000002; - } - result.txnidMostBits_ = txnidMostBits_; - if (((from_bitField0_ & 0x00000004) == 0x00000004)) { - to_bitField0_ |= 0x00000004; - } - result.pendingAckOp_ = pendingAckOp_; - if (((from_bitField0_ & 0x00000008) == 0x00000008)) { - to_bitField0_ |= 0x00000008; - } - result.ackType_ = ackType_; - if (((bitField0_ & 0x00000010) == 0x00000010)) { - pendingAckMetadata_ = java.util.Collections.unmodifiableList(pendingAckMetadata_); - bitField0_ = (bitField0_ & ~0x00000010); - } - result.pendingAckMetadata_ = pendingAckMetadata_; - result.bitField0_ = to_bitField0_; - return result; - } - - public Builder mergeFrom(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry other) { - if (other == org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadataEntry.getDefaultInstance()) return this; - if (other.hasTxnidLeastBits()) { - setTxnidLeastBits(other.getTxnidLeastBits()); - } - if (other.hasTxnidMostBits()) { - setTxnidMostBits(other.getTxnidMostBits()); - } - if (other.hasPendingAckOp()) { - setPendingAckOp(other.getPendingAckOp()); - } - if (other.hasAckType()) { - setAckType(other.getAckType()); - } - if (!other.pendingAckMetadata_.isEmpty()) { - if (pendingAckMetadata_.isEmpty()) { - pendingAckMetadata_ = other.pendingAckMetadata_; - bitField0_ = (bitField0_ & ~0x00000010); - } else { - ensurePendingAckMetadataIsMutable(); - pendingAckMetadata_.addAll(other.pendingAckMetadata_); - } - - } - return this; - } - - public final boolean isInitialized() { - return true; - } - - public Builder mergeFrom(org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - throw new java.io.IOException("Merge from CodedInputStream is disabled"); - } - public Builder mergeFrom( - org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream input, - org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - while (true) { - int tag = input.readTag(); - switch (tag) { - case 0: - - return this; - default: { - if (!input.skipField(tag)) { - - return this; - } - break; - } - case 8: { - bitField0_ |= 0x00000001; - txnidLeastBits_ = input.readUInt64(); - break; - } - case 16: { - bitField0_ |= 0x00000002; - txnidMostBits_ = input.readUInt64(); - break; - } - case 24: { - int rawValue = input.readEnum(); - org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp value = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.valueOf(rawValue); - if (value != null) { - bitField0_ |= 0x00000004; - pendingAckOp_ = value; - } - break; - } - case 32: { - int rawValue = input.readEnum(); - org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType value = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.valueOf(rawValue); - if (value != null) { - bitField0_ |= 0x00000008; - ackType_ = value; - } - break; - } - case 42: { - org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.Builder subBuilder = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.newBuilder(); - input.readMessage(subBuilder, extensionRegistry); - addPendingAckMetadata(subBuilder.buildPartial()); - break; - } - } - } - } - - private int bitField0_; - - // optional uint64 txnid_least_bits = 1; - private long txnidLeastBits_ ; - public boolean hasTxnidLeastBits() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - public long getTxnidLeastBits() { - return txnidLeastBits_; - } - public Builder setTxnidLeastBits(long value) { - bitField0_ |= 0x00000001; - txnidLeastBits_ = value; - - return this; - } - public Builder clearTxnidLeastBits() { - bitField0_ = (bitField0_ & ~0x00000001); - txnidLeastBits_ = 0L; - - return this; - } - - // optional uint64 txnid_most_bits = 2; - private long txnidMostBits_ ; - public boolean hasTxnidMostBits() { - return ((bitField0_ & 0x00000002) == 0x00000002); - } - public long getTxnidMostBits() { - return txnidMostBits_; - } - public Builder setTxnidMostBits(long value) { - bitField0_ |= 0x00000002; - txnidMostBits_ = value; - - return this; - } - public Builder clearTxnidMostBits() { - bitField0_ = (bitField0_ & ~0x00000002); - txnidMostBits_ = 0L; - - return this; - } - - // optional .pulsar.transaction.PendingAckOp pending_ack_op = 3; - private org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp pendingAckOp_ = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.ACK; - public boolean hasPendingAckOp() { - return ((bitField0_ & 0x00000004) == 0x00000004); - } - public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp getPendingAckOp() { - return pendingAckOp_; - } - public Builder setPendingAckOp(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000004; - pendingAckOp_ = value; - - return this; - } - public Builder clearPendingAckOp() { - bitField0_ = (bitField0_ & ~0x00000004); - pendingAckOp_ = org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckOp.ACK; - - return this; - } - - // optional .pulsar.proto.CommandAck.AckType ack_type = 4; - private org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType ackType_ = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.Individual; - public boolean hasAckType() { - return ((bitField0_ & 0x00000008) == 0x00000008); - } - public org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType getAckType() { - return ackType_; - } - public Builder setAckType(org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000008; - ackType_ = value; - - return this; - } - public Builder clearAckType() { - bitField0_ = (bitField0_ & ~0x00000008); - ackType_ = org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType.Individual; - - return this; - } - - // repeated .pulsar.transaction.PendingAckMetadata pendingAckMetadata = 5; - private java.util.List pendingAckMetadata_ = - java.util.Collections.emptyList(); - private void ensurePendingAckMetadataIsMutable() { - if (!((bitField0_ & 0x00000010) == 0x00000010)) { - pendingAckMetadata_ = new java.util.ArrayList(pendingAckMetadata_); - bitField0_ |= 0x00000010; - } - } - - public java.util.List getPendingAckMetadataList() { - return java.util.Collections.unmodifiableList(pendingAckMetadata_); - } - public int getPendingAckMetadataCount() { - return pendingAckMetadata_.size(); - } - public org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata getPendingAckMetadata(int index) { - return pendingAckMetadata_.get(index); - } - public Builder setPendingAckMetadata( - int index, org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata value) { - if (value == null) { - throw new NullPointerException(); - } - ensurePendingAckMetadataIsMutable(); - pendingAckMetadata_.set(index, value); - - return this; - } - public Builder setPendingAckMetadata( - int index, org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.Builder builderForValue) { - ensurePendingAckMetadataIsMutable(); - pendingAckMetadata_.set(index, builderForValue.build()); - - return this; - } - public Builder addPendingAckMetadata(org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata value) { - if (value == null) { - throw new NullPointerException(); - } - ensurePendingAckMetadataIsMutable(); - pendingAckMetadata_.add(value); - - return this; - } - public Builder addPendingAckMetadata( - int index, org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata value) { - if (value == null) { - throw new NullPointerException(); - } - ensurePendingAckMetadataIsMutable(); - pendingAckMetadata_.add(index, value); - - return this; - } - public Builder addPendingAckMetadata( - org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.Builder builderForValue) { - ensurePendingAckMetadataIsMutable(); - pendingAckMetadata_.add(builderForValue.build()); - - return this; - } - public Builder addPendingAckMetadata( - int index, org.apache.pulsar.broker.transaction.proto.TransactionPendingAck.PendingAckMetadata.Builder builderForValue) { - ensurePendingAckMetadataIsMutable(); - pendingAckMetadata_.add(index, builderForValue.build()); - - return this; - } - public Builder addAllPendingAckMetadata( - java.lang.Iterable values) { - ensurePendingAckMetadataIsMutable(); - super.addAll(values, pendingAckMetadata_); - - return this; - } - public Builder clearPendingAckMetadata() { - pendingAckMetadata_ = java.util.Collections.emptyList(); - bitField0_ = (bitField0_ & ~0x00000010); - - return this; - } - public Builder removePendingAckMetadata(int index) { - ensurePendingAckMetadataIsMutable(); - pendingAckMetadata_.remove(index); - - return this; - } - - // @@protoc_insertion_point(builder_scope:pulsar.transaction.PendingAckMetadataEntry) - } - - static { - defaultInstance = new PendingAckMetadataEntry(true); - defaultInstance.initFields(); - } - - // @@protoc_insertion_point(class_scope:pulsar.transaction.PendingAckMetadataEntry) - } - - - static { - } - - // @@protoc_insertion_point(outer_class_scope) -} diff --git a/pulsar-broker/src/main/proto/TransactionPendingAck.proto b/pulsar-broker/src/main/proto/TransactionPendingAck.proto new file mode 100644 index 0000000000000..0e61aa2c37594 --- /dev/null +++ b/pulsar-broker/src/main/proto/TransactionPendingAck.proto @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +syntax = "proto2"; + +import "pulsar-common/src/main/proto/PulsarApi.proto"; +package pulsar.proto; +option java_package = "org.apache.pulsar.broker.transaction.pendingack.proto"; +option optimize_for = SPEED; + +enum PendingAckOp { + ACK = 1; + COMMIT = 2; + ABORT = 3; +} + +message PendingAckMetadata { + required uint64 ledgerId = 1; + required uint64 entryId = 2; + repeated int64 ack_set = 3; + optional int32 batch_size = 4; +} + +message PendingAckMetadataEntry { + optional PendingAckOp pending_ack_op = 1; + optional pulsar.proto.CommandAck.AckType ack_type = 2; + optional uint64 txnid_least_bits = 3; + optional uint64 txnid_most_bits = 4; + repeated PendingAckMetadata pending_ack_metadata = 5; +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index 6a5e2540b34a1..f35c82ca970aa 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -118,6 +118,8 @@ protected void startBroker() throws Exception { conf.setWebServicePortTls(Optional.of(0)); conf.setTransactionCoordinatorEnabled(true); conf.setBrokerDeduplicationEnabled(true); + conf.setMinPendingAckTimerTaskIntervalTime(1); + conf.setMaxPendingAckTimerTaskIntervalTime(10); serviceConfigurationList.add(conf); PulsarService pulsar = spy(new PulsarService(conf)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index daaabcf53e8a2..464a4932c3b36 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -22,9 +22,7 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.transaction.TransactionTestBase; @@ -134,12 +132,12 @@ public void individualPendingAckReplayTest() throws Exception { //in order to test pending ack replay admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); Transaction commitTxn = pulsarClient.newTransaction() .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); Transaction txn = pulsarClient.newTransaction() .withTransactionTimeout(5, TimeUnit.MILLISECONDS).build().get(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); // this messageIds are ack by transaction for (int i = 0; i < pendingAckMessageIds.size(); i++) { From de975916c2d6eda49c669c9e9ca66610d10d5e98 Mon Sep 17 00:00:00 2001 From: congbo Date: Sun, 7 Feb 2021 20:25:15 +0800 Subject: [PATCH 06/18] Change exception --- .../broker/service/persistent/PersistentSubscription.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index a9f960e61e6bc..c8593d092c7bd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -174,7 +174,7 @@ void setReplicated(boolean replicated) { @Override public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { if (!pendingAckHandle.checkIfReady()) { - throw new BrokerServiceException.ServiceUnitNotReadyException("Pending ack handle not init " + throw new BrokerServiceException.NotAllowedException("Pending ack handle not init " + "complete! topicName : " + topicName + " subName : " + subName); } cursor.updateLastActive(); From ed268e1eb57b836a51d59f47fbc4ba8f59f74b32 Mon Sep 17 00:00:00 2001 From: congbo Date: Mon, 1 Mar 2021 19:31:08 +0800 Subject: [PATCH 07/18] Fix some test --- .../service/BrokerServiceException.java | 8 +++ .../persistent/PersistentSubscription.java | 2 +- .../pendingack/impl/PendingAckHandleImpl.java | 10 ++-- .../PersistentSubscriptionTest.java | 60 ++++++++++++++++++- .../transaction/TransactionConsumeTest.java | 6 ++ .../transaction/TransactionProduceTest.java | 4 ++ .../client/impl/TransactionEndToEndTest.java | 7 +++ .../client/api/PulsarClientException.java | 16 +++++ pulsar-client-cpp/include/pulsar/Result.h | 1 + pulsar-client-cpp/lib/ClientConnection.cc | 3 + pulsar-client-cpp/lib/Result.cc | 3 + .../apache/pulsar/client/impl/ClientCnx.java | 2 + pulsar-common/src/main/proto/PulsarApi.proto | 1 + .../transaction/TransactionTest.java | 3 + 14 files changed, 117 insertions(+), 9 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java index 4e0c8ced72cf4..7805af96f597e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java @@ -187,6 +187,12 @@ public TopicPoliciesCacheNotInitException() { } } + public static class PendingAckNotRecoverException extends BrokerServiceException { + public PendingAckNotRecoverException(String msg) { + super(msg); + } + } + public static org.apache.pulsar.common.api.proto.ServerError getClientErrorCode(Throwable t) { return getClientErrorCode(t, true); } @@ -234,6 +240,8 @@ private static ServerError getClientErrorCode(Throwable t, boolean checkCauseIfU return ServerError.TransactionConflict; } else if (t instanceof CoordinatorException.TransactionNotFoundException) { return ServerError.TransactionNotFound; + } else if (t instanceof PendingAckNotRecoverException) { + return ServerError.PendingAckNotRecover; } else { if (checkCauseIfUnknown) { return getClientErrorCode(t.getCause(), false); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index c8593d092c7bd..4507b75c71247 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -174,7 +174,7 @@ void setReplicated(boolean replicated) { @Override public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { if (!pendingAckHandle.checkIfReady()) { - throw new BrokerServiceException.NotAllowedException("Pending ack handle not init " + throw new BrokerServiceException.PendingAckNotRecoverException("Pending ack handle not init " + "complete! topicName : " + topicName + " subName : " + subName); } cursor.updateLastActive(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 1554cef5c2a57..c157c2fcc9b23 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -119,12 +119,10 @@ public PendingAckHandleImpl(PersistentSubscription persistentSubscription) { pendingAckStoreProvider.newPendingAckStore(persistentSubscription); this.pendingAckStoreFuture.thenAccept(pendingAckStore -> { - if (pendingAckStore instanceof MLPendingAckStore) { - changeToInitializingState(); - pendingAckStore.replayAsync(this, - ((PersistentTopic) persistentSubscription.getTopic()).getBrokerService() - .getPulsar().getTransactionReplayExecutor()); - } + changeToInitializingState(); + pendingAckStore.replayAsync(this, + ((PersistentTopic) persistentSubscription.getTopic()).getBrokerService() + .getPulsar().getTransactionReplayExecutor()); }).exceptionally(e -> { log.error("PendingAckHandleImpl init fail! TopicName : {}, SubName: {}", topicName, subName, e); return null; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index c1bda17219abe..11a637b7cc907 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -31,6 +31,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -40,6 +41,8 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedLedger; @@ -60,6 +63,10 @@ import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.PersistentTopicTest; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferProvider; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; +import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; +import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleState; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.api.proto.CommandSubscribe; @@ -70,6 +77,7 @@ import org.apache.pulsar.zookeeper.ZooKeeperCache; import org.apache.pulsar.zookeeper.ZooKeeperDataCache; import org.apache.zookeeper.ZooKeeper; +import org.awaitility.Awaitility; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.slf4j.Logger; @@ -111,6 +119,48 @@ public void setup() throws Exception { svcConfig.setTransactionCoordinatorEnabled(true); pulsarMock = spy(new PulsarService(svcConfig)); doReturn(new TopicTransactionBufferProvider()).when(pulsarMock).getTransactionBufferProvider(); + doReturn(new TransactionPendingAckStoreProvider() { + @Override + public CompletableFuture newPendingAckStore(PersistentSubscription subscription) { + return CompletableFuture.completedFuture(new PendingAckStore() { + @Override + public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService executorService) { + try { + Field field = PendingAckHandleState.class.getDeclaredField("state"); + field.setAccessible(true); + field.set(pendingAckHandle, PendingAckHandleState.State.Ready); + } catch (NoSuchFieldException | IllegalAccessException e) { + fail(); + } + } + + @Override + public CompletableFuture closeAsync() { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendIndividualAck(TxnID txnID, List> positions) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) { + return CompletableFuture.completedFuture(null); + } + }); + } + }).when(pulsarMock).getTransactionPendingAckStoreProvider(); doReturn(svcConfig).when(pulsarMock).getConfiguration(); doReturn(mock(Compactor.class)).when(pulsarMock).getCompactor(); @@ -233,8 +283,14 @@ public void testCanAcknowledgeAndAbortForTransaction() throws BrokerServiceExcep }).when(cursorMock).asyncDelete(any(List.class), any(AsyncCallbacks.DeleteCallback.class), any()); doReturn(CommandSubscribe.SubType.Exclusive).when(consumerMock).subType(); - - persistentSubscription.addConsumer(consumerMock); + Awaitility.await().atMost(3, TimeUnit.SECONDS).until(() -> { + try { + persistentSubscription.addConsumer(consumerMock); + return true; + } catch (Exception e) { + return false; + } + }); // Single ack for txn1 persistentSubscription.transactionIndividualAcknowledge(txnID1, positionsPair); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java index 0f3c6b1e49cbb..adaf34ae10516 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java @@ -46,6 +46,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.protocol.Commands; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -105,6 +106,9 @@ public void noSortedTest() throws Exception { .subscriptionType(SubscriptionType.Shared) .subscribe(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(exclusiveConsumer::isConnected); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(sharedConsumer::isConnected); + long mostSigBits = 2L; long leastSigBits = 5L; TxnID txnID = new TxnID(mostSigBits, leastSigBits); @@ -180,6 +184,8 @@ public void sortedTest() throws Exception { .subscriptionName("shared-test") .subscriptionType(SubscriptionType.Shared) .subscribe(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(exclusiveConsumer::isConnected); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(sharedConsumer::isConnected); long mostSigBits = 2L; long leastSigBits = 5L; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java index 6a42e1b623535..6d2fbcb60d5ea 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java @@ -59,6 +59,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.protocol.Commands; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -275,6 +276,8 @@ public void ackCommitTest() throws Exception { .subscriptionType(SubscriptionType.Shared) .subscribe(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + for (int i = 0; i < incomingMessageCnt; i++) { Message message = consumer.receive(); log.info("receive messageId: {}", message.getMessageId()); @@ -338,6 +341,7 @@ public void ackAbortTest() throws Exception { .enableBatchIndexAcknowledgment(true) .subscriptionType(SubscriptionType.Shared) .subscribe(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); for (int i = 0; i < incomingMessageCnt; i++) { Message message = consumer.receive(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 2f696a7fdc517..9e5c7099e3916 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -132,6 +132,7 @@ private void produceCommitTest(boolean enableBatch) throws Exception { .subscriptionName("test") .enableBatchIndexAcknowledgment(true) .subscribe(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); ProducerBuilder producerBuilder = pulsarClient .newProducer() @@ -207,6 +208,7 @@ public void produceAbortTest() throws Exception { .subscriptionName(subName) .enableBatchIndexAcknowledgment(true) .subscribe(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); // Can't receive transaction messages before abort. Message message = consumer.receive(2, TimeUnit.SECONDS); @@ -297,6 +299,7 @@ private void txnAckTest(boolean batchEnable, int maxBatchSize, .enableBatchIndexAcknowledgment(true) .subscriptionType(subscriptionType) .subscribe(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); @Cleanup Producer producer = pulsarClient.newProducer() @@ -369,6 +372,7 @@ public void txnMessageAckTest() throws Exception { .enableBatchIndexAcknowledgment(true) .acknowledgmentGroupTime(0, TimeUnit.MILLISECONDS) .subscribe(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); @Cleanup Producer producer = pulsarClient @@ -490,6 +494,7 @@ public void txnCumulativeAckTest(boolean batchEnable, int maxBatchSize, Subscrip .subscriptionType(subscriptionType) .ackTimeout(1, TimeUnit.MINUTES) .subscribe(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); @Cleanup Producer producer = pulsarClient.newProducer() @@ -633,6 +638,7 @@ public void txnMetadataHandlerRecoverTest() throws Exception { .subscriptionName("test") .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscribe(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); for (int i = 0; i < txnCnt * messageCnt; i++) { Message message = consumer.receive(); @@ -649,6 +655,7 @@ public void produceTxnMessageOrderTest() throws Exception { .topic(topic) .subscriptionName("test") .subscribe(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); @Cleanup Producer producer = pulsarClient.newProducer() diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java index 86542032fed5e..837e512c18cc9 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java @@ -864,6 +864,22 @@ public TransactionConflictException(String msg) { } } + /** + * PendingAck not recover exception thrown by Pulsar client. + */ + public static class PendingAckNotRecoverException extends PulsarClientException { + /** + * Constructs a {@code PendingAckNotRecoverException} with the specified detail message. + * + * @param msg + * The detail message (which is saved for later retrieval + * by the {@link #getMessage()} method) + */ + public PendingAckNotRecoverException(String msg) { + super(msg); + } + } + // wrap an exception to enriching more info messages. public static Throwable wrap(Throwable t, String msg) { msg += "\n" + t.getMessage(); diff --git a/pulsar-client-cpp/include/pulsar/Result.h b/pulsar-client-cpp/include/pulsar/Result.h index 3b4cbf0417c01..12aebc28ecf86 100644 --- a/pulsar-client-cpp/include/pulsar/Result.h +++ b/pulsar-client-cpp/include/pulsar/Result.h @@ -86,6 +86,7 @@ enum Result ResultTransactionConflict, /// Transaction ack conflict ResultTransactionNotFound, /// Transaction not found ResultProducerFenced, /// Producer was fenced by broker + ResultPendingAckNotRecover, /// When pending ack not recover completely }; // Return string representation of result code diff --git a/pulsar-client-cpp/lib/ClientConnection.cc b/pulsar-client-cpp/lib/ClientConnection.cc index 69f889c3ef1d9..968143ba7e80d 100644 --- a/pulsar-client-cpp/lib/ClientConnection.cc +++ b/pulsar-client-cpp/lib/ClientConnection.cc @@ -129,6 +129,9 @@ static Result getResult(ServerError serverError) { case ProducerFenced: return ResultProducerFenced; + + case PendingAckNotRecover: + return ResultPendingAckNotRecover; } // NOTE : Do not add default case in the switch above. In future if we get new cases for // ServerError and miss them in the switch above we would like to get notified. Adding diff --git a/pulsar-client-cpp/lib/Result.cc b/pulsar-client-cpp/lib/Result.cc index 86d7417b73674..6b278dc0d1138 100644 --- a/pulsar-client-cpp/lib/Result.cc +++ b/pulsar-client-cpp/lib/Result.cc @@ -156,6 +156,9 @@ const char* strResult(Result result) { case ResultProducerFenced: return "ResultProducerFenced"; + + case ResultPendingAckNotRecover: + return "ResultPendingAckNotRecover"; }; // NOTE : Do not add default case in the switch above. In future if we get new cases for // ServerError and miss them in the switch above we would like to get notified. Adding diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 291474118eb3a..84ac98ee9baab 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -1116,6 +1116,8 @@ private PulsarClientException getPulsarClientException(ServerError error, String return new PulsarClientException.TransactionConflictException(errorMsg); case ProducerFenced: return new PulsarClientException.ProducerFencedException(errorMsg); + case PendingAckNotRecover: + return new PulsarClientException.PendingAckNotRecoverException(errorMsg); case UnknownError: default: return new PulsarClientException(errorMsg); diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto index ee5ac707164a8..57aa1899f31cb 100644 --- a/pulsar-common/src/main/proto/PulsarApi.proto +++ b/pulsar-common/src/main/proto/PulsarApi.proto @@ -224,6 +224,7 @@ enum ServerError { // use this error to indicate that this producer is now permanently // fenced. Applications are now supposed to close it and create a // new producer + PendingAckNotRecover = 26; // When pending ack not recover completely } enum AuthMethod { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java index 41524097b41c9..b0faf7ff9db11 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java @@ -29,6 +29,7 @@ import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.Test; @@ -97,6 +98,7 @@ public void transferNormalTest(String serviceUrl) throws Exception { .subscriptionType(SubscriptionType.Shared) .enableBatchIndexAcknowledgment(true) .subscribe(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(transferConsumer::isConnected); log.info("transfer consumer create finished"); @Cleanup @@ -112,6 +114,7 @@ public void transferNormalTest(String serviceUrl) throws Exception { .subscriptionName("integration-test") .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscribe(); + Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(balanceUpdateConsumer::isConnected); log.info("balance update consumer create finished"); while(true) { From 14d8268013fc02fda04e1bfe908fbd31c2fda78e Mon Sep 17 00:00:00 2001 From: congbo Date: Mon, 1 Mar 2021 22:19:42 +0800 Subject: [PATCH 08/18] Fix pending ack test. --- .../broker/transaction/pendingack/impl/MLPendingAckStore.java | 2 +- .../transaction/pendingack/impl/MLPendingAckStoreProvider.java | 3 ++- pulsar-client-cpp/lib/Result.cc | 2 +- .../src/main/java/org/apache/pulsar/client/impl/ClientCnx.java | 1 - 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index ea26d9cae646c..29e991a5031ca 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -59,7 +59,7 @@ public class MLPendingAckStore implements PendingAckStore { private final ManagedCursor cursor; - private static final String PENDING_ACK_STORE_SUFFIX = "-transaction-pendingack"; + private static final String PENDING_ACK_STORE_SUFFIX = "_transaction_pendingack"; private static final String PENDING_ACK_STORE_CURSOR_NAME = "pendingack"; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java index 0332f3607e178..f37ebe3045b10 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java @@ -54,6 +54,7 @@ public CompletableFuture newPendingAckStore(PersistentSubscript originPersistentTopic.getBrokerService().getManagedLedgerFactory() .asyncOpen(TopicName.get(pendingAckTopicName).getPersistenceNamingEncoding(), + originPersistentTopic.getManagedLedger().getConfig(), new AsyncCallbacks.OpenLedgerCallback() { @Override public void openLedgerComplete(ManagedLedger ledger, Object ctx) { @@ -89,7 +90,7 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { log.error("Open MLPendingAckStore managedLedger failed.", exception); pendingAckStoreFuture.completeExceptionally(exception); } - }, null); + }, () -> true, null); return pendingAckStoreFuture; } } \ No newline at end of file diff --git a/pulsar-client-cpp/lib/Result.cc b/pulsar-client-cpp/lib/Result.cc index 6b278dc0d1138..b05afcff204b0 100644 --- a/pulsar-client-cpp/lib/Result.cc +++ b/pulsar-client-cpp/lib/Result.cc @@ -157,7 +157,7 @@ const char* strResult(Result result) { case ResultProducerFenced: return "ResultProducerFenced"; - case ResultPendingAckNotRecover: + case ResultPendingAckNotRecover: return "ResultPendingAckNotRecover"; }; // NOTE : Do not add default case in the switch above. In future if we get new cases for diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 84ac98ee9baab..7d5273b024444 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -426,7 +426,6 @@ protected void handleAckResponse(CommandAckResponse ackResponse) { } } - @Override protected void handleMessage(CommandMessage cmdMessage, ByteBuf headersAndPayload) { checkArgument(state == State.Ready); From 3d38a40b610147e48c5233f4724757743dba04b2 Mon Sep 17 00:00:00 2001 From: congbo Date: Tue, 16 Mar 2021 22:09:51 +0800 Subject: [PATCH 09/18] Fix the test --- .../pulsar/broker/service/TransactionMarkerDeleteTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java index 5a28cc4abada2..b60a7b052246f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java @@ -33,6 +33,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.api.proto.MarkersMessageIdData; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.Markers; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -64,6 +65,7 @@ public void testTransactionMarkerDelete() throws Exception { doReturn(configuration).when(pulsarService).getConfig(); doReturn(true).when(configuration).isTransactionCoordinatorEnabled(); doReturn(managedLedger).when(topic).getManagedLedger(); + doReturn(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getLocalName()).when(topic).getName(); ManagedCursor cursor = managedLedger.openCursor("test"); PersistentSubscription persistentSubscription = new PersistentSubscription(topic, "test", managedLedger.openCursor("test"), false); From 5a12efc9e2136a6ec52d9f71377b74d0d03bc809 Mon Sep 17 00:00:00 2001 From: congbo Date: Wed, 17 Mar 2021 19:02:59 +0800 Subject: [PATCH 10/18] Delete the callback --- .../pendingack/PendingAckStore.java | 21 +- .../impl/AppendPendingAckLogCallBack.java | 35 -- .../impl/InMemoryPendingAckStore.java | 21 +- .../pendingack/impl/MLPendingAckStore.java | 28 +- .../pendingack/impl/PendingAckHandleImpl.java | 397 ++++++++---------- .../PersistentSubscriptionTest.java | 22 +- .../client/impl/TransactionEndToEndTest.java | 2 - 7 files changed, 219 insertions(+), 307 deletions(-) delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/AppendPendingAckLogCallBack.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java index e75dfdb6eb3a0..3da676eb827d0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java @@ -24,7 +24,6 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.MutablePair; -import org.apache.pulsar.broker.transaction.pendingack.impl.AppendPendingAckLogCallBack; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.proto.CommandAck.AckType; @@ -53,38 +52,34 @@ public interface PendingAckStore { * * @param txnID {@link TxnID} transaction id. * @param positions {@link List} the list of position and batch size. - * @param callBack {@link AppendPendingAckLogCallBack} the call back of append pending ack log. + * @return a future represents the result of this operation */ - void appendIndividualAck(TxnID txnID, List> positions, - AppendPendingAckLogCallBack callBack); + CompletableFuture appendIndividualAck(TxnID txnID, List> positions); /** * Append the cumulative pending ack operation to the ack persistent store. * * @param txnID {@link TxnID} transaction id. * @param position {@link PositionImpl} the pending ack position. - * @param callBack {@link AppendPendingAckLogCallBack} the call back of append pending ack log. + * @return a future represents the result of this operation */ - void appendCumulativeAck(TxnID txnID, PositionImpl position, - AppendPendingAckLogCallBack callBack); + CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position); /** * Append the pending ack commit mark to the ack persistent store. * * @param txnID {@link TxnID} the transaction id for add commit mark. * @param ackType {@link AckType} the ack type of the commit. - * @param callBack {@link AppendPendingAckLogCallBack} the call back of append pending ack log. + * @return a future represents the result of this operation */ - void appendCommitMark(TxnID txnID, AckType ackType, - AppendPendingAckLogCallBack callBack); + CompletableFuture appendCommitMark(TxnID txnID, AckType ackType); /** * Append the pending ack abort mark to the ack persistent store. * * @param txnID {@link Position} the txnID * @param ackType {@link AckType} the ack type of the abort. - * @param callBack {@link AppendPendingAckLogCallBack} the call back of append pending ack log. + * @return a future represents the result of this operation */ - void appendAbortMark(TxnID txnID, AckType ackType, - AppendPendingAckLogCallBack callBack); + CompletableFuture appendAbortMark(TxnID txnID, AckType ackType); } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/AppendPendingAckLogCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/AppendPendingAckLogCallBack.java deleted file mode 100644 index b9b0fff824918..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/AppendPendingAckLogCallBack.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.transaction.pendingack.impl; - -import org.apache.bookkeeper.mledger.ManagedLedgerException; - -public interface AppendPendingAckLogCallBack { - - /** - * Append pending ack log complete call back. - */ - void addComplete(); - - /** - * Append pending ack log fail call back. - * @param managedLedgerException {@link ManagedLedgerException} the exception when append pending ack log. - */ - void addFailed(ManagedLedgerException managedLedgerException); -} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java index 36fad715597a6..d882c80c47863 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java @@ -43,27 +43,24 @@ public CompletableFuture closeAsync() { } @Override - public void appendIndividualAck(TxnID txnID, List> positions, - AppendPendingAckLogCallBack callBack) { - callBack.addComplete(); + public CompletableFuture appendIndividualAck(TxnID txnID, + List> positions) { + return CompletableFuture.completedFuture(null); } @Override - public void appendCumulativeAck(TxnID txnID, PositionImpl position, - AppendPendingAckLogCallBack callBack) { - callBack.addComplete(); + public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { + return CompletableFuture.completedFuture(null); } @Override - public void appendCommitMark(TxnID txnID, AckType ackType, - AppendPendingAckLogCallBack callBack) { - callBack.addComplete(); + public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) { + return CompletableFuture.completedFuture(null); } @Override - public void appendAbortMark(TxnID txnID, AckType ackType, - AppendPendingAckLogCallBack callBack) { - callBack.addComplete(); + public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) { + return CompletableFuture.completedFuture(null); } } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index c8cb7d7d7a6d5..1a6bd0c990851 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -135,8 +135,8 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) { } @Override - public void appendIndividualAck(TxnID txnID, List> positions, - AppendPendingAckLogCallBack callBack) { + public CompletableFuture appendIndividualAck(TxnID txnID, + List> positions) { PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ACK); pendingAckMetadataEntry.setAckType(AckType.Individual); @@ -156,11 +156,11 @@ public void appendIndividualAck(TxnID txnID, List appendCumulativeAck(TxnID txnID, PositionImpl position) { PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ACK); pendingAckMetadataEntry.setAckType(AckType.Cumulative); @@ -173,28 +173,27 @@ public void appendCumulativeAck(TxnID txnID, PositionImpl position, AppendPendin pendingAckMetadata.setLedgerId(position.getLedgerId()); pendingAckMetadata.setEntryId(position.getEntryId()); pendingAckMetadataEntry.addAllPendingAckMetadatas(Collections.singleton(pendingAckMetadata)); - appendCommon(pendingAckMetadataEntry, txnID, callBack); + return appendCommon(pendingAckMetadataEntry, txnID); } @Override - public void appendCommitMark(TxnID txnID, AckType ackType, - AppendPendingAckLogCallBack callBack) { + public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) { PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.COMMIT); pendingAckMetadataEntry.setAckType(ackType); - appendCommon(pendingAckMetadataEntry, txnID, callBack); + return appendCommon(pendingAckMetadataEntry, txnID); } @Override - public void appendAbortMark(TxnID txnID, AckType ackType, AppendPendingAckLogCallBack callBack) { + public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) { PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ABORT); pendingAckMetadataEntry.setAckType(ackType); - appendCommon(pendingAckMetadataEntry, txnID, callBack); + return appendCommon(pendingAckMetadataEntry, txnID); } - private void appendCommon(PendingAckMetadataEntry pendingAckMetadataEntry, TxnID txnID, - AppendPendingAckLogCallBack callBack) { + private CompletableFuture appendCommon(PendingAckMetadataEntry pendingAckMetadataEntry, TxnID txnID) { + CompletableFuture completableFuture = new CompletableFuture<>(); pendingAckMetadataEntry.setTxnidLeastBits(txnID.getLeastSigBits()); pendingAckMetadataEntry.setTxnidMostBits(txnID.getMostSigBits()); int transactionMetadataEntrySize = pendingAckMetadataEntry.getSerializedSize(); @@ -227,7 +226,7 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { } buf.release(); - callBack.addComplete(); + completableFuture.complete(null); if (!metadataPositions.isEmpty()) { PositionImpl firstPosition = metadataPositions.firstEntry().getKey(); @@ -272,9 +271,10 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}][{}] MLPendingAckStore message append fail exception : {}, operation : {}", managedLedger.getName(), ctx, exception, pendingAckMetadataEntry.getPendingAckOp()); buf.release(); - callBack.addFailed(exception); + completableFuture.completeExceptionally(exception); } } , null); + return completableFuture; } class PendingAckReplay implements Runnable { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 7619c8dd036ea..c344c44eaf8ba 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -29,7 +29,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; @@ -138,90 +137,86 @@ public CompletableFuture individualAcknowledgeMessage(TxnID txnID, CompletableFuture completableFuture = new CompletableFuture<>(); this.pendingAckStoreFuture.thenAccept(pendingAckStore -> - pendingAckStore.appendIndividualAck(txnID, positions, - new AppendPendingAckLogCallBack() { - @Override - public void addComplete() { - for (MutablePair positionIntegerMutablePair : positions) { - PositionImpl position = positionIntegerMutablePair.left; - - // If try to ack message already acked by committed transaction or - // normal acknowledge,throw exception. - if (((ManagedCursorImpl) persistentSubscription.getCursor()) - .isMessageDeleted(position)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to ack message:" + position + " already acked before."; - log.error(errorMsg); - completableFuture - .completeExceptionally(new TransactionConflictException(errorMsg)); - return; - } - - if (position.hasAckSet()) { - //in order to jude the bit set is over lap, so set the covering - // the batch size bit to 1,should know the two - // bit set don't have the same point is 0 - BitSetRecyclable bitSetRecyclable = - BitSetRecyclable.valueOf(position.getAckSet()); - if (positionIntegerMutablePair.right > bitSetRecyclable.size()) { - bitSetRecyclable.set(positionIntegerMutablePair.right); - } - bitSetRecyclable.set(positionIntegerMutablePair.right, bitSetRecyclable.size()); - long[] ackSetOverlap = bitSetRecyclable.toLongArray(); - bitSetRecyclable.recycle(); - if (isAckSetOverlap(ackSetOverlap, - ((ManagedCursorImpl) persistentSubscription.getCursor()) - .getBatchPositionAckSet(position))) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" - + txnID + " try to ack message:" - + position + " already acked before."; - log.error(errorMsg); - completableFuture - .completeExceptionally(new TransactionConflictException(errorMsg)); - return; - } - - if (individualAckPositions != null - && individualAckPositions.containsKey(position) - && isAckSetOverlap(individualAckPositions - .get(position).getLeft().getAckSet(), ackSetOverlap)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" - + txnID + " try to ack batch message:" - + position + " in pending ack status."; - log.error(errorMsg); - completableFuture - .completeExceptionally(new TransactionConflictException(errorMsg)); - return; - } - } else { - if (individualAckPositions != null - && individualAckPositions.containsKey(position)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" - + txnID + " try to ack message:" - + position + " in pending ack status."; - log.error(errorMsg); - completableFuture - .completeExceptionally(new TransactionConflictException(errorMsg)); - return; - } - } - } - - handleIndividualAck(txnID, positions); - completableFuture.complete(null); + pendingAckStore.appendIndividualAck(txnID, positions).thenAccept(v -> { + for (MutablePair positionIntegerMutablePair : positions) { + PositionImpl position = positionIntegerMutablePair.left; + + // If try to ack message already acked by committed transaction or + // normal acknowledge,throw exception. + if (((ManagedCursorImpl) persistentSubscription.getCursor()) + .isMessageDeleted(position)) { + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to ack message:" + position + " already acked before."; + log.error(errorMsg); + completableFuture + .completeExceptionally(new TransactionConflictException(errorMsg)); + return; + } + + if (position.hasAckSet()) { + //in order to jude the bit set is over lap, so set the covering + // the batch size bit to 1,should know the two + // bit set don't have the same point is 0 + BitSetRecyclable bitSetRecyclable = + BitSetRecyclable.valueOf(position.getAckSet()); + if (positionIntegerMutablePair.right > bitSetRecyclable.size()) { + bitSetRecyclable.set(positionIntegerMutablePair.right); + } + bitSetRecyclable.set(positionIntegerMutablePair.right, bitSetRecyclable.size()); + long[] ackSetOverlap = bitSetRecyclable.toLongArray(); + bitSetRecyclable.recycle(); + if (isAckSetOverlap(ackSetOverlap, + ((ManagedCursorImpl) persistentSubscription.getCursor()) + .getBatchPositionAckSet(position))) { + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + + txnID + " try to ack message:" + + position + " already acked before."; + log.error(errorMsg); + completableFuture + .completeExceptionally(new TransactionConflictException(errorMsg)); + return; } - @Override - public void addFailed(ManagedLedgerException managedLedgerException) { - //we also modify the in memory state when append fail, - // because we don't know the persistent state, when were play it, - // it will produce the wrong operation. so we append fail, - // we should wait tc time out or client abort this transaction. - handleIndividualAck(txnID, positions); - completableFuture.completeExceptionally(managedLedgerException); + if (individualAckPositions != null + && individualAckPositions.containsKey(position) + && isAckSetOverlap(individualAckPositions + .get(position).getLeft().getAckSet(), ackSetOverlap)) { + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + + txnID + " try to ack batch message:" + + position + " in pending ack status."; + log.error(errorMsg); + completableFuture + .completeExceptionally(new TransactionConflictException(errorMsg)); + return; + } + } else { + if (individualAckPositions != null + && individualAckPositions.containsKey(position)) { + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + + txnID + " try to ack message:" + + position + " in pending ack status."; + log.error(errorMsg); + completableFuture + .completeExceptionally(new TransactionConflictException(errorMsg)); + return; } } - )); + } + + handleIndividualAck(txnID, positions); + completableFuture.complete(null); + }).exceptionally(e -> { + //we also modify the in memory state when append fail, + // because we don't know the persistent state, when were play it, + // it will produce the wrong operation. so we append fail, + // we should wait tc time out or client abort this transaction. + handleIndividualAck(txnID, positions); + completableFuture.completeExceptionally(e); + return null; + })).exceptionally(e -> { + completableFuture.completeExceptionally(e); + return null; + }); return completableFuture; } @@ -248,48 +243,44 @@ public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, CompletableFuture completableFuture = new CompletableFuture<>(); this.pendingAckStoreFuture.thenAccept(pendingAckStore -> - pendingAckStore.appendCumulativeAck(txnID, position, new AppendPendingAckLogCallBack() { - @Override - public void addComplete() { - if (log.isDebugEnabled()) { - log.debug("[{}][{}] TxnID:[{}] Cumulative ack on {}.", topicName, - subName, txnID.toString(), position); - } - - if (position.compareTo((PositionImpl) persistentSubscription.getCursor() - .getMarkDeletedPosition()) <= 0) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to cumulative ack position: " + position + " within range of cursor's " - + "markDeletePosition: " - + persistentSubscription.getCursor().getMarkDeletedPosition(); - log.error(errorMsg); - completableFuture.completeExceptionally(new TransactionConflictException(errorMsg)); - return; - } - - if (cumulativeAckOfTransaction != null && (!cumulativeAckOfTransaction.getKey().equals(txnID) - || compareToWithAckSet(position, cumulativeAckOfTransaction.getValue()) <= 0)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to cumulative batch ack position: " + position + " within range of current " - + "currentPosition: " + cumulativeAckOfTransaction.getValue(); - log.error(errorMsg); - completableFuture.completeExceptionally(new TransactionConflictException(errorMsg)); - return; - } + pendingAckStore.appendCumulativeAck(txnID, position).thenAccept(v -> { + if (log.isDebugEnabled()) { + log.debug("[{}][{}] TxnID:[{}] Cumulative ack on {}.", topicName, + subName, txnID.toString(), position); + } - handleCumulativeAck(txnID, position); - completableFuture.complete(null); + if (position.compareTo((PositionImpl) persistentSubscription.getCursor() + .getMarkDeletedPosition()) <= 0) { + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to cumulative ack position: " + position + " within range of cursor's " + + "markDeletePosition: " + + persistentSubscription.getCursor().getMarkDeletedPosition(); + log.error(errorMsg); + completableFuture.completeExceptionally(new TransactionConflictException(errorMsg)); + return; } - @Override - public void addFailed(ManagedLedgerException managedLedgerException) { - //we also modify the in memory state when append fail, because we don't know the persistent - // state, when wereplay it, it will produce the wrong operation. so we append fail, we should - // wait tc time out or client abort this transaction. - handleCumulativeAck(txnID, position); - completableFuture.completeExceptionally(managedLedgerException); + if (cumulativeAckOfTransaction != null && (!cumulativeAckOfTransaction.getKey().equals(txnID) + || compareToWithAckSet(position, cumulativeAckOfTransaction.getValue()) <= 0)) { + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to cumulative batch ack position: " + position + " within range of current " + + "currentPosition: " + cumulativeAckOfTransaction.getValue(); + log.error(errorMsg); + completableFuture.completeExceptionally(new TransactionConflictException(errorMsg)); + return; } - })).exceptionally(e -> { + + handleCumulativeAck(txnID, position); + completableFuture.complete(null); + }).exceptionally(e -> { + //we also modify the in memory state when append fail, because we don't know the persistent + // state, when wereplay it, it will produce the wrong operation. so we append fail, we should + // wait tc time out or client abort this transaction. + handleCumulativeAck(txnID, position); + completableFuture.completeExceptionally(e); + return null; + }) + ).exceptionally(e -> { completableFuture.completeExceptionally(e); return null; }); @@ -310,29 +301,24 @@ public synchronized CompletableFuture commitTxn(TxnID txnID, Map pendingAckStore - .appendCommitMark(txnID, AckType.Cumulative, new AppendPendingAckLogCallBack() { - @Override - public void addComplete() { - if (log.isDebugEnabled()) { - log.debug("[{}] Transaction pending ack store commit txnId : [{}] success!", - topicName, txnID); - } - persistentSubscription.acknowledgeMessage( - Collections.singletonList(cumulativeAckOfTransaction.getValue()), - AckType.Cumulative, properties); - cumulativeAckOfTransaction = null; - commitFuture.complete(null); - } - - @Override - public void addFailed(ManagedLedgerException managedLedgerException) { - log.error("[{}] Transaction pending ack store commit txnId : [{}] fail!", - topicName, txnID, managedLedgerException); - commitFuture.completeExceptionally(managedLedgerException); - } - })).exceptionally(e -> { + .appendCommitMark(txnID, AckType.Cumulative).thenAccept(v -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Transaction pending ack store commit txnId : [{}] success!", + topicName, txnID); + } + persistentSubscription.acknowledgeMessage( + Collections.singletonList(cumulativeAckOfTransaction.getValue()), + AckType.Cumulative, properties); + cumulativeAckOfTransaction = null; + commitFuture.complete(null); + }).exceptionally(e -> { + log.error("[{}] Transaction pending ack store commit txnId : [{}] fail!", + topicName, txnID, e); commitFuture.completeExceptionally(e); return null; + })).exceptionally(e -> { + commitFuture.completeExceptionally(e); + return null; }); } else { commitFuture.complete(null); @@ -342,26 +328,20 @@ public void addFailed(ManagedLedgerException managedLedgerException) { HashMap pendingAckMessageForCurrentTxn = individualAckOfTransaction.get(txnID); pendingAckStoreFuture.thenAccept(pendingAckStore -> - pendingAckStore.appendCommitMark(txnID, AckType.Individual, new AppendPendingAckLogCallBack() { - @Override - public void addComplete() { - if (log.isDebugEnabled()) { - log.debug("[{}] Transaction pending ack store commit txnId : [{}] success!", - topicName, txnID); - } - individualAckCommitCommon(txnID, pendingAckMessageForCurrentTxn, properties); - commitFuture.complete(null); - handleLowWaterMark(txnID, lowWaterMark); + pendingAckStore.appendCommitMark(txnID, AckType.Individual).thenAccept(v -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Transaction pending ack store commit txnId : [{}] success!", + topicName, txnID); } - - @Override - public void addFailed(ManagedLedgerException managedLedgerException) { - log.error("[{}] Transaction pending ack store commit txnId : [{}] fail!", - topicName, txnID, managedLedgerException); - commitFuture.completeExceptionally(managedLedgerException); - } - }) - ).exceptionally(e -> { + individualAckCommitCommon(txnID, pendingAckMessageForCurrentTxn, properties); + commitFuture.complete(null); + handleLowWaterMark(txnID, lowWaterMark); + }).exceptionally(e -> { + log.error("[{}] Transaction pending ack store commit txnId : [{}] fail!", + topicName, txnID, e); + commitFuture.completeExceptionally(e); + return null; + })).exceptionally(e -> { commitFuture.completeExceptionally(e); return null; }); @@ -380,27 +360,21 @@ public synchronized CompletableFuture abortTxn(TxnID txnId, Consumer consu CompletableFuture abortFuture = new CompletableFuture<>(); if (this.cumulativeAckOfTransaction != null) { pendingAckStoreFuture.thenAccept(pendingAckStore -> - pendingAckStore.appendAbortMark(txnId, AckType.Cumulative, - new AppendPendingAckLogCallBack() { - @Override - public void addComplete() { - if (log.isDebugEnabled()) { - log.debug("[{}] Transaction pending ack store abort txnId : [{}] success!", - topicName, txnId); - } - if (cumulativeAckOfTransaction.getKey().equals(txnId)) { - cumulativeAckOfTransaction = null; - } - persistentSubscription.redeliverUnacknowledgedMessages(consumer); - abortFuture.complete(null); - } - - @Override - public void addFailed(ManagedLedgerException managedLedgerException) { - log.error("[{}] Transaction pending ack store abort txnId : [{}] fail!", - topicName, txnId, managedLedgerException); - abortFuture.completeExceptionally(managedLedgerException); - } + pendingAckStore.appendAbortMark(txnId, AckType.Cumulative).thenAccept(v -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Transaction pending ack store abort txnId : [{}] success!", + topicName, txnId); + } + if (cumulativeAckOfTransaction.getKey().equals(txnId)) { + cumulativeAckOfTransaction = null; + } + persistentSubscription.redeliverUnacknowledgedMessages(consumer); + abortFuture.complete(null); + }).exceptionally(e -> { + log.error("[{}] Transaction pending ack store abort txnId : [{}] fail!", + topicName, txnId, e); + abortFuture.completeExceptionally(e); + return null; }) ).exceptionally(e -> { abortFuture.completeExceptionally(e); @@ -411,26 +385,21 @@ public void addFailed(ManagedLedgerException managedLedgerException) { individualAckOfTransaction.get(txnId); if (pendingAckMessageForCurrentTxn != null) { pendingAckStoreFuture.thenAccept(pendingAckStore -> - pendingAckStore.appendAbortMark(txnId, AckType.Individual, new AppendPendingAckLogCallBack() { - @Override - public void addComplete() { - if (log.isDebugEnabled()) { - log.debug("[{}] Transaction pending ack store abort txnId : [{}] success!", - topicName, txnId); - } - individualAckAbortCommon(txnId, pendingAckMessageForCurrentTxn); - persistentSubscription.redeliverUnacknowledgedMessages(consumer, - new ArrayList<>(pendingAckMessageForCurrentTxn.values())); - abortFuture.complete(null); - handleLowWaterMark(txnId, lowWaterMark); - } - - @Override - public void addFailed(ManagedLedgerException managedLedgerException) { - log.error("[{}] Transaction pending ack store abort txnId : [{}] fail!", - topicName, txnId, managedLedgerException); - abortFuture.completeExceptionally(managedLedgerException); + pendingAckStore.appendAbortMark(txnId, AckType.Individual).thenAccept(v -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Transaction pending ack store abort txnId : [{}] success!", + topicName, txnId); } + individualAckAbortCommon(txnId, pendingAckMessageForCurrentTxn); + persistentSubscription.redeliverUnacknowledgedMessages(consumer, + new ArrayList<>(pendingAckMessageForCurrentTxn.values())); + abortFuture.complete(null); + handleLowWaterMark(txnId, lowWaterMark); + }).exceptionally(e -> { + log.error("[{}] Transaction pending ack store abort txnId : [{}] fail!", + topicName, txnId, e); + abortFuture.completeExceptionally(e); + return null; }) ).exceptionally(e -> { abortFuture.completeExceptionally(e); @@ -453,20 +422,15 @@ private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { && firstTxn.getLeastSigBits() <= lowWaterMark) { this.pendingAckStoreFuture.whenComplete((pendingAckStore, throwable) -> { if (throwable == null) { - pendingAckStore.appendAbortMark(txnID, AckType.Individual, new AppendPendingAckLogCallBack() { - @Override - public void addComplete() { - log.warn("[{}] Transaction pending ack handle low water mark success! txnId : [{}], " - + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark); - individualAckOfTransaction.remove(firstTxn); - handleLowWaterMark(txnID, lowWaterMark); - } - - @Override - public void addFailed(ManagedLedgerException managedLedgerException) { - log.warn("[{}] Transaction pending ack handle low water mark fail! txnId : [{}], " - + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark); - } + pendingAckStore.appendAbortMark(txnID, AckType.Individual).thenAccept(v -> { + log.warn("[{}] Transaction pending ack handle low water mark success! txnId : [{}], " + + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark); + individualAckOfTransaction.remove(firstTxn); + handleLowWaterMark(txnID, lowWaterMark); + }).exceptionally(e -> { + log.warn("[{}] Transaction pending ack handle low water mark fail! txnId : [{}], " + + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark); + return null; }); } }); @@ -628,10 +592,9 @@ && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) > 0 protected void handleCumulativeAckRecover(TxnID txnID, PositionImpl position) { if ((position.compareTo((PositionImpl) persistentSubscription.getCursor() - .getMarkDeletedPosition()) <= 0) || (cumulativeAckOfTransaction != null - && (!cumulativeAckOfTransaction.getKey().equals(txnID) - || compareToWithAckSet(position, cumulativeAckOfTransaction.getValue()) <= 0))) { - } else { + .getMarkDeletedPosition()) > 0) && (cumulativeAckOfTransaction == null + || (cumulativeAckOfTransaction.getKey().equals(txnID) + && compareToWithAckSet(position, cumulativeAckOfTransaction.getValue()) > 0))) { handleCumulativeAck(txnID, position); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index f490378568f70..aa881e88847d0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -62,10 +62,8 @@ import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.PersistentTopicTest; import org.apache.pulsar.broker.transaction.buffer.impl.InMemTransactionBufferProvider; -import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferProvider; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; -import org.apache.pulsar.broker.transaction.pendingack.impl.AppendPendingAckLogCallBack; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleState; import org.apache.pulsar.client.api.transaction.TxnID; @@ -141,27 +139,23 @@ public CompletableFuture closeAsync() { } @Override - public void appendIndividualAck(TxnID txnID, List> positions, - AppendPendingAckLogCallBack callBack) { - callBack.addComplete(); + public CompletableFuture appendIndividualAck(TxnID txnID, List> positions) { + return CompletableFuture.completedFuture(null); } @Override - public void appendCumulativeAck(TxnID txnID, PositionImpl position, - AppendPendingAckLogCallBack callBack) { - callBack.addComplete(); + public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { + return CompletableFuture.completedFuture(null); } @Override - public void appendCommitMark(TxnID txnID, AckType ackType, - AppendPendingAckLogCallBack callBack) { - callBack.addComplete(); + public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) { + return CompletableFuture.completedFuture(null); } @Override - public void appendAbortMark(TxnID txnID, AckType ackType, - AppendPendingAckLogCallBack callBack) { - callBack.addComplete(); + public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) { + return CompletableFuture.completedFuture(null); } }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index c8329c5b528b2..72fb4b0458dd7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -39,14 +39,12 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.pulsar.broker.TransactionMetadataStoreService; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.transaction.TransactionTestBase; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.PulsarClient; From 2347bc47b4dd3d878a58f76d0a46823f1eb4f20b Mon Sep 17 00:00:00 2001 From: congbo Date: Thu, 25 Mar 2021 09:39:09 +0800 Subject: [PATCH 11/18] Delete pending ack topic create pending ack --- .../broker/service/persistent/PersistentSubscription.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index c0faefa856881..e3cf9922afcae 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -57,6 +57,7 @@ import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.transaction.pendingack.PendingAckHandle; +import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleDisabled; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; import org.apache.pulsar.client.api.transaction.TxnID; @@ -133,7 +134,8 @@ public PersistentSubscription(PersistentTopic topic, String subscriptionName, Ma this.setReplicated(replicated); if (topic.getBrokerService().getPulsar().getConfig().isTransactionCoordinatorEnabled() && !checkTopicIsEventsNames(topic.getName()) - && !topic.getName().contains(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getLocalName())) { + && !topic.getName().contains(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getLocalName()) + && !topic.getName().contains(MLPendingAckStore.PENDING_ACK_STORE_SUFFIX)) { this.pendingAckHandle = new PendingAckHandleImpl(this); } else { this.pendingAckHandle = new PendingAckHandleDisabled(); From 3f79b567c965a280eeb56821b155ad302070dd18 Mon Sep 17 00:00:00 2001 From: congbo Date: Sun, 25 Apr 2021 17:06:49 +0800 Subject: [PATCH 12/18] Add some debug log --- .../pulsar/broker/ServiceConfiguration.java | 9 +----- pulsar-broker/pom.xml | 9 ------ .../apache/pulsar/broker/PulsarService.java | 8 +---- .../nonpersistent/NonPersistentTopic.java | 6 ++-- .../pendingack/impl/PendingAckHandleImpl.java | 30 +++++++++++-------- pulsar-client-cpp/include/pulsar/Result.h | 1 + .../apache/pulsar/client/impl/ClientCnx.java | 1 + 7 files changed, 25 insertions(+), 39 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index cb322aa4f467e..8ebd6f6f532cc 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -218,13 +218,6 @@ public class ServiceConfiguration implements PulsarConfiguration { ) private int numCacheExecutorThreadPoolSize = 10; - @FieldContext( - category = CATEGORY_SERVER, - doc = "Number of threads to use for pulsar broker service." - + " The executor in thread pool will do transaction recover" - ) - private int numTransactionExecutorThreadPoolSize = Runtime.getRuntime().availableProcessors(); - @FieldContext(category = CATEGORY_SERVER, doc = "Max concurrent web requests") private int maxConcurrentHttpRequests = 1024; @@ -2047,7 +2040,7 @@ public class ServiceConfiguration implements PulsarConfiguration { doc = "Number of threads to use for pulsar transaction replay PendingAckStore or TransactionBuffer." + "Default is 5" ) - private int numTransactionReplayThreadPoolSize = 5; + private int numTransactionReplayThreadPoolSize = Runtime.getRuntime().availableProcessors(); @FieldContext( category = CATEGORY_TRANSACTION, diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 5709090eab24d..8bf6f1bcd408b 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -409,15 +409,6 @@ - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - true - - maven-dependency-plugin diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index d7a99523987b6..dfed0a9c31482 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -224,7 +224,6 @@ public class PulsarService implements AutoCloseable { private TransactionMetadataStoreService transactionMetadataStoreService; private TransactionBufferProvider transactionBufferProvider; private TransactionBufferClient transactionBufferClient; - private ScheduledExecutorService transactionExecutor; private HashedWheelTimer transactionTimer; private BrokerInterceptor brokerInterceptor; @@ -304,7 +303,7 @@ public PulsarService(ServiceConfiguration config, new DefaultThreadFactory("zk-cache-callback")); if (config.isTransactionCoordinatorEnabled()) { - this.transactionReplayExecutor = Executors.newScheduledThreadPool(config.getNumExecutorThreadPoolSize(), + this.transactionReplayExecutor = Executors.newScheduledThreadPool(config.getNumTransactionExecutorThreadPoolSize(), new DefaultThreadFactory("transaction-replay")); } else { this.transactionReplayExecutor = null; @@ -465,8 +464,6 @@ public CompletableFuture closeAsync() { transactionBufferClient.close(); } - executorServicesShutdown.shutdown(transactionExecutor); - if (coordinationService != null) { coordinationService.close(); } @@ -755,9 +752,6 @@ public Boolean get() { // Register pulsar system namespaces and start transaction meta store service if (config.isTransactionCoordinatorEnabled()) { - this.transactionExecutor = Executors.newScheduledThreadPool( - config.getNumTransactionExecutorThreadPoolSize(), - new DefaultThreadFactory("pulsar-transaction")); this.transactionBufferSnapshotService = new SystemTopicBaseTxnBufferSnapshotService(getClient()); this.transactionTimer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-transaction-timer")); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 35cfb5cf05e9e..63b16fd9bcbf8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -293,15 +293,15 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs future.complete(consumer); } }).exceptionally(e -> { - if (e instanceof ConsumerBusyException) { + if (e.getCause() instanceof ConsumerBusyException) { log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, consumerName); - } else if (e instanceof SubscriptionBusyException) { + } else if (e.getCause() instanceof SubscriptionBusyException) { log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage()); } decrementUsageCount(); - future.completeExceptionally(e); + future.completeExceptionally(e.getCause()); return null; }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 04daffca1d4a4..69ad26cc68aae 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -140,6 +140,12 @@ public CompletableFuture individualAcknowledgeMessage(TxnID txnID, pendingAckStore.appendIndividualAck(txnID, positions).thenAccept(v -> { synchronized (org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl.this) { for (MutablePair positionIntegerMutablePair : positions) { + + if (log.isDebugEnabled()) { + log.debug("[{}] individualAcknowledgeMessage position: [{}], " + + "txnId: [{}], subName: [{}]", topicName, + positionIntegerMutablePair.left, txnID, subName); + } PositionImpl position = positionIntegerMutablePair.left; // If try to ack message already acked by committed transaction or @@ -249,8 +255,8 @@ public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, this.pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendCumulativeAck(txnID, position).thenAccept(v -> { if (log.isDebugEnabled()) { - log.debug("[{}][{}] TxnID:[{}] Cumulative ack on {}.", topicName, - subName, txnID.toString(), position); + log.debug("[{}] cumulativeAcknowledgeMessage position: [{}], " + + "txnID:[{}], subName: [{}].", topicName, txnID.toString(), position, subName); } if (position.compareTo((PositionImpl) persistentSubscription.getCursor() @@ -307,8 +313,8 @@ public synchronized CompletableFuture commitTxn(TxnID txnID, Map pendingAckStore .appendCommitMark(txnID, AckType.Cumulative).thenAccept(v -> { if (log.isDebugEnabled()) { - log.debug("[{}] Transaction pending ack store commit txnId : [{}] success!", - topicName, txnID); + log.debug("[{}] Transaction pending ack store commit txnId : [{}] " + + "success! subName: [{}]", topicName, txnID, subName); } persistentSubscription.acknowledgeMessage( Collections.singletonList(cumulativeAckOfTransaction.getValue()), @@ -335,8 +341,8 @@ public synchronized CompletableFuture commitTxn(TxnID txnID, Map pendingAckMessageForCurrentTxn = individualAckOfTransaction.get(txnID); if (log.isDebugEnabled()) { - log.debug("[{}] Transaction pending ack store commit txnId : [{}] success!", - topicName, txnID); + log.debug("[{}] Transaction pending ack store commit txnId : " + + "[{}] success! subName: [{}]", topicName, txnID, subName); } individualAckCommitCommon(txnID, pendingAckMessageForCurrentTxn, properties); commitFuture.complete(null); @@ -361,7 +367,6 @@ public synchronized CompletableFuture commitTxn(TxnID txnID, Map abortTxn(TxnID txnId, Consumer consumer, long lowWaterMark) { if (!checkIfReady()) { - log.error("[{}] abortTxn", txnId); return FutureUtil.failedFuture(new ServiceUnitNotReadyException("PendingAckHandle not replay complete!")); } CompletableFuture abortFuture = new CompletableFuture<>(); @@ -369,8 +374,8 @@ public synchronized CompletableFuture abortTxn(TxnID txnId, Consumer consu pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendAbortMark(txnId, AckType.Cumulative).thenAccept(v -> { if (log.isDebugEnabled()) { - log.debug("[{}] Transaction pending ack store abort txnId : [{}] success!", - topicName, txnId); + log.debug("[{}] Transaction pending ack store abort txnId : [{}] success! subName: [{}]", + topicName, txnId, subName); } if (cumulativeAckOfTransaction.getKey().equals(txnId)) { cumulativeAckOfTransaction = null; @@ -388,15 +393,16 @@ public synchronized CompletableFuture abortTxn(TxnID txnId, Consumer consu return null; }); } else if (this.individualAckOfTransaction != null) { - log.error("[{}] individualAckOfTransaction", txnId); pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore.appendAbortMark(txnId, AckType.Individual).thenAccept(v -> { synchronized (PendingAckHandleImpl.this) { HashMap pendingAckMessageForCurrentTxn = individualAckOfTransaction.get(txnId); if (pendingAckMessageForCurrentTxn != null) { - log.error("[{}] Transaction pending ack store abort txnId : [{}] success!", - topicName, txnId); + if (log.isDebugEnabled()) { + log.debug("[{}] Transaction pending ack store abort txnId : [{}] success! " + + "subName: [{}]", topicName, txnId, subName); + } individualAckAbortCommon(txnId, pendingAckMessageForCurrentTxn); persistentSubscription.redeliverUnacknowledgedMessages(consumer, new ArrayList<>(pendingAckMessageForCurrentTxn.values())); diff --git a/pulsar-client-cpp/include/pulsar/Result.h b/pulsar-client-cpp/include/pulsar/Result.h index 96f360349fd8b..91f6858495591 100644 --- a/pulsar-client-cpp/include/pulsar/Result.h +++ b/pulsar-client-cpp/include/pulsar/Result.h @@ -86,6 +86,7 @@ enum Result ResultTransactionConflict, /// Transaction ack conflict ResultTransactionNotFound, /// Transaction not found ResultProducerFenced, /// Producer was fenced by broker + ResultMemoryBufferIsFull, /// Client-wide memory limit has been reached }; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 12fceb16311c9..7927a458cee05 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -426,6 +426,7 @@ protected void handleAckResponse(CommandAckResponse ackResponse) { } } + @Override protected void handleMessage(CommandMessage cmdMessage, ByteBuf headersAndPayload) { checkArgument(state == State.Ready); From 8a2c078ccee15be51bef5305ed1a017b60586134 Mon Sep 17 00:00:00 2001 From: congbo Date: Sun, 25 Apr 2021 17:31:45 +0800 Subject: [PATCH 13/18] Fix some codestyle --- .../src/main/java/org/apache/pulsar/broker/PulsarService.java | 3 ++- .../transaction/pendingack/impl/PendingAckHandleImpl.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index dfed0a9c31482..c3a4e5cd59e99 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -303,7 +303,8 @@ public PulsarService(ServiceConfiguration config, new DefaultThreadFactory("zk-cache-callback")); if (config.isTransactionCoordinatorEnabled()) { - this.transactionReplayExecutor = Executors.newScheduledThreadPool(config.getNumTransactionExecutorThreadPoolSize(), + this.transactionReplayExecutor = Executors.newScheduledThreadPool( + config.getNumTransactionReplayThreadPoolSize(), new DefaultThreadFactory("transaction-replay")); } else { this.transactionReplayExecutor = null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 69ad26cc68aae..440f2314357c9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -142,8 +142,8 @@ public CompletableFuture individualAcknowledgeMessage(TxnID txnID, for (MutablePair positionIntegerMutablePair : positions) { if (log.isDebugEnabled()) { - log.debug("[{}] individualAcknowledgeMessage position: [{}], " + - "txnId: [{}], subName: [{}]", topicName, + log.debug("[{}] individualAcknowledgeMessage position: [{}], " + + "txnId: [{}], subName: [{}]", topicName, positionIntegerMutablePair.left, txnID, subName); } PositionImpl position = positionIntegerMutablePair.left; From b668c0ba32da31fbea471dda767e9b2bdd5caaad Mon Sep 17 00:00:00 2001 From: congbo Date: Sun, 25 Apr 2021 21:39:14 +0800 Subject: [PATCH 14/18] Fix some test --- .../broker/service/PersistentTopicTest.java | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 544e0edbeccd2..8787ec36cf636 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -40,9 +40,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import java.io.IOException; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -574,8 +572,8 @@ public void testMaxSameAddressProducers() throws Exception { PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); - InetAddress address1 = InetAddress.getLoopbackAddress(); - InetAddress address2 = InetAddress.getLocalHost(); + InetAddress address1 = InetAddress.getByName("127.0.0.1"); + InetAddress address2 = InetAddress.getByName("0.0.0.0"); String ipAddress1 = address1.getHostAddress(); String ipAddress2 = address2.getHostAddress(); @@ -827,9 +825,9 @@ private void testMaxConsumersShared() throws Exception { Consumer consumer3 = new Consumer(sub, SubType.Shared, topic.getName(), 3 /* consumer id */, 0, "Cons3"/* consumer name */, 50000, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, InitialPosition.Latest, null); - addConsumerToSubscription.invoke(topic, sub, consumer3); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub, consumer3)).get(); fail("should have failed"); - } catch (InvocationTargetException e) { + } catch (ExecutionException e) { assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); } @@ -851,9 +849,9 @@ private void testMaxConsumersShared() throws Exception { Consumer consumer5 = new Consumer(sub2, SubType.Shared, topic.getName(), 5 /* consumer id */, 0, "Cons5"/* consumer name */, 50000, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, InitialPosition.Latest, null); - addConsumerToSubscription.invoke(topic, sub2, consumer5); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer5)).get(); fail("should have failed"); - } catch (InvocationTargetException e) { + } catch (ExecutionException e) { assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); } } @@ -922,9 +920,9 @@ private void testMaxConsumersFailover() throws Exception { Consumer consumer3 = new Consumer(sub, SubType.Failover, topic.getName(), 3 /* consumer id */, 0, "Cons3"/* consumer name */, 50000, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, InitialPosition.Latest, null); - addConsumerToSubscription.invoke(topic, sub, consumer3); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub, consumer3)).get(); fail("should have failed"); - } catch (InvocationTargetException e) { + } catch (ExecutionException e) { assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); } @@ -946,9 +944,9 @@ private void testMaxConsumersFailover() throws Exception { Consumer consumer5 = new Consumer(sub2, SubType.Failover, topic.getName(), 5 /* consumer id */, 0, "Cons5"/* consumer name */, 50000, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, InitialPosition.Latest, null); - addConsumerToSubscription.invoke(topic, sub2, consumer5); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer5)).get(); fail("should have failed"); - } catch (InvocationTargetException e) { + } catch (ExecutionException e) { assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); } } @@ -1007,8 +1005,8 @@ public void testMaxSameAddressConsumers() throws Exception { PersistentSubscription sub1 = new PersistentSubscription(topic, "sub1", cursorMock, false); PersistentSubscription sub2 = new PersistentSubscription(topic, "sub2", cursorMock, false); - InetAddress address1 = InetAddress.getLoopbackAddress(); - InetAddress address2 = InetAddress.getLocalHost(); + InetAddress address1 = InetAddress.getByName("127.0.0.1"); + InetAddress address2 = InetAddress.getByName("0.0.0.0"); String ipAddress1 = address1.getHostAddress(); String ipAddress2 = address2.getHostAddress(); @@ -1026,14 +1024,14 @@ public void testMaxSameAddressConsumers() throws Exception { // 1. add consumer1 with ipAddress1 to sub1 Consumer consumer1 = getMockedConsumerWithSpecificAddress(topic, sub1, 1, address1); - addConsumerToSubscription.invoke(topic, sub1, consumer1); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub1, consumer1)).get(); assertEquals(topic.getNumberOfConsumers(), 1); assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress1), 1); assertEquals(sub1.getNumberOfSameAddressConsumers(ipAddress1), 1); // 2. add consumer2 with ipAddress1 to sub2 Consumer consumer2 = getMockedConsumerWithSpecificAddress(topic, sub2, 2, address1); - addConsumerToSubscription.invoke(topic, sub2, consumer2); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer2)).get(); assertEquals(topic.getNumberOfConsumers(), 2); assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress1), 2); assertEquals(sub1.getNumberOfSameAddressConsumers(ipAddress1), 1); @@ -1041,7 +1039,7 @@ public void testMaxSameAddressConsumers() throws Exception { // 3. add consumer3 with ipAddress2 to sub1 Consumer consumer3 = getMockedConsumerWithSpecificAddress(topic, sub1, 3, address2); - addConsumerToSubscription.invoke(topic, sub1, consumer3); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub1, consumer3)).get(); assertEquals(topic.getNumberOfConsumers(), 3); assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress1), 2); assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress2), 1); @@ -1050,7 +1048,7 @@ public void testMaxSameAddressConsumers() throws Exception { // 4. add consumer4 with ipAddress2 to sub2 Consumer consumer4 = getMockedConsumerWithSpecificAddress(topic, sub2, 4, address2); - addConsumerToSubscription.invoke(topic, sub2, consumer4); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer4)).get(); assertEquals(topic.getNumberOfConsumers(), 4); assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress1), 2); assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress2), 2); @@ -1060,9 +1058,10 @@ public void testMaxSameAddressConsumers() throws Exception { // 5. add consumer5 with ipAddress1 to sub1 but reach maxSameAddressConsumersPerTopic try { Consumer consumer5 = getMockedConsumerWithSpecificAddress(topic, sub1, 5, address1); - addConsumerToSubscription.invoke(topic, sub1, consumer5); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub1, consumer5)).get(); + fail("should have failed"); - } catch (InvocationTargetException e) { + } catch (ExecutionException e) { assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); } assertEquals(topic.getNumberOfConsumers(), 4); @@ -1072,9 +1071,9 @@ public void testMaxSameAddressConsumers() throws Exception { // 6. add consumer6 with ipAddress2 to sub2 but reach maxSameAddressConsumersPerTopic try { Consumer consumer6 = getMockedConsumerWithSpecificAddress(topic, sub2, 6, address2); - addConsumerToSubscription.invoke(topic, sub2, consumer6); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer6)).get(); fail("should have failed"); - } catch (InvocationTargetException e) { + } catch (ExecutionException e) { assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); } assertEquals(topic.getNumberOfConsumers(), 4); From 3c2ae9b1cb00d6c8c8f180647cfbf5c896e1cb7d Mon Sep 17 00:00:00 2001 From: congbo Date: Sun, 25 Apr 2021 22:09:30 +0800 Subject: [PATCH 15/18] Fix some catch exception. --- .../java/org/apache/pulsar/broker/PulsarService.java | 2 -- .../broker/service/persistent/PersistentTopic.java | 11 ++++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index c3a4e5cd59e99..2b49329efbe82 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -480,8 +480,6 @@ public CompletableFuture closeAsync() { transactionReplayExecutor.shutdown(); } - state = State.Closed; - isClosedCondition.signalAll(); // add timeout handling for closing executors asyncCloseFutures.add(executorServicesShutdown.handle()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 8b8a6463e3275..b918c6ca77d4c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -738,6 +738,7 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs decrementUsageCount(); future.completeExceptionally(e); + return; } if (log.isDebugEnabled()) { log.debug("[{}] [{}] [{}] Subscribe failed -- count: {}", topic, subscriptionName, @@ -753,10 +754,10 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs future.complete(consumer); } }).exceptionally(e -> { - if (e instanceof ConsumerBusyException) { + if (e.getCause() instanceof ConsumerBusyException) { log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, consumerName); - } else if (e instanceof SubscriptionBusyException) { + } else if (e.getCause() instanceof SubscriptionBusyException) { log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage()); } @@ -767,7 +768,11 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs }).exceptionally(ex -> { log.error("[{}] Failed to create subscription: {} error: {}", topic, subscriptionName, ex); decrementUsageCount(); - future.completeExceptionally(new PersistenceException(ex)); + if (ex.getCause() instanceof NotAllowedException) { + future.completeExceptionally(ex.getCause()); + } else { + future.completeExceptionally(new PersistenceException(ex)); + } return null; }); From f9ac2134732f361abea73086e44a6b335646fe8d Mon Sep 17 00:00:00 2001 From: congbo Date: Mon, 10 May 2021 09:29:12 +0800 Subject: [PATCH 16/18] Merge master and add some log --- .../pulsar/broker/TransactionMetadataStoreService.java | 8 ++------ .../java/org/apache/pulsar/broker/service/ServerCnx.java | 2 +- .../pulsar/broker/service/persistent/PersistentTopic.java | 4 +++- .../transaction/buffer/impl/TopicTransactionBuffer.java | 3 ++- .../buffer/impl/TransactionBufferHandlerImpl.java | 2 +- .../transaction/pendingack/impl/MLPendingAckStore.java | 7 ++++--- .../transaction/pendingack/impl/PendingAckHandleImpl.java | 6 +++--- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 552e9a8462549..450dcbb63f8f3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -399,17 +399,13 @@ private CompletableFuture endTxnInTransactionBuffer(TxnID txnID, int txnAc } private static boolean isRetryableException(Throwable e) { - if (e instanceof TransactionMetadataStoreStateException + return e instanceof TransactionMetadataStoreStateException || e instanceof RequestTimeoutException || e instanceof ManagedLedgerException || e instanceof BrokerPersistenceException || e instanceof LookupException || e instanceof ReachMaxPendingOpsException - || e instanceof ConnectException) { - return true; - } else { - return false; - } + || e instanceof ConnectException; } private CompletableFuture endTxnInTransactionMetadataStore(TxnID txnID, int txnAction) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 40a59486831f8..5340d1f23f1da 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1912,7 +1912,7 @@ protected void handleEndTxnOnPartition(CommandEndTxnOnPartition command) { if (topicFuture != null) { topicFuture.whenComplete((optionalTopic, t) -> { if (!optionalTopic.isPresent()) { - log.error("handleEndTxnOnPartition faile ! The topic {} does not exist in broker, " + log.error("handleEndTxnOnPartition fail ! The topic {} does not exist in broker, " + "txnId: [{}], txnAction: [{}]", topic, txnID, TxnAction.valueOf(txnAction)); ctx.writeAndFlush(Commands.newEndTxnOnPartitionResponse( requestId, ServerError.ServiceNotReady, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index c35126144b432..0ff722ac7bd9a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -103,6 +103,7 @@ import org.apache.pulsar.broker.stats.ReplicationMetrics; import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferDisable; +import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.admin.OffloadProcessStatus; import org.apache.pulsar.client.api.MessageId; @@ -308,7 +309,8 @@ public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerS if (brokerService.getPulsar().getConfiguration().isTransactionCoordinatorEnabled() && !checkTopicIsEventsNames(topic) && !topicName.getEncodedLocalName().startsWith(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getLocalName()) - && !topicName.getEncodedLocalName().startsWith(MLTransactionLogImpl.TRANSACTION_LOG_PREFIX)) { + && !topicName.getEncodedLocalName().startsWith(MLTransactionLogImpl.TRANSACTION_LOG_PREFIX) + && !topicName.getEncodedLocalName().endsWith(MLPendingAckStore.PENDING_ACK_STORE_SUFFIX)) { this.transactionBuffer = brokerService.getPulsar() .getTransactionBufferProvider().newTransactionBuffer(this, transactionCompletableFuture); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 9014d5b92c701..abf71e92643a4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -178,7 +178,7 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { } private void handleTransactionMessage(TxnID txnId, Position position) { - if (!ongoingTxns.containsKey(txnId)) { + if (!ongoingTxns.containsKey(txnId) && !aborts.containsKey(txnId)) { ongoingTxns.put(txnId, (PositionImpl) position); PositionImpl firstPosition = ongoingTxns.get(ongoingTxns.firstKey()); //max read position is less than first ongoing transaction message position, so entryId -1 @@ -191,6 +191,7 @@ private void handleTransactionMessage(TxnID txnId, Position position) { public CompletableFuture openTransactionBufferReader(TxnID txnID, long startSequenceId) { return null; } + @Override public CompletableFuture commitTxn(TxnID txnID, long lowWaterMark) { if (log.isDebugEnabled()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java index 4da82a29a54bc..0c3452762f2b0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java @@ -148,9 +148,9 @@ private CompletableFuture endTxn(long requestId, String topic, ByteBuf cm } }); } catch (ExecutionException e) { + log.error("endTxn channel is not active exception", e); cache.invalidate(topic); cb.completeExceptionally(new PulsarClientException.LookupException(e.getCause().getMessage())); - pendingRequests.remove(requestId); op.recycle(); } return cb; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index 7fb48ea908828..527c3215b1f37 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -37,6 +37,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack; import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadata; @@ -205,8 +206,8 @@ private CompletableFuture appendCommon(PendingAckMetadataEntry pendingAckM @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { if (log.isDebugEnabled()) { - log.debug("[{}][{}] MLPendingAckStore message append success at {}, operation : {}", - managedLedger.getName(), ctx, position, pendingAckMetadataEntry.getPendingAckOp()); + log.debug("[{}][{}] MLPendingAckStore message append success at {} txnId: {}, operation : {}", + managedLedger.getName(), ctx, position, txnID, pendingAckMetadataEntry.getPendingAckOp()); } // store the persistent position in to memory if (pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.ABORT @@ -272,7 +273,7 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}][{}] MLPendingAckStore message append fail exception : {}, operation : {}", managedLedger.getName(), ctx, exception, pendingAckMetadataEntry.getPendingAckOp()); buf.release(); - completableFuture.completeExceptionally(exception); + completableFuture.completeExceptionally(new PersistenceException(exception)); } } , null); return completableFuture; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 440f2314357c9..dcc4377fa5021 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -220,7 +220,7 @@ && isAckSetOverlap(individualAckPositions // it will produce the wrong operation. so we append fail, // we should wait tc time out or client abort this transaction. handleIndividualAck(txnID, positions); - completableFuture.completeExceptionally(e); + completableFuture.completeExceptionally(e.getCause()); } return null; })).exceptionally(e -> { @@ -287,7 +287,7 @@ public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, // state, when wereplay it, it will produce the wrong operation. so we append fail, we should // wait tc time out or client abort this transaction. handleCumulativeAck(txnID, position); - completableFuture.completeExceptionally(e); + completableFuture.completeExceptionally(e.getCause()); return null; }) ).exceptionally(e -> { @@ -354,7 +354,7 @@ public synchronized CompletableFuture commitTxn(TxnID txnID, Map { log.error("[{}] Transaction pending ack store commit txnId : [{}] fail!", topicName, txnID, e); - commitFuture.completeExceptionally(e); + commitFuture.completeExceptionally(e.getCause()); return null; })).exceptionally(e -> { commitFuture.completeExceptionally(e); From aa50b7444daa3bc572fb926499672b55f56198e5 Mon Sep 17 00:00:00 2001 From: congbo Date: Wed, 12 May 2021 17:55:05 +0800 Subject: [PATCH 17/18] Fix some comment --- .../persistent/PersistentSubscription.java | 10 +++++----- .../persistent/PersistentSubscriptionTest.java | 2 +- .../transaction/TransactionConsumeTest.java | 8 ++++---- .../transaction/TransactionProduceTest.java | 4 ++-- .../pendingack/PendingAckInMemoryDeleteTest.java | 6 +++--- .../pendingack/PendingAckPersistentTest.java | 10 ++++------ .../client/impl/TransactionEndToEndTest.java | 15 +++++++-------- .../integration/transaction/TransactionTest.java | 4 ++-- 8 files changed, 28 insertions(+), 31 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 1e0c307b721b2..04e6c9b83b440 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -133,11 +133,11 @@ public PersistentSubscription(PersistentTopic topic, String subscriptionName, Ma this.fullName = MoreObjects.toStringHelper(this).add("topic", topicName).add("name", subName).toString(); this.expiryMonitor = new PersistentMessageExpiryMonitor(topicName, subscriptionName, cursor, this); this.setReplicated(replicated); - if (topic.getBrokerService().getPulsar().getConfig().isTransactionCoordinatorEnabled() - && !checkTopicIsEventsNames(topic.getName()) - && !topic.getName().contains(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getLocalName()) - && !topic.getName().contains(MLPendingAckStore.PENDING_ACK_STORE_SUFFIX) - && !topic.getName().contains(MLTransactionLogImpl.TRANSACTION_LOG_PREFIX)) { + if (topic.getBrokerService().getPulsar().getConfiguration().isTransactionCoordinatorEnabled() + && !checkTopicIsEventsNames(topicName) + && !topicName.startsWith(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getLocalName()) + && !topicName.startsWith(MLTransactionLogImpl.TRANSACTION_LOG_PREFIX) + && !topicName.endsWith(MLPendingAckStore.PENDING_ACK_STORE_SUFFIX)) { this.pendingAckHandle = new PendingAckHandleImpl(this); } else { this.pendingAckHandle = new PendingAckHandleDisabled(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index 8811a78c68722..ba4e14d7422f0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -284,7 +284,7 @@ public void testCanAcknowledgeAndAbortForTransaction() throws BrokerServiceExcep }).when(cursorMock).asyncDelete(any(List.class), any(AsyncCallbacks.DeleteCallback.class), any()); doReturn(CommandSubscribe.SubType.Exclusive).when(consumerMock).subType(); - Awaitility.await().atMost(3, TimeUnit.SECONDS).until(() -> { + Awaitility.await().until(() -> { try { persistentSubscription.addConsumer(consumerMock); return true; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java index 1d5033c66ae92..644ea9e3326dc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java @@ -106,8 +106,8 @@ public void noSortedTest() throws Exception { .subscriptionType(SubscriptionType.Shared) .subscribe(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(exclusiveConsumer::isConnected); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(sharedConsumer::isConnected); + Awaitility.await().until(exclusiveConsumer::isConnected); + Awaitility.await().until(sharedConsumer::isConnected); long mostSigBits = 2L; long leastSigBits = 5L; @@ -184,8 +184,8 @@ public void sortedTest() throws Exception { .subscriptionName("shared-test") .subscriptionType(SubscriptionType.Shared) .subscribe(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(exclusiveConsumer::isConnected); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(sharedConsumer::isConnected); + Awaitility.await().until(exclusiveConsumer::isConnected); + Awaitility.await().until(sharedConsumer::isConnected); long mostSigBits = 2L; long leastSigBits = 5L; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java index f09bcd2aea008..a5bae35d6ec48 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java @@ -280,7 +280,7 @@ public void ackCommitTest() throws Exception { .subscriptionType(SubscriptionType.Shared) .subscribe(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + Awaitility.await().until(consumer::isConnected); for (int i = 0; i < incomingMessageCnt; i++) { Message message = consumer.receive(); @@ -343,7 +343,7 @@ public void ackAbortTest() throws Exception { .enableBatchIndexAcknowledgment(true) .subscriptionType(SubscriptionType.Shared) .subscribe(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + Awaitility.await().until(consumer::isConnected); for (int i = 0; i < incomingMessageCnt; i++) { Message message = consumer.receive(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java index d6453688ee89e..db7d1a81da81e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java @@ -290,7 +290,7 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { (ConcurrentSkipListMap) field.get(managedCursor); if (retryCnt == 0) { //one message are not ack - Awaitility.await().atMost(2000, TimeUnit.MILLISECONDS).until(() -> { + Awaitility.await().until(() -> { return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 1; }); @@ -298,7 +298,7 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1); } else { //two message are not ack - Awaitility.await().atMost(2000, TimeUnit.MILLISECONDS).until(() -> { + Awaitility.await().until(() -> { return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 2; }); @@ -306,7 +306,7 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { //this message is in one batch point consumer.acknowledge(messageIds[0]); - Awaitility.await().atMost(2000, TimeUnit.MILLISECONDS).until(() -> { + Awaitility.await().until(() -> { return batchDeletedIndexes.size() == 1; }); assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index 8b31b22c56c8e..da2442675b5d5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -130,7 +130,7 @@ public void individualPendingAckReplayTest() throws Exception { //in order to test pending ack replay admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + Awaitility.await().until(consumer::isConnected); Transaction commitTxn = pulsarClient.newTransaction() .withTransactionTimeout(30, TimeUnit.SECONDS).build().get(); @@ -159,7 +159,7 @@ public void individualPendingAckReplayTest() throws Exception { // replay this pending ack admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); - Awaitility.await().atMost(4000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + Awaitility.await().until(consumer::isConnected); abortTxn = pulsarClient.newTransaction() .withTransactionTimeout(30, TimeUnit.SECONDS).build().get(); @@ -203,7 +203,6 @@ public void individualPendingAckReplayTest() throws Exception { // in order to check out the pending ack cursor is clear whether or not. Awaitility.await() - .atMost(15000, TimeUnit.MILLISECONDS) .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition()) .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == -1); } @@ -246,7 +245,7 @@ public void cumulativePendingAckReplayTest() throws Exception { Transaction txn = pulsarClient.newTransaction() .withTransactionTimeout(30, TimeUnit.SECONDS).build().get(); - Awaitility.await().atMost(1000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + Awaitility.await().until(consumer::isConnected); for (int i = 0; i < pendingAckMessageIds.size(); i++) { try { @@ -266,7 +265,7 @@ public void cumulativePendingAckReplayTest() throws Exception { commitTxn.commit().get(); admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + Awaitility.await().until(consumer::isConnected); for (int i = 0; i < pendingAckMessageIds.size(); i++) { try { @@ -296,7 +295,6 @@ public void cumulativePendingAckReplayTest() throws Exception { // in order to check out the pending ack cursor is clear whether or not. Awaitility.await() - .atMost(15000, TimeUnit.MILLISECONDS) .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition()) .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == -1); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 31df14ce9a6ec..58d87f6db6943 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -46,7 +46,6 @@ import org.apache.pulsar.broker.transaction.TransactionTestBase; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.PulsarClient; @@ -142,7 +141,7 @@ private void produceCommitTest(boolean enableBatch) throws Exception { .subscriptionName("test") .enableBatchIndexAcknowledgment(true) .subscribe(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + Awaitility.await().until(consumer::isConnected); ProducerBuilder producerBuilder = pulsarClient .newProducer() @@ -216,7 +215,7 @@ public void produceAbortTest() throws Exception { .subscriptionName(subName) .enableBatchIndexAcknowledgment(true) .subscribe(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + Awaitility.await().until(consumer::isConnected); // Can't receive transaction messages before abort. Message message = consumer.receive(2, TimeUnit.SECONDS); @@ -307,7 +306,7 @@ private void txnAckTest(boolean batchEnable, int maxBatchSize, .enableBatchIndexAcknowledgment(true) .subscriptionType(subscriptionType) .subscribe(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + Awaitility.await().until(consumer::isConnected); @Cleanup Producer producer = pulsarClient.newProducer() @@ -383,7 +382,7 @@ public void txnMessageAckTest() throws Exception { .enableBatchIndexAcknowledgment(true) .acknowledgmentGroupTime(0, TimeUnit.MILLISECONDS) .subscribe(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + Awaitility.await().until(consumer::isConnected); @Cleanup Producer producer = pulsarClient @@ -507,7 +506,7 @@ private void txnCumulativeAckTest(boolean batchEnable, int maxBatchSize, Subscri .subscriptionType(subscriptionType) .ackTimeout(1, TimeUnit.MINUTES) .subscribe(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + Awaitility.await().until(consumer::isConnected); @Cleanup Producer producer = pulsarClient.newProducer() @@ -654,7 +653,7 @@ public void txnMetadataHandlerRecoverTest() throws Exception { .subscriptionName("test") .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscribe(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + Awaitility.await().until(consumer::isConnected); for (int i = 0; i < txnCnt * messageCnt; i++) { Message message = consumer.receive(); @@ -671,7 +670,7 @@ public void produceTxnMessageOrderTest() throws Exception { .topic(topic) .subscriptionName("test") .subscribe(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(consumer::isConnected); + Awaitility.await().until(consumer::isConnected); @Cleanup Producer producer = pulsarClient.newProducer() diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java index fe143218ef205..20110cd544ae5 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java @@ -100,7 +100,7 @@ public void transferNormalTest(Supplier serviceUrl) throws Exception { .subscriptionType(SubscriptionType.Shared) .enableBatchIndexAcknowledgment(true) .subscribe(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(transferConsumer::isConnected); + Awaitility.await().until(transferConsumer::isConnected); log.info("transfer consumer create finished"); @Cleanup @@ -116,7 +116,7 @@ public void transferNormalTest(Supplier serviceUrl) throws Exception { .subscriptionName("integration-test") .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscribe(); - Awaitility.await().atMost(3000, TimeUnit.MILLISECONDS).until(balanceUpdateConsumer::isConnected); + Awaitility.await().until(balanceUpdateConsumer::isConnected); log.info("balance update consumer create finished"); while (true) { From 9a58f9c0e3167493b0d610b293932175514dffe8 Mon Sep 17 00:00:00 2001 From: congbo Date: Wed, 12 May 2021 20:39:30 +0800 Subject: [PATCH 18/18] Fix some comment --- .../service/nonpersistent/NonPersistentTopic.java | 15 +++++++++------ .../persistent/PersistentSubscription.java | 2 +- .../pendingack/impl/MLPendingAckStore.java | 4 ++-- .../service/TransactionMarkerDeleteTest.java | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 63b16fd9bcbf8..50a50720bfc46 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -267,7 +267,7 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs name -> new NonPersistentSubscription(this, subscriptionName)); Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, 0, cnx, cnx.getAuthRole(), metadata, readCompacted, initialPosition, keySharedMeta); - addConsumerToSubscription(subscription, consumer).thenAccept(v -> { + addConsumerToSubscription(subscription, consumer).thenRun(() -> { if (!cnx.isActive()) { try { consumer.close(); @@ -281,6 +281,7 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs decrementUsageCount(); future.completeExceptionally(e); + return; } if (log.isDebugEnabled()) { log.debug("[{}] [{}] [{}] Subscribe failed -- count: {}", topic, subscriptionName, @@ -293,15 +294,16 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs future.complete(consumer); } }).exceptionally(e -> { - if (e.getCause() instanceof ConsumerBusyException) { + Throwable throwable = e.getCause(); + if (throwable instanceof ConsumerBusyException) { log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, consumerName); - } else if (e.getCause() instanceof SubscriptionBusyException) { + } else if (throwable instanceof SubscriptionBusyException) { log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage()); } decrementUsageCount(); - future.completeExceptionally(e.getCause()); + future.completeExceptionally(throwable); return null; }); @@ -854,11 +856,12 @@ public void checkGC() { stopReplProducers().thenCompose(v -> delete(true, false, true)) .thenRun(() -> log.info("[{}] Topic deleted successfully due to inactivity", topic)) .exceptionally(e -> { - if (e.getCause() instanceof TopicBusyException) { + Throwable throwable = e.getCause(); + if (throwable instanceof TopicBusyException) { // topic became active again if (log.isDebugEnabled()) { log.debug("[{}] Did not delete busy topic: {}", topic, - e.getCause().getMessage()); + throwable.getMessage()); } replicators.forEach((region, replicator) -> replicator.startProducer()); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 04e6c9b83b440..320fdc615ff0a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -133,7 +133,7 @@ public PersistentSubscription(PersistentTopic topic, String subscriptionName, Ma this.fullName = MoreObjects.toStringHelper(this).add("topic", topicName).add("name", subName).toString(); this.expiryMonitor = new PersistentMessageExpiryMonitor(topicName, subscriptionName, cursor, this); this.setReplicated(replicated); - if (topic.getBrokerService().getPulsar().getConfiguration().isTransactionCoordinatorEnabled() + if (topic.getBrokerService().getPulsar().getConfig().isTransactionCoordinatorEnabled() && !checkTopicIsEventsNames(topicName) && !topicName.startsWith(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getLocalName()) && !topicName.startsWith(MLTransactionLogImpl.TRANSACTION_LOG_PREFIX) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index 527c3215b1f37..14aaa5feb3e26 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -62,9 +62,9 @@ public class MLPendingAckStore implements PendingAckStore { private final ManagedCursor cursor; - public static final String PENDING_ACK_STORE_SUFFIX = "__transaction_pendingack"; + public static final String PENDING_ACK_STORE_SUFFIX = "__transaction_pending_ack"; - private static final String PENDING_ACK_STORE_CURSOR_NAME = "pendingack"; + private static final String PENDING_ACK_STORE_CURSOR_NAME = "__pending_ack_state"; private final SpscArrayQueue entryQueue; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java index b8db67838e7e3..65b6bd945a32d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java @@ -102,7 +102,7 @@ public void testMarkerDeleteTimes() throws Exception { doReturn(brokerService).when(topic).getBrokerService(); doReturn(pulsarService).when(brokerService).getPulsar(); doReturn(configuration).when(pulsarService).getConfig(); - doReturn(true).when(configuration).isTransactionCoordinatorEnabled(); + doReturn(false).when(configuration).isTransactionCoordinatorEnabled(); doReturn(managedLedger).when(topic).getManagedLedger(); ManagedCursor cursor = managedLedger.openCursor("test"); PersistentSubscription persistentSubscription = spy(new PersistentSubscription(topic, "test",