diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java index be898f3465a44..03419dee14d8b 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java @@ -392,6 +392,13 @@ void markDelete(Position position, Map properties) */ Position getMarkDeletedPosition(); + /** + * Get the persistent newest mark deleted position on this cursor. + * + * @return the persistent mark deleted position + */ + Position getPersistentMarkDeletedPosition(); + /** * Rewind the cursor to the mark deleted position to replay all the already read but not yet mark deleted messages. * diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index dc89dbb1a56ca..0d249a15a08ee 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -116,6 +116,9 @@ public class ManagedCursorImpl implements ManagedCursor { protected volatile PositionImpl markDeletePosition; + // this position is have persistent mark delete position + protected volatile PositionImpl persistentMarkDeletePosition; + protected static final AtomicReferenceFieldUpdater READ_POSITION_UPDATER = AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, PositionImpl.class, "readPosition"); protected volatile PositionImpl readPosition; @@ -500,6 +503,7 @@ private void recoveredCursor(PositionImpl position, Map properties messagesConsumedCounter = -getNumberOfEntries(Range.openClosed(position, ledger.getLastPosition())); markDeletePosition = position; + persistentMarkDeletePosition = position; readPosition = ledger.getNextValidPosition(position); lastMarkDeleteEntry = new MarkDeleteEntry(markDeletePosition, properties, null, null); // assign cursor-ledger so, it can be deleted when new ledger will be switched @@ -576,7 +580,8 @@ public void asyncReadEntries(int numberOfEntriesToRead, long maxSizeBytes, ReadE Object ctx, PositionImpl maxPosition) { checkArgument(numberOfEntriesToRead > 0); if (isClosed()) { - callback.readEntriesFailed(new ManagedLedgerException("Cursor was already closed"), ctx); + callback.readEntriesFailed(new ManagedLedgerException + .CursorAlreadyClosedException("Cursor was already closed"), ctx); return; } @@ -628,7 +633,8 @@ public void asyncGetNthEntry(int n, IndividualDeletedEntries deletedEntries, Rea Object ctx) { checkArgument(n > 0); if (isClosed()) { - callback.readEntryFailed(new ManagedLedgerException("Cursor was already closed"), ctx); + callback.readEntryFailed(new ManagedLedgerException + .CursorAlreadyClosedException("Cursor was already closed"), ctx); return; } @@ -781,7 +787,7 @@ public void asyncReadEntriesOrWait(int maxEntries, long maxSizeBytes, ReadEntrie } } - private boolean isClosed() { + public boolean isClosed() { return state == State.Closed || state == State.Closing; } @@ -1597,7 +1603,8 @@ public void asyncMarkDelete(final Position position, Map propertie checkArgument(position instanceof PositionImpl); if (isClosed()) { - callback.markDeleteFailed(new ManagedLedgerException("Cursor was already closed"), ctx); + callback.markDeleteFailed(new ManagedLedgerException + .CursorAlreadyClosedException("Cursor was already closed"), ctx); return; } @@ -1673,7 +1680,8 @@ protected void internalAsyncMarkDelete(final PositionImpl newPosition, Map 0) { + persistentMarkDeletePosition = mdEntry.newPosition; + } + } finally { lock.writeLock().unlock(); } @@ -1835,7 +1848,8 @@ public void deleteFailed(ManagedLedgerException exception, Object ctx) { @Override public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallback callback, Object ctx) { if (isClosed()) { - callback.deleteFailed(new ManagedLedgerException("Cursor was already closed"), ctx); + callback.deleteFailed(new ManagedLedgerException + .CursorAlreadyClosedException("Cursor was already closed"), ctx); return; } @@ -2064,6 +2078,11 @@ public Position getMarkDeletedPosition() { return markDeletePosition; } + @Override + public Position getPersistentMarkDeletedPosition() { + return this.persistentMarkDeletePosition; + } + @Override public void rewind() { lock.writeLock().lock(); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java index 856a7e8b1248d..0f6386d1fe1db 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java @@ -161,7 +161,7 @@ public void run() { Thread.sleep(1); } } catch (ManagedLedgerException e) { - if (!e.getMessage().equals("Cursor was already closed")) { + if (!(e instanceof ManagedLedgerException.CursorAlreadyClosedException)) { gotException.set(true); } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java index baf1c9f06553b..549027827efdd 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java @@ -130,6 +130,11 @@ public Position getMarkDeletedPosition() { return position; } + @Override + public Position getPersistentMarkDeletedPosition() { + return position; + } + @Override public String getName() { return name; diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 8615e58e3c205..a09077fe20792 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -221,13 +221,6 @@ public class ServiceConfiguration implements PulsarConfiguration { ) private int numCacheExecutorThreadPoolSize = 10; - @FieldContext( - category = CATEGORY_SERVER, - doc = "Number of threads to use for pulsar broker service." - + " The executor in thread pool will do transaction recover" - ) - private int numTransactionExecutorThreadPoolSize = Runtime.getRuntime().availableProcessors(); - @FieldContext(category = CATEGORY_SERVER, doc = "Max concurrent web requests") private int maxConcurrentHttpRequests = 1024; @@ -2039,6 +2032,20 @@ public class ServiceConfiguration implements PulsarConfiguration { private String transactionBufferProviderClassName = "org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferProvider"; + @FieldContext( + category = CATEGORY_TRANSACTION, + doc = "Class name for transaction pending ack store provider" + ) + private String transactionPendingAckStoreProviderClassName = + "org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStoreProvider"; + + @FieldContext( + category = CATEGORY_TRANSACTION, + doc = "Number of threads to use for pulsar transaction replay PendingAckStore or TransactionBuffer." + + "Default is 5" + ) + private int numTransactionReplayThreadPoolSize = Runtime.getRuntime().availableProcessors(); + @FieldContext( category = CATEGORY_TRANSACTION, doc = "Transaction buffer take snapshot transaction count" diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 5e924abfde654..b752f42fd1fe4 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -477,7 +477,10 @@ com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} true - **/ResourceUsage.proto + + **/ResourceUsage.proto + **/TransactionPendingAck.proto + @@ -495,7 +498,10 @@ lightproto-maven-plugin ${lightproto-maven-plugin.version} - ${project.basedir}/src/main/proto/ResourceUsage.proto + + ${project.basedir}/src/main/proto/TransactionPendingAck.proto + ${project.basedir}/src/main/proto/ResourceUsage.proto + generated-sources/lightproto/java generated-sources/lightproto/java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 51fba117fb4cf..6ed22f5304416 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -106,6 +106,7 @@ import org.apache.pulsar.broker.storage.ManagedLedgerStorage; import org.apache.pulsar.broker.transaction.buffer.TransactionBufferProvider; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferClientImpl; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; import org.apache.pulsar.broker.validator.MultipleListenerValidator; import org.apache.pulsar.broker.web.WebService; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -223,7 +224,6 @@ public class PulsarService implements AutoCloseable { private TransactionMetadataStoreService transactionMetadataStoreService; private TransactionBufferProvider transactionBufferProvider; private TransactionBufferClient transactionBufferClient; - private ScheduledExecutorService transactionExecutor; private HashedWheelTimer transactionTimer; private BrokerInterceptor brokerInterceptor; @@ -240,6 +240,9 @@ public class PulsarService implements AutoCloseable { private MetadataStoreExtended configurationMetadataStore; private PulsarResources pulsarResources; + private TransactionPendingAckStoreProvider transactionPendingAckStoreProvider; + private final ScheduledExecutorService transactionReplayExecutor; + public enum State { Init, Started, Closing, Closed } @@ -287,6 +290,14 @@ public PulsarService(ServiceConfiguration config, new DefaultThreadFactory("pulsar")); this.cacheExecutor = Executors.newScheduledThreadPool(config.getNumCacheExecutorThreadPoolSize(), new DefaultThreadFactory("zk-cache-callback")); + + if (config.isTransactionCoordinatorEnabled()) { + this.transactionReplayExecutor = Executors.newScheduledThreadPool( + config.getNumTransactionReplayThreadPoolSize(), + new DefaultThreadFactory("transaction-replay")); + } else { + this.transactionReplayExecutor = null; + } } public MetadataStoreExtended createConfigurationMetadataStore() throws MetadataStoreException { @@ -445,8 +456,6 @@ public CompletableFuture closeAsync() { transactionBufferClient.close(); } - executorServicesShutdown.shutdown(transactionExecutor); - if (coordinationService != null) { coordinationService.close(); } @@ -458,6 +467,10 @@ public CompletableFuture closeAsync() { configurationMetadataStore.close(); } + if (transactionReplayExecutor != null) { + transactionReplayExecutor.shutdown(); + } + // add timeout handling for closing executors asyncCloseFutures.add(executorServicesShutdown.handle()); @@ -729,9 +742,6 @@ public Boolean get() { // Register pulsar system namespaces and start transaction meta store service if (config.isTransactionCoordinatorEnabled()) { - this.transactionExecutor = Executors.newScheduledThreadPool( - config.getNumTransactionExecutorThreadPoolSize(), - new DefaultThreadFactory("pulsar-transaction")); this.transactionBufferSnapshotService = new SystemTopicBaseTxnBufferSnapshotService(getClient()); this.transactionTimer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-transaction-timer")); @@ -744,6 +754,8 @@ public Boolean get() { transactionBufferProvider = TransactionBufferProvider .newProvider(config.getTransactionBufferProviderClassName()); + transactionPendingAckStoreProvider = TransactionPendingAckStoreProvider + .newProvider(config.getTransactionPendingAckStoreProviderClassName()); } this.metricsGenerator = new MetricsGenerator(this); @@ -1170,6 +1182,10 @@ public ScheduledExecutorService getCacheExecutor() { return cacheExecutor; } + public ScheduledExecutorService getTransactionReplayExecutor() { + return transactionReplayExecutor; + } + public ScheduledExecutorService getLoadManagerExecutor() { return loadManagerExecutor; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 552e9a8462549..450dcbb63f8f3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -399,17 +399,13 @@ private CompletableFuture endTxnInTransactionBuffer(TxnID txnID, int txnAc } private static boolean isRetryableException(Throwable e) { - if (e instanceof TransactionMetadataStoreStateException + return e instanceof TransactionMetadataStoreStateException || e instanceof RequestTimeoutException || e instanceof ManagedLedgerException || e instanceof BrokerPersistenceException || e instanceof LookupException || e instanceof ReachMaxPendingOpsException - || e instanceof ConnectException) { - return true; - } else { - return false; - } + || e instanceof ConnectException; } private CompletableFuture endTxnInTransactionMetadataStore(TxnID txnID, int txnAction) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 23178b357820b..fafd098670171 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -253,19 +253,18 @@ protected int getNumberOfSameAddressConsumers(final String clientAddress, return count; } - protected void addConsumerToSubscription(Subscription subscription, Consumer consumer) - throws BrokerServiceException { + protected CompletableFuture addConsumerToSubscription(Subscription subscription, Consumer consumer) { if (isConsumersExceededOnTopic()) { log.warn("[{}] Attempting to add consumer to topic which reached max consumers limit", topic); - throw new ConsumerBusyException("Topic reached max consumers limit"); + return FutureUtil.failedFuture(new ConsumerBusyException("Topic reached max consumers limit")); } if (isSameAddressConsumersExceededOnTopic(consumer)) { log.warn("[{}] Attempting to add consumer to topic which reached max same address consumers limit", topic); - throw new ConsumerBusyException("Topic reached max same address consumers limit"); + return FutureUtil.failedFuture(new ConsumerBusyException("Topic reached max same address consumers limit")); } - subscription.addConsumer(consumer); + return subscription.addConsumer(consumer); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 118d11b8f8746..f905cb52fd377 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -128,7 +128,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo int priorityLevel, String consumerName, int maxUnackedMessages, TransportCnx cnx, String appId, Map metadata, boolean readCompacted, InitialPosition subscriptionInitialPosition, - KeySharedMeta keySharedMeta) throws BrokerServiceException { + KeySharedMeta keySharedMeta) { this.subscription = subscription; this.subType = subType; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index c03e6fc233c60..8320fc83e6440 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1912,7 +1912,7 @@ protected void handleEndTxnOnPartition(CommandEndTxnOnPartition command) { if (topicFuture != null) { topicFuture.whenComplete((optionalTopic, t) -> { if (!optionalTopic.isPresent()) { - log.error("handleEndTxnOnPartition faile ! The topic {} does not exist in broker, " + log.error("handleEndTxnOnPartition fail ! The topic {} does not exist in broker, " + "txnId: [{}], txnAction: [{}]", topic, txnID, TxnAction.valueOf(txnAction)); ctx.writeAndFlush(Commands.newEndTxnOnPartitionResponse( requestId, ServerError.ServiceNotReady, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java index 997831e19cb33..555b10db34474 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Subscription.java @@ -37,7 +37,7 @@ public interface Subscription { String getName(); - void addConsumer(Consumer consumer) throws BrokerServiceException; + CompletableFuture addConsumer(Consumer consumer); default void removeConsumer(Consumer consumer) throws BrokerServiceException { removeConsumer(consumer, false); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java index 7cce496e10791..0fb05c654c7fd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentSubscription.java @@ -46,6 +46,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ConsumerStats; import org.apache.pulsar.common.policies.data.NonPersistentSubscriptionStats; +import org.apache.pulsar.common.util.FutureUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,11 +98,11 @@ public boolean isReplicated() { } @Override - public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { + public synchronized CompletableFuture addConsumer(Consumer consumer) { updateLastActive(); if (IS_FENCED_UPDATER.get(this) == TRUE) { log.warn("Attempting to add consumer {} on a fenced subscription", consumer); - throw new SubscriptionFencedException("Subscription is fenced"); + return FutureUtil.failedFuture(new SubscriptionFencedException("Subscription is fenced")); } if (dispatcher == null || !dispatcher.isConsumerConnected()) { @@ -160,7 +161,7 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce } break; default: - throw new ServerMetadataException("Unsupported subscription type"); + return FutureUtil.failedFuture(new ServerMetadataException("Unsupported subscription type")); } if (previousDispatcher != null) { @@ -173,11 +174,16 @@ public synchronized void addConsumer(Consumer consumer) throws BrokerServiceExce } } else { if (consumer.subType() != dispatcher.getType()) { - throw new SubscriptionBusyException("Subscription is of different type"); + return FutureUtil.failedFuture(new SubscriptionBusyException("Subscription is of different type")); } } - dispatcher.addConsumer(consumer); + try { + dispatcher.addConsumer(consumer); + return CompletableFuture.completedFuture(null); + } catch (BrokerServiceException brokerServiceException) { + return FutureUtil.failedFuture(brokerServiceException); + } } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 54a35b5d627bd..50a50720bfc46 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -265,13 +265,24 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs NonPersistentSubscription subscription = subscriptions.computeIfAbsent(subscriptionName, name -> new NonPersistentSubscription(this, subscriptionName)); - - try { - Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, 0, - cnx, cnx.getAuthRole(), metadata, readCompacted, initialPosition, keySharedMeta); - addConsumerToSubscription(subscription, consumer); + Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, 0, + cnx, cnx.getAuthRole(), metadata, readCompacted, initialPosition, keySharedMeta); + addConsumerToSubscription(subscription, consumer).thenRun(() -> { if (!cnx.isActive()) { - consumer.close(); + try { + consumer.close(); + } catch (BrokerServiceException e) { + if (e instanceof ConsumerBusyException) { + log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, + consumerName); + } else if (e instanceof SubscriptionBusyException) { + log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage()); + } + + decrementUsageCount(); + future.completeExceptionally(e); + return; + } if (log.isDebugEnabled()) { log.debug("[{}] [{}] [{}] Subscribe failed -- count: {}", topic, subscriptionName, consumer.consumerName(), currentUsageCount()); @@ -282,17 +293,19 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs log.info("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId); future.complete(consumer); } - } catch (BrokerServiceException e) { - if (e instanceof ConsumerBusyException) { + }).exceptionally(e -> { + Throwable throwable = e.getCause(); + if (throwable instanceof ConsumerBusyException) { log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, consumerName); - } else if (e instanceof SubscriptionBusyException) { + } else if (throwable instanceof SubscriptionBusyException) { log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage()); } decrementUsageCount(); - future.completeExceptionally(e); - } + future.completeExceptionally(throwable); + return null; + }); return future; } @@ -843,11 +856,12 @@ public void checkGC() { stopReplProducers().thenCompose(v -> delete(true, false, true)) .thenRun(() -> log.info("[{}] Topic deleted successfully due to inactivity", topic)) .exceptionally(e -> { - if (e.getCause() instanceof TopicBusyException) { + Throwable throwable = e.getCause(); + if (throwable instanceof TopicBusyException) { // topic became active again if (log.isDebugEnabled()) { log.debug("[{}] Did not delete busy topic: {}", topic, - e.getCause().getMessage()); + throwable.getMessage()); } replicators.forEach((region, replicator) -> replicator.startProducer()); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 66c666b98ce4c..320fdc615ff0a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service.persistent; +import static org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsEventsNames; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import java.util.Collections; @@ -56,6 +57,7 @@ import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.transaction.pendingack.PendingAckHandle; +import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleDisabled; import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; import org.apache.pulsar.client.api.transaction.TxnID; @@ -71,6 +73,7 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.Markers; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -130,7 +133,11 @@ public PersistentSubscription(PersistentTopic topic, String subscriptionName, Ma this.fullName = MoreObjects.toStringHelper(this).add("topic", topicName).add("name", subName).toString(); this.expiryMonitor = new PersistentMessageExpiryMonitor(topicName, subscriptionName, cursor, this); this.setReplicated(replicated); - if (topic.getBrokerService().getPulsar().getConfig().isTransactionCoordinatorEnabled()) { + if (topic.getBrokerService().getPulsar().getConfig().isTransactionCoordinatorEnabled() + && !checkTopicIsEventsNames(topicName) + && !topicName.startsWith(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getLocalName()) + && !topicName.startsWith(MLTransactionLogImpl.TRANSACTION_LOG_PREFIX) + && !topicName.endsWith(MLPendingAckStore.PENDING_ACK_STORE_SUFFIX)) { this.pendingAckHandle = new PendingAckHandleImpl(this); } else { this.pendingAckHandle = new PendingAckHandleDisabled(); @@ -172,78 +179,93 @@ void setReplicated(boolean replicated) { } @Override - public synchronized void addConsumer(Consumer consumer) throws BrokerServiceException { - cursor.updateLastActive(); - if (IS_FENCED_UPDATER.get(this) == TRUE) { - log.warn("Attempting to add consumer {} on a fenced subscription", consumer); - throw new SubscriptionFencedException("Subscription is fenced"); - } - - if (dispatcher == null || !dispatcher.isConsumerConnected()) { - Dispatcher previousDispatcher = null; - boolean useStreamingDispatcher = topic.getBrokerService().getPulsar() - .getConfiguration().isStreamingDispatch(); - switch (consumer.subType()) { - case Exclusive: - if (dispatcher == null || dispatcher.getType() != SubType.Exclusive) { - previousDispatcher = dispatcher; - dispatcher = useStreamingDispatcher ? new PersistentStreamingDispatcherSingleActiveConsumer(cursor, - SubType.Exclusive, 0, topic, this) : - new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Exclusive, 0, - topic, this); - } - break; - case Shared: - if (dispatcher == null || dispatcher.getType() != SubType.Shared) { - previousDispatcher = dispatcher; - dispatcher = useStreamingDispatcher ? new PersistentStreamingDispatcherMultipleConsumers(topic, - cursor, this) : new PersistentDispatcherMultipleConsumers(topic, - cursor, this); - } - break; - case Failover: - int partitionIndex = TopicName.getPartitionIndex(topicName); - if (partitionIndex < 0) { - // For non partition topics, use a negative index so dispatcher won't sort consumers before picking - // an active consumer for the topic. - partitionIndex = -1; + public CompletableFuture addConsumer(Consumer consumer) { + return pendingAckHandle.pendingAckHandleFuture().thenCompose(future -> { + synchronized (PersistentSubscription.this) { + cursor.updateLastActive(); + if (IS_FENCED_UPDATER.get(this) == TRUE) { + log.warn("Attempting to add consumer {} on a fenced subscription", consumer); + return FutureUtil.failedFuture(new SubscriptionFencedException("Subscription is fenced")); } - if (dispatcher == null || dispatcher.getType() != SubType.Failover) { - previousDispatcher = dispatcher; - dispatcher = useStreamingDispatcher ? new PersistentStreamingDispatcherSingleActiveConsumer(cursor, - SubType.Failover, partitionIndex, topic, this) : - new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Failover, - partitionIndex, topic, this); - } - break; - case Key_Shared: - if (dispatcher == null || dispatcher.getType() != SubType.Key_Shared) { - previousDispatcher = dispatcher; - KeySharedMeta ksm = consumer.getKeySharedMeta(); - dispatcher = new PersistentStickyKeyDispatcherMultipleConsumers(topic, cursor, this, - topic.getBrokerService().getPulsar().getConfiguration(), ksm); + if (dispatcher == null || !dispatcher.isConsumerConnected()) { + Dispatcher previousDispatcher = null; + boolean useStreamingDispatcher = topic.getBrokerService().getPulsar() + .getConfiguration().isStreamingDispatch(); + switch (consumer.subType()) { + case Exclusive: + if (dispatcher == null || dispatcher.getType() != SubType.Exclusive) { + previousDispatcher = dispatcher; + dispatcher = useStreamingDispatcher + ? new PersistentStreamingDispatcherSingleActiveConsumer( + cursor, SubType.Exclusive, 0, topic, this) + : new PersistentDispatcherSingleActiveConsumer( + cursor, SubType.Exclusive, 0, topic, this); + } + break; + case Shared: + if (dispatcher == null || dispatcher.getType() != SubType.Shared) { + previousDispatcher = dispatcher; + dispatcher = useStreamingDispatcher + ? new PersistentStreamingDispatcherMultipleConsumers( + topic, cursor, this) + : new PersistentDispatcherMultipleConsumers(topic, cursor, this); + } + break; + case Failover: + int partitionIndex = TopicName.getPartitionIndex(topicName); + if (partitionIndex < 0) { + // For non partition topics, use a negative index so + // dispatcher won't sort consumers before picking + // an active consumer for the topic. + partitionIndex = -1; + } + + if (dispatcher == null || dispatcher.getType() != SubType.Failover) { + previousDispatcher = dispatcher; + dispatcher = useStreamingDispatcher + ? new PersistentStreamingDispatcherSingleActiveConsumer( + cursor, SubType.Failover, partitionIndex, topic, this) : + new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Failover, + partitionIndex, topic, this); + } + break; + case Key_Shared: + if (dispatcher == null || dispatcher.getType() != SubType.Key_Shared) { + previousDispatcher = dispatcher; + KeySharedMeta ksm = consumer.getKeySharedMeta(); + dispatcher = new PersistentStickyKeyDispatcherMultipleConsumers(topic, cursor, this, + topic.getBrokerService().getPulsar().getConfiguration(), ksm); + } + break; + default: + return FutureUtil.failedFuture( + new ServerMetadataException("Unsupported subscription type")); + } + + if (previousDispatcher != null) { + previousDispatcher.close().thenRun(() -> { + log.info("[{}][{}] Successfully closed previous dispatcher", topicName, subName); + }).exceptionally(ex -> { + log.error("[{}][{}] Failed to close previous dispatcher", topicName, subName, ex); + return null; + }); + } + } else { + if (consumer.subType() != dispatcher.getType()) { + return FutureUtil.failedFuture( + new SubscriptionBusyException("Subscription is of different type")); + } } - break; - default: - throw new ServerMetadataException("Unsupported subscription type"); - } - if (previousDispatcher != null) { - previousDispatcher.close().thenRun(() -> { - log.info("[{}][{}] Successfully closed previous dispatcher", topicName, subName); - }).exceptionally(ex -> { - log.error("[{}][{}] Failed to close previous dispatcher", topicName, subName, ex); - return null; - }); - } - } else { - if (consumer.subType() != dispatcher.getType()) { - throw new SubscriptionBusyException("Subscription is of different type"); + try { + dispatcher.addConsumer(consumer); + return CompletableFuture.completedFuture(null); + } catch (BrokerServiceException brokerServiceException) { + return FutureUtil.failedFuture(brokerServiceException); + } } - } - - dispatcher.addConsumer(consumer); + }); } @Override @@ -750,11 +772,11 @@ public CompletableFuture close() { if (dispatcher != null && dispatcher.isConsumerConnected()) { return FutureUtil.failedFuture(new SubscriptionBusyException("Subscription has active consumers")); } - IS_FENCED_UPDATER.set(this, TRUE); - log.info("[{}][{}] Successfully closed subscription [{}]", topicName, subName, cursor); + return this.pendingAckHandle.close().thenAccept(v -> { + IS_FENCED_UPDATER.set(this, TRUE); + log.info("[{}][{}] Successfully closed subscription [{}]", topicName, subName, cursor); + }); } - - return CompletableFuture.completedFuture(null); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index fe95c8fed5d51..04d812a52a485 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -103,6 +103,7 @@ import org.apache.pulsar.broker.stats.ReplicationMetrics; import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferDisable; +import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.admin.OffloadProcessStatus; import org.apache.pulsar.client.api.MessageId; @@ -308,7 +309,8 @@ public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerS if (brokerService.getPulsar().getConfiguration().isTransactionCoordinatorEnabled() && !checkTopicIsEventsNames(topic) && !topicName.getEncodedLocalName().startsWith(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getLocalName()) - && !topicName.getEncodedLocalName().startsWith(MLTransactionLogImpl.TRANSACTION_LOG_PREFIX)) { + && !topicName.getEncodedLocalName().startsWith(MLTransactionLogImpl.TRANSACTION_LOG_PREFIX) + && !topicName.getEncodedLocalName().endsWith(MLPendingAckStore.PENDING_ACK_STORE_SUFFIX)) { this.transactionBuffer = brokerService.getPulsar() .getTransactionBufferProvider().newTransactionBuffer(this, transactionCompletableFuture); } else { @@ -724,16 +726,26 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs : 0; subscriptionFuture.thenAccept(subscription -> { - try { - Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, - maxUnackedMessages, cnx, cnx.getAuthRole(), metadata, - readCompacted, initialPosition, keySharedMeta); - addConsumerToSubscription(subscription, consumer); - + Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, + maxUnackedMessages, cnx, cnx.getAuthRole(), metadata, + readCompacted, initialPosition, keySharedMeta); + addConsumerToSubscription(subscription, consumer).thenAccept(v -> { checkBackloggedCursors(); - if (!cnx.isActive()) { - consumer.close(); + try { + consumer.close(); + } catch (BrokerServiceException e) { + if (e instanceof ConsumerBusyException) { + log.warn("[{}][{}] Consumer {} {} already connected", + topic, subscriptionName, consumerId, consumerName); + } else if (e instanceof SubscriptionBusyException) { + log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage()); + } + + decrementUsageCount(); + future.completeExceptionally(e); + return; + } if (log.isDebugEnabled()) { log.debug("[{}] [{}] [{}] Subscribe failed -- count: {}", topic, subscriptionName, consumer.consumerName(), currentUsageCount()); @@ -747,17 +759,18 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs log.info("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId); future.complete(consumer); } - } catch (BrokerServiceException e) { - if (e instanceof ConsumerBusyException) { + }).exceptionally(e -> { + if (e.getCause() instanceof ConsumerBusyException) { log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, consumerName); - } else if (e instanceof SubscriptionBusyException) { + } else if (e.getCause() instanceof SubscriptionBusyException) { log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage()); } decrementUsageCount(); future.completeExceptionally(e); - } + return null; + }); }).exceptionally(ex -> { log.error("[{}] Failed to create subscription: {} error: {}", topic, subscriptionName, ex); decrementUsageCount(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 2917ab9a7bae3..abf71e92643a4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -98,7 +98,7 @@ public TopicTransactionBuffer(PersistentTopic topic, CompletableFuture tra .getConfiguration().getTransactionBufferSnapshotMaxTransactionCount(); this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); - this.topic.getBrokerService().getPulsar().getTransactionExecutor() + this.topic.getBrokerService().getPulsar().getTransactionReplayExecutor() .execute(new TopicTransactionBufferRecover(new TopicTransactionBufferRecoverCallBack() { @Override public void recoverComplete() { @@ -178,7 +178,7 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { } private void handleTransactionMessage(TxnID txnId, Position position) { - if (!ongoingTxns.containsKey(txnId)) { + if (!ongoingTxns.containsKey(txnId) && !aborts.containsKey(txnId)) { ongoingTxns.put(txnId, (PositionImpl) position); PositionImpl firstPosition = ongoingTxns.get(ongoingTxns.firstKey()); //max read position is less than first ongoing transaction message position, so entryId -1 @@ -191,6 +191,7 @@ private void handleTransactionMessage(TxnID txnId, Position position) { public CompletableFuture openTransactionBufferReader(TxnID txnID, long startSequenceId) { return null; } + @Override public CompletableFuture commitTxn(TxnID txnID, long lowWaterMark) { if (log.isDebugEnabled()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java index 4da82a29a54bc..0c3452762f2b0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java @@ -148,9 +148,9 @@ private CompletableFuture endTxn(long requestId, String topic, ByteBuf cm } }); } catch (ExecutionException e) { + log.error("endTxn channel is not active exception", e); cache.invalidate(topic); cb.completeExceptionally(new PulsarClientException.LookupException(e.getCause().getMessage())); - pendingRequests.remove(requestId); op.recycle(); } return cb; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java index a7ab2ea0bd1e6..e4dd0784eba6d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckHandle.java @@ -53,8 +53,8 @@ public interface PendingAckHandle { * @throws NotAllowedException if Use this method incorrectly eg. not use * PositionImpl or cumulative ack with a list of positions. */ - CompletableFuture individualAcknowledgeMessage(TxnID txnID, - List> positions); + CompletableFuture individualAcknowledgeMessage(TxnID txnID, List> positions); /** * Acknowledge message(s) for an ongoing transaction. @@ -121,4 +121,18 @@ CompletableFuture individualAcknowledgeMessage(TxnID txnID, * @param position {@link Position} which position need to clear */ void clearIndividualPosition(Position position); -} + + /** + * Pending ack recover whether ready future. + * + * @return the future of result. + */ + CompletableFuture pendingAckHandleFuture(); + + /** + * Close the pending ack handle. + * + * @return the future of this operation. + */ + CompletableFuture close(); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java new file mode 100644 index 0000000000000..3f2cc51f8ecb6 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckReplyCallBack.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack; + +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadataEntry; + +/** + * Call back for pending ack reply. + */ +public interface PendingAckReplyCallBack { + + /** + * Pending ack replay complete callback for pending ack store. + */ + void replayComplete(); + + /** + * Handle metadata entry. + * + * @param pendingAckMetadataEntry {@link PendingAckMetadataEntry} the metadata entry of pending ack + */ + void handleMetadataEntry(PendingAckMetadataEntry pendingAckMetadataEntry); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java new file mode 100644 index 0000000000000..3da676eb827d0 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckStore.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.api.proto.CommandAck.AckType; + +/** + * To store transaction pending ack. + */ +public interface PendingAckStore { + /** + * Replay pending ack to recover the pending ack subscription pending ack state. + * + * @param pendingAckHandle the handle of pending ack + * @param executorService the replay executor service + */ + void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService executorService); + + /** + * Close the transaction pending ack store. + * + * @return a future represents the result of this operation + */ + CompletableFuture closeAsync(); + + /** + * Append the individual pending ack operation to the ack persistent store. + * + * @param txnID {@link TxnID} transaction id. + * @param positions {@link List} the list of position and batch size. + * @return a future represents the result of this operation + */ + CompletableFuture appendIndividualAck(TxnID txnID, List> positions); + + /** + * Append the cumulative pending ack operation to the ack persistent store. + * + * @param txnID {@link TxnID} transaction id. + * @param position {@link PositionImpl} the pending ack position. + * @return a future represents the result of this operation + */ + CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position); + + /** + * Append the pending ack commit mark to the ack persistent store. + * + * @param txnID {@link TxnID} the transaction id for add commit mark. + * @param ackType {@link AckType} the ack type of the commit. + * @return a future represents the result of this operation + */ + CompletableFuture appendCommitMark(TxnID txnID, AckType ackType); + + /** + * Append the pending ack abort mark to the ack persistent store. + * + * @param txnID {@link Position} the txnID + * @param ackType {@link AckType} the ack type of the abort. + * @return a future represents the result of this operation + */ + CompletableFuture appendAbortMark(TxnID txnID, AckType ackType); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java new file mode 100644 index 0000000000000..e1a14d766d52b --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/TransactionPendingAckStoreProvider.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack; + +import static com.google.common.base.Preconditions.checkArgument; +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; + +/** + * Provider of transaction pending ack store. + */ +public interface TransactionPendingAckStoreProvider { + + /** + * Construct a provider from the provided class. + * + * @param providerClassName {@link String} the provider class name + * @return an instance of transaction buffer provider. + */ + static TransactionPendingAckStoreProvider newProvider(String providerClassName) throws IOException { + Class providerClass; + try { + providerClass = Class.forName(providerClassName); + Object obj = providerClass.newInstance(); + checkArgument(obj instanceof TransactionPendingAckStoreProvider, + "The factory has to be an instance of " + + TransactionPendingAckStoreProvider.class.getName()); + + return (TransactionPendingAckStoreProvider) obj; + } catch (Exception e) { + throw new IOException(e); + } + } + + /** + * Open the pending ack store. + * + * @param subscription {@link PersistentSubscription} + * @return a future represents the result of the operation. + * an instance of {@link PendingAckStore} is returned + * if the operation succeeds. + */ + CompletableFuture newPendingAckStore(PersistentSubscription subscription); + +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java new file mode 100644 index 0000000000000..0f41e2cd3c455 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/TransactionPendingAckStoreProviderException.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.exceptions; + +import org.apache.pulsar.broker.transaction.buffer.exceptions.TransactionBufferException; + +/** + * Transaction pending ack store provider exception. + */ +public class TransactionPendingAckStoreProviderException extends TransactionBufferException { + + public TransactionPendingAckStoreProviderException(String message) { + super(message); + } + +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/package-info.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/package-info.java new file mode 100644 index 0000000000000..314cceec797fd --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/exceptions/package-info.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * The implementation for pending ack exceptions. + */ +package org.apache.pulsar.broker.transaction.pendingack.exceptions; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java new file mode 100644 index 0000000000000..d882c80c47863 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStore.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.api.proto.CommandAck.AckType; + +/** + * In memory implementation of {@link PendingAckStore}. + */ +public class InMemoryPendingAckStore implements PendingAckStore { + + @Override + public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService scheduledExecutorService) { + pendingAckHandle.changeToReadyState(); + } + + @Override + public CompletableFuture closeAsync() { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendIndividualAck(TxnID txnID, + List> positions) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) { + return CompletableFuture.completedFuture(null); + } + +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStoreProvider.java new file mode 100644 index 0000000000000..1958236c05e53 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/InMemoryPendingAckStoreProvider.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import java.util.concurrent.CompletableFuture; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; + +public class InMemoryPendingAckStoreProvider implements TransactionPendingAckStoreProvider { + + @Override + public CompletableFuture newPendingAckStore(PersistentSubscription subscription) { + return CompletableFuture.completedFuture(new InMemoryPendingAckStore()); + } +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java new file mode 100644 index 0000000000000..dd7989d6da2ce --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckReplyCallBack.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack; +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadata; +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadataEntry; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.api.proto.CommandAck.AckType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * MLPendingAckStore reply call back. + */ +public class MLPendingAckReplyCallBack implements PendingAckReplyCallBack { + + private final PendingAckHandleImpl pendingAckHandle; + + public MLPendingAckReplyCallBack(PendingAckHandleImpl pendingAckHandle) { + this.pendingAckHandle = pendingAckHandle; + } + + @Override + public void replayComplete() { + log.info("Topic name : [{}], SubName : [{}] pending ack state reply success!", + pendingAckHandle.getTopicName(), pendingAckHandle.getSubName()); + + if (pendingAckHandle.changeToReadyState()) { + pendingAckHandle.completeHandleFuture(); + log.info("Topic name : [{}], SubName : [{}] pending ack state reply success!", + pendingAckHandle.getTopicName(), pendingAckHandle.getSubName()); + } else { + log.error("Topic name : [{}], SubName : [{}] pending ack state reply fail!", + pendingAckHandle.getTopicName(), pendingAckHandle.getSubName()); + } + } + + @Override + public void handleMetadataEntry(PendingAckMetadataEntry pendingAckMetadataEntry) { + TxnID txnID = new TxnID(pendingAckMetadataEntry.getTxnidMostBits(), + pendingAckMetadataEntry.getTxnidLeastBits()); + AckType ackType = pendingAckMetadataEntry.getAckType(); + switch (pendingAckMetadataEntry.getPendingAckOp()) { + case ABORT: + pendingAckHandle.handleAbort(txnID, ackType); + break; + case COMMIT: + pendingAckHandle.handleCommit(txnID, ackType, Collections.emptyMap()); + break; + case ACK: + if (ackType == AckType.Cumulative) { + PendingAckMetadata pendingAckMetadata = + pendingAckMetadataEntry.getPendingAckMetadatasList().get(0); + pendingAckHandle.handleCumulativeAckRecover(txnID, + PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId())); + } else { + List> positions = new ArrayList<>(); + pendingAckMetadataEntry.getPendingAckMetadatasList().forEach(pendingAckMetadata -> { + if (pendingAckMetadata.getAckSetsCount() == 0) { + positions.add(new MutablePair<>(PositionImpl.get(pendingAckMetadata.getLedgerId(), + pendingAckMetadata.getEntryId()), pendingAckMetadata.getBatchSize())); + } else { + PositionImpl position = + PositionImpl.get(pendingAckMetadata.getLedgerId(), pendingAckMetadata.getEntryId()); + if (pendingAckMetadata.getAckSetsCount() > 0) { + long[] ackSets = new long[pendingAckMetadata.getAckSetsCount()]; + for (int i = 0; i < pendingAckMetadata.getAckSetsCount(); i++) { + ackSets[i] = pendingAckMetadata.getAckSetAt(i); + } + position.setAckSet(ackSets); + } + positions.add(new MutablePair<>(position, pendingAckMetadata.getBatchSize())); + } + }); + pendingAckHandle.handleIndividualAckRecover(txnID, positions); + } + break; + default: + throw new IllegalStateException("Transaction pending ack replay " + + "error with illegal state : " + pendingAckMetadataEntry.getPendingAckOp()); + + } + } + + private static final Logger log = LoggerFactory.getLogger(MLPendingAckReplyCallBack.class); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java new file mode 100644 index 0000000000000..14aaa5feb3e26 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -0,0 +1,396 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import com.google.common.collect.ComparisonChain; +import io.netty.buffer.ByteBuf; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadata; +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckMetadataEntry; +import org.apache.pulsar.broker.transaction.pendingack.proto.PendingAckOp; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.api.proto.CommandAck.AckType; +import org.apache.pulsar.common.naming.TopicName; +import org.jctools.queues.MessagePassingQueue; +import org.jctools.queues.SpscArrayQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The implement of the pending ack store by manageLedger. + */ +public class MLPendingAckStore implements PendingAckStore { + + + private final ManagedLedger managedLedger; + + private final ManagedCursor cursor; + + public static final String PENDING_ACK_STORE_SUFFIX = "__transaction_pending_ack"; + + private static final String PENDING_ACK_STORE_CURSOR_NAME = "__pending_ack_state"; + + private final SpscArrayQueue entryQueue; + + //this is for replay + private final PositionImpl lastConfirmedEntry; + + private PositionImpl currentLoadPosition; + + /** + * The map is for pending ack store clear useless data. + *

+ * When ack message append to pending ack store, it will store the position which is persistent as key. + *

+ * When ack message append to pending ack store, it will store the position which is the max position of this + * ack by the original topic as value. + *

+ * It will judge the position with the max sub cursor position whether smaller than the subCursor mark + * delete position. + *

+ * If the max position is smaller than the subCursor mark delete position, the log cursor will mark delete + * the position. + */ + private final ConcurrentSkipListMap metadataPositions; + + private final ManagedCursor subManagedCursor; + + public MLPendingAckStore(ManagedLedger managedLedger, ManagedCursor cursor, + ManagedCursor subManagedCursor) { + this.managedLedger = managedLedger; + this.cursor = cursor; + this.currentLoadPosition = (PositionImpl) this.cursor.getMarkDeletedPosition(); + this.entryQueue = new SpscArrayQueue<>(2000); + this.lastConfirmedEntry = (PositionImpl) managedLedger.getLastConfirmedEntry(); + this.metadataPositions = new ConcurrentSkipListMap<>(); + this.subManagedCursor = subManagedCursor; + } + + @Override + public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService transactionReplayExecutor) { + transactionReplayExecutor + .execute(new PendingAckReplay(new MLPendingAckReplyCallBack(pendingAckHandle))); + } + + //TODO can control the number of entry to read + private void readAsync(int numberOfEntriesToRead, + AsyncCallbacks.ReadEntriesCallback readEntriesCallback) { + cursor.asyncReadEntries(numberOfEntriesToRead, readEntriesCallback, System.nanoTime(), PositionImpl.latest); + } + + @Override + public CompletableFuture closeAsync() { + CompletableFuture completableFuture = new CompletableFuture<>(); + cursor.asyncClose(new AsyncCallbacks.CloseCallback() { + @Override + public void closeComplete(Object ctx) { + try { + managedLedger.close(); + } catch (Exception e) { + completableFuture.completeExceptionally(e); + } + completableFuture.complete(null); + } + + @Override + public void closeFailed(ManagedLedgerException exception, Object ctx) { + completableFuture.completeExceptionally(exception); + } + }, null); + return completableFuture; + } + + @Override + public CompletableFuture appendIndividualAck(TxnID txnID, + List> positions) { + PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); + pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ACK); + pendingAckMetadataEntry.setAckType(AckType.Individual); + List pendingAckMetadataList = new ArrayList<>(); + positions.forEach(positionIntegerMutablePair -> { + PendingAckMetadata pendingAckMetadata = new PendingAckMetadata(); + PositionImpl position = positionIntegerMutablePair.getLeft(); + int batchSize = positionIntegerMutablePair.getRight(); + if (positionIntegerMutablePair.getLeft().getAckSet() != null) { + for (long l : position.getAckSet()) { + pendingAckMetadata.addAckSet(l); + } + } + pendingAckMetadata.setLedgerId(position.getLedgerId()); + pendingAckMetadata.setEntryId(position.getEntryId()); + pendingAckMetadata.setBatchSize(batchSize); + pendingAckMetadataList.add(pendingAckMetadata); + }); + pendingAckMetadataEntry.addAllPendingAckMetadatas(pendingAckMetadataList); + return appendCommon(pendingAckMetadataEntry, txnID); + } + + @Override + public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { + PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); + pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ACK); + pendingAckMetadataEntry.setAckType(AckType.Cumulative); + PendingAckMetadata pendingAckMetadata = new PendingAckMetadata(); + if (position.getAckSet() != null) { + for (long l : position.getAckSet()) { + pendingAckMetadata.addAckSet(l); + } + } + pendingAckMetadata.setLedgerId(position.getLedgerId()); + pendingAckMetadata.setEntryId(position.getEntryId()); + pendingAckMetadataEntry.addAllPendingAckMetadatas(Collections.singleton(pendingAckMetadata)); + return appendCommon(pendingAckMetadataEntry, txnID); + } + + @Override + public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) { + PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); + pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.COMMIT); + pendingAckMetadataEntry.setAckType(ackType); + return appendCommon(pendingAckMetadataEntry, txnID); + } + + @Override + public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) { + PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); + pendingAckMetadataEntry.setPendingAckOp(PendingAckOp.ABORT); + pendingAckMetadataEntry.setAckType(ackType); + return appendCommon(pendingAckMetadataEntry, txnID); + } + + private CompletableFuture appendCommon(PendingAckMetadataEntry pendingAckMetadataEntry, TxnID txnID) { + CompletableFuture completableFuture = new CompletableFuture<>(); + pendingAckMetadataEntry.setTxnidLeastBits(txnID.getLeastSigBits()); + pendingAckMetadataEntry.setTxnidMostBits(txnID.getMostSigBits()); + int transactionMetadataEntrySize = pendingAckMetadataEntry.getSerializedSize(); + ByteBuf buf = PulsarByteBufAllocator.DEFAULT.buffer(transactionMetadataEntrySize, transactionMetadataEntrySize); + pendingAckMetadataEntry.writeTo(buf); + managedLedger.asyncAddEntry(buf, new AsyncCallbacks.AddEntryCallback() { + + @Override + public void addComplete(Position position, ByteBuf entryData, Object ctx) { + if (log.isDebugEnabled()) { + log.debug("[{}][{}] MLPendingAckStore message append success at {} txnId: {}, operation : {}", + managedLedger.getName(), ctx, position, txnID, pendingAckMetadataEntry.getPendingAckOp()); + } + // store the persistent position in to memory + if (pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.ABORT + && pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.COMMIT) { + Optional optional = pendingAckMetadataEntry.getPendingAckMetadatasList() + .stream().max((o1, o2) -> ComparisonChain.start().compare(o1.getLedgerId(), + o2.getLedgerId()).compare(o1.getEntryId(), o2.getEntryId()).result()); + optional.ifPresent(pendingAckMetadata -> + metadataPositions.compute((PositionImpl) position, (thisPosition, otherPosition) -> { + PositionImpl nowPosition = PositionImpl.get(pendingAckMetadata.getLedgerId(), + pendingAckMetadata.getEntryId()); + if (otherPosition == null) { + return nowPosition; + } else { + return nowPosition.compareTo(otherPosition) > 0 ? nowPosition : otherPosition; + } + })); + } + + buf.release(); + completableFuture.complete(null); + + if (!metadataPositions.isEmpty()) { + PositionImpl firstPosition = metadataPositions.firstEntry().getKey(); + PositionImpl deletePosition = metadataPositions.firstEntry().getKey(); + while (!metadataPositions.isEmpty() + && metadataPositions.firstKey() != null + && subManagedCursor.getPersistentMarkDeletedPosition() != null + && metadataPositions.firstEntry().getValue() + .compareTo((PositionImpl) subManagedCursor.getPersistentMarkDeletedPosition()) <= 0) { + deletePosition = metadataPositions.firstKey(); + metadataPositions.remove(metadataPositions.firstKey()); + } + + if (firstPosition != deletePosition) { + PositionImpl finalDeletePosition = deletePosition; + cursor.asyncMarkDelete(deletePosition, + new AsyncCallbacks.MarkDeleteCallback() { + @Override + public void markDeleteComplete(Object ctx) { + if (log.isDebugEnabled()) { + log.debug("[{}] Transaction pending ack store mark delete position : " + + "[{}] success", managedLedger.getName(), + finalDeletePosition); + } + } + + @Override + public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { + if (log.isDebugEnabled()) { + log.error("[{}] Transaction pending ack store mark delete position : " + + "[{}] fail!", managedLedger.getName(), + finalDeletePosition, exception); + } + } + }, null); + } + } + } + + @Override + public void addFailed(ManagedLedgerException exception, Object ctx) { + log.error("[{}][{}] MLPendingAckStore message append fail exception : {}, operation : {}", + managedLedger.getName(), ctx, exception, pendingAckMetadataEntry.getPendingAckOp()); + buf.release(); + completableFuture.completeExceptionally(new PersistenceException(exception)); + } + } , null); + return completableFuture; + } + + class PendingAckReplay implements Runnable { + + private final FillEntryQueueCallback fillEntryQueueCallback; + private final PendingAckReplyCallBack pendingAckReplyCallBack; + + PendingAckReplay(PendingAckReplyCallBack pendingAckReplyCallBack) { + this.fillEntryQueueCallback = new FillEntryQueueCallback(); + this.pendingAckReplyCallBack = pendingAckReplyCallBack; + } + + @Override + public void run() { + try { + while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0) { + if (((ManagedCursorImpl) cursor).isClosed()) { + log.warn("[{}] MLPendingAckStore cursor have been closed, close replay thread.", + cursor.getManagedLedger().getName()); + return; + } + fillEntryQueueCallback.fillQueue(); + Entry entry = entryQueue.poll(); + if (entry != null) { + ByteBuf buffer = entry.getDataBuffer(); + currentLoadPosition = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); + PendingAckMetadataEntry pendingAckMetadataEntry = new PendingAckMetadataEntry(); + pendingAckMetadataEntry.parseFrom(buffer, buffer.readableBytes()); + // store the persistent position in to memory + // store the max position of this entry retain + if (pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.ABORT + && pendingAckMetadataEntry.getPendingAckOp() != PendingAckOp.COMMIT) { + Optional optional = pendingAckMetadataEntry.getPendingAckMetadatasList() + .stream().max((o1, o2) -> ComparisonChain.start().compare(o1.getLedgerId(), + o2.getLedgerId()).compare(o1.getEntryId(), o2.getEntryId()).result()); + + optional.ifPresent(pendingAckMetadata -> + metadataPositions.compute(PositionImpl.get(entry.getLedgerId(), entry.getEntryId()), + (thisPosition, otherPosition) -> { + PositionImpl nowPosition = PositionImpl + .get(pendingAckMetadata.getLedgerId(), + pendingAckMetadata.getEntryId()); + if (otherPosition == null) { + return nowPosition; + } else { + return nowPosition.compareTo(otherPosition) > 0 ? nowPosition + : otherPosition; + } + })); + } + pendingAckReplyCallBack.handleMetadataEntry(pendingAckMetadataEntry); + entry.release(); + } else { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + if (Thread.interrupted()) { + log.error("[{}]Transaction pending " + + "replay thread interrupt!", managedLedger.getName(), e); + } + } + } + } + } catch (Exception e) { + log.error("[{}] Pending ack recover fail!", subManagedCursor.getManagedLedger().getName(), e); + return; + } + pendingAckReplyCallBack.replayComplete(); + } + } + + class FillEntryQueueCallback implements AsyncCallbacks.ReadEntriesCallback { + + private final AtomicLong outstandingReadsRequests = new AtomicLong(0); + + void fillQueue() { + if (entryQueue.size() < entryQueue.capacity() && outstandingReadsRequests.get() == 0) { + if (cursor.hasMoreEntries()) { + outstandingReadsRequests.incrementAndGet(); + readAsync(100, this); + } + } + } + + @Override + public void readEntriesComplete(List entries, Object ctx) { + entryQueue.fill(new MessagePassingQueue.Supplier() { + private int i = 0; + @Override + public Entry get() { + Entry entry = entries.get(i); + i++; + return entry; + } + }, entries.size()); + + outstandingReadsRequests.decrementAndGet(); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + log.error("MLPendingAckStore stat reply fail!", exception); + outstandingReadsRequests.decrementAndGet(); + } + + } + + public static String getTransactionPendingAckStoreSuffix(String originTopicName, String subName) { + return TopicName.get(originTopicName) + "-" + subName + PENDING_ACK_STORE_SUFFIX; + } + + public static String getTransactionPendingAckStoreCursorName() { + return PENDING_ACK_STORE_CURSOR_NAME; + } + + private static final Logger log = LoggerFactory.getLogger(MLPendingAckStore.class); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java new file mode 100644 index 0000000000000..0741f06df8fa9 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import java.util.concurrent.CompletableFuture; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; +import org.apache.pulsar.broker.transaction.pendingack.exceptions.TransactionPendingAckStoreProviderException; +import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; +import org.apache.pulsar.common.naming.TopicName; + +/** + * Provider is for MLPendingAckStore. + */ +@Slf4j +public class MLPendingAckStoreProvider implements TransactionPendingAckStoreProvider { + + @Override + public CompletableFuture newPendingAckStore(PersistentSubscription subscription) { + CompletableFuture pendingAckStoreFuture = new CompletableFuture<>(); + + if (subscription == null) { + pendingAckStoreFuture.completeExceptionally( + new TransactionPendingAckStoreProviderException("The subscription is null.")); + return pendingAckStoreFuture; + } + + PersistentTopic originPersistentTopic = (PersistentTopic) subscription.getTopic(); + String pendingAckTopicName = MLPendingAckStore + .getTransactionPendingAckStoreSuffix(originPersistentTopic.getName(), subscription.getName()); + + originPersistentTopic.getBrokerService().getManagedLedgerFactory() + .asyncOpen(TopicName.get(pendingAckTopicName).getPersistenceNamingEncoding(), + originPersistentTopic.getManagedLedger().getConfig(), + new AsyncCallbacks.OpenLedgerCallback() { + @Override + public void openLedgerComplete(ManagedLedger ledger, Object ctx) { + ledger.asyncOpenCursor(MLPendingAckStore.getTransactionPendingAckStoreCursorName(), + InitialPosition.Earliest, new AsyncCallbacks.OpenCursorCallback() { + @Override + public void openCursorComplete(ManagedCursor cursor, Object ctx) { + pendingAckStoreFuture + .complete(new MLPendingAckStore(ledger, cursor, + subscription.getCursor())); + } + + @Override + public void openCursorFailed(ManagedLedgerException exception, Object ctx) { + log.error("Open MLPendingAckStore cursor failed.", exception); + pendingAckStoreFuture.completeExceptionally(exception); + } + }, null); + } + + @Override + public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { + log.error("Open MLPendingAckStore managedLedger failed.", exception); + pendingAckStoreFuture.completeExceptionally(exception); + } + }, () -> true, null); + return pendingAckStoreFuture; + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java index f58054a9d2c66..cb86ff7202033 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleDisabled.java @@ -35,6 +35,9 @@ */ public class PendingAckHandleDisabled implements PendingAckHandle { + private final CompletableFuture pendingAckHandleCompletableFuture = + CompletableFuture.completedFuture(PendingAckHandleDisabled.this); + @Override public CompletableFuture individualAcknowledgeMessage(TxnID txnID, List> positions) { @@ -68,6 +71,17 @@ public boolean checkIsCanDeleteConsumerPendingAck(PositionImpl position) { @Override public void clearIndividualPosition(Position position) { - //no operation + //no-op + } + + @Override + public CompletableFuture pendingAckHandleFuture() { + return pendingAckHandleCompletableFuture; } + + @Override + public CompletableFuture close() { + return CompletableFuture.completedFuture(null); + } + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index d08b4028ffe6c..dcc4377fa5021 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -26,7 +26,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; import lombok.extern.slf4j.Slf4j; @@ -37,9 +36,13 @@ import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException; +import org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.transaction.pendingack.PendingAckHandle; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.util.FutureUtil; @@ -50,7 +53,7 @@ * The default implementation of {@link PendingAckHandle}. */ @Slf4j -public class PendingAckHandleImpl implements PendingAckHandle { +public class PendingAckHandleImpl extends PendingAckHandleState implements PendingAckHandle { /** * The map is for transaction with position witch was individual acked by this transaction. @@ -95,14 +98,35 @@ public class PendingAckHandleImpl implements PendingAckHandle { private final PersistentSubscription persistentSubscription; + private final CompletableFuture pendingAckStoreFuture; + + private final CompletableFuture pendingAckHandleCompletableFuture = new CompletableFuture<>(); + public PendingAckHandleImpl(PersistentSubscription persistentSubscription) { + super(State.None); this.topicName = persistentSubscription.getTopicName(); this.subName = persistentSubscription.getName(); this.persistentSubscription = persistentSubscription; + + TransactionPendingAckStoreProvider pendingAckStoreProvider = + ((PersistentTopic) this.persistentSubscription.getTopic()) + .getBrokerService().getPulsar().getTransactionPendingAckStoreProvider(); + this.pendingAckStoreFuture = + pendingAckStoreProvider.newPendingAckStore(persistentSubscription); + + this.pendingAckStoreFuture.thenAccept(pendingAckStore -> { + changeToInitializingState(); + pendingAckStore.replayAsync(this, + ((PersistentTopic) persistentSubscription.getTopic()).getBrokerService() + .getPulsar().getTransactionReplayExecutor()); + }).exceptionally(e -> { + log.error("PendingAckHandleImpl init fail! TopicName : {}, SubName: {}", topicName, subName, e); + return null; + }); } @Override - public synchronized CompletableFuture individualAcknowledgeMessage(TxnID txnID, + public CompletableFuture individualAcknowledgeMessage(TxnID txnID, List> positions) { if (txnID == null) { return FutureUtil.failedFuture(new NotAllowedException("TransactionID can not be null.")); @@ -111,99 +135,104 @@ public synchronized CompletableFuture individualAcknowledgeMessage(TxnID t return FutureUtil.failedFuture(new NotAllowedException("Positions can not be null.")); } CompletableFuture completableFuture = new CompletableFuture<>(); - for (MutablePair positionIntegerMutablePair : positions) { - PositionImpl position = positionIntegerMutablePair.left; - - // If try to ack message already acked by committed transaction or normal acknowledge, throw exception. - if (((ManagedCursorImpl) persistentSubscription.getCursor()) - .isMessageDeleted(position)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to ack message:" + position + " already acked before."; - log.error(errorMsg); - return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); - } - - if (position.hasAckSet()) { - //in order to jude the bit set is over lap, so set the covering the batch size bit to 1, - // should know the two bit set don't have the same point is 0 - BitSetRecyclable bitSetRecyclable = BitSetRecyclable.valueOf(position.getAckSet()); - if (positionIntegerMutablePair.right > bitSetRecyclable.size()) { - bitSetRecyclable.set(positionIntegerMutablePair.right); - } - bitSetRecyclable.set(positionIntegerMutablePair.right, bitSetRecyclable.size()); - long[] ackSetOverlap = bitSetRecyclable.toLongArray(); - bitSetRecyclable.recycle(); - if (isAckSetOverlap(ackSetOverlap, - ((ManagedCursorImpl) persistentSubscription.getCursor()).getBatchPositionAckSet(position))) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to ack message:" + position + " already acked before."; - log.error(errorMsg); - return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); - } - if (this.individualAckPositions != null && individualAckPositions.containsKey(position) - && isAckSetOverlap(individualAckPositions.get(position).getLeft().getAckSet(), ackSetOverlap)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to ack batch message:" + position + " in pending ack status."; - log.error(errorMsg); - return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); - } - } else { - if (this.individualAckPositions != null && this.individualAckPositions.containsKey(position)) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to ack message:" + position + " in pending ack status."; - log.error(errorMsg); - return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); - } - } - } - for (int i = 0; i < positions.size(); i++) { - if (log.isDebugEnabled()) { - log.debug("[{}][{}] TxnID:[{}] Individual acks on {}", topicName, subName, txnID.toString(), positions); - } - if (individualAckOfTransaction == null) { - individualAckOfTransaction = new LinkedMap<>(); - } - - if (individualAckPositions == null) { - individualAckPositions = new ConcurrentSkipListMap<>(); - } - - PositionImpl position = positions.get(i).left; - - if (position.hasAckSet()) { - - HashMap pendingAckMessageForCurrentTxn = - individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); - - if (pendingAckMessageForCurrentTxn.containsKey(position)) { - andAckSet(pendingAckMessageForCurrentTxn.get(position), position); - } else { - pendingAckMessageForCurrentTxn.put(position, position); - } - - if (!individualAckPositions.containsKey(position)) { - this.individualAckPositions.put(position, positions.get(i)); - } else { - MutablePair positionPair = this.individualAckPositions.get(position); - positionPair.setRight(positions.get(i).right); - andAckSet(positionPair.getLeft(), position); - } - - } else { - HashMap pendingAckMessageForCurrentTxn = - individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); - pendingAckMessageForCurrentTxn.put(position, position); - this.individualAckPositions.putIfAbsent(position, positions.get(i)); - } - } - completableFuture.complete(null); + this.pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendIndividualAck(txnID, positions).thenAccept(v -> { + synchronized (org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl.this) { + for (MutablePair positionIntegerMutablePair : positions) { + + if (log.isDebugEnabled()) { + log.debug("[{}] individualAcknowledgeMessage position: [{}], " + + "txnId: [{}], subName: [{}]", topicName, + positionIntegerMutablePair.left, txnID, subName); + } + PositionImpl position = positionIntegerMutablePair.left; + + // If try to ack message already acked by committed transaction or + // normal acknowledge,throw exception. + if (((ManagedCursorImpl) persistentSubscription.getCursor()) + .isMessageDeleted(position)) { + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to ack message:" + position + " already acked before."; + log.error(errorMsg); + completableFuture + .completeExceptionally(new TransactionConflictException(errorMsg)); + return; + } + + if (position.hasAckSet()) { + //in order to jude the bit set is over lap, so set the covering + // the batch size bit to 1,should know the two + // bit set don't have the same point is 0 + BitSetRecyclable bitSetRecyclable = + BitSetRecyclable.valueOf(position.getAckSet()); + if (positionIntegerMutablePair.right > bitSetRecyclable.size()) { + bitSetRecyclable.set(positionIntegerMutablePair.right); + } + bitSetRecyclable.set(positionIntegerMutablePair.right, bitSetRecyclable.size()); + long[] ackSetOverlap = bitSetRecyclable.toLongArray(); + bitSetRecyclable.recycle(); + if (isAckSetOverlap(ackSetOverlap, + ((ManagedCursorImpl) persistentSubscription.getCursor()) + .getBatchPositionAckSet(position))) { + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + + txnID + " try to ack message:" + + position + " already acked before."; + log.error(errorMsg); + completableFuture + .completeExceptionally(new TransactionConflictException(errorMsg)); + return; + } + + if (individualAckPositions != null + && individualAckPositions.containsKey(position) + && isAckSetOverlap(individualAckPositions + .get(position).getLeft().getAckSet(), ackSetOverlap)) { + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + + txnID + " try to ack batch message:" + + position + " in pending ack status."; + log.error(errorMsg); + completableFuture + .completeExceptionally(new TransactionConflictException(errorMsg)); + return; + } + } else { + if (individualAckPositions != null + && individualAckPositions.containsKey(position)) { + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + + txnID + " try to ack message:" + + position + " in pending ack status."; + log.error(errorMsg); + completableFuture + .completeExceptionally(new TransactionConflictException(errorMsg)); + return; + } + } + } + + handleIndividualAck(txnID, positions); + completableFuture.complete(null); + } + }).exceptionally(e -> { + synchronized (PendingAckHandleImpl.this) { + // we also modify the in memory state when append fail, + // because we don't know the persistent state, when were replay it, + // it will produce the wrong operation. so we append fail, + // we should wait tc time out or client abort this transaction. + handleIndividualAck(txnID, positions); + completableFuture.completeExceptionally(e.getCause()); + } + return null; + })).exceptionally(e -> { + completableFuture.completeExceptionally(e); + return null; + }); return completableFuture; } @Override - public synchronized CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, - List positions) { + public CompletableFuture cumulativeAcknowledgeMessage(TxnID txnID, + List positions) { if (txnID == null) { return FutureUtil.failedFuture(new NotAllowedException("TransactionID can not be null.")); @@ -221,94 +250,182 @@ public synchronized CompletableFuture cumulativeAcknowledgeMessage(TxnID t PositionImpl position = positions.get(0); - if (position.compareTo((PositionImpl) persistentSubscription.getCursor().getMarkDeletedPosition()) <= 0) { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to cumulative ack position: " + position + " within range of cursor's " - + "markDeletePosition: " + persistentSubscription.getCursor().getMarkDeletedPosition(); - log.error(errorMsg); - return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); - } + CompletableFuture completableFuture = new CompletableFuture<>(); - if (log.isDebugEnabled()) { - log.debug("[{}][{}] TxnID:[{}] Cumulative ack on {}.", topicName, subName, txnID.toString(), position); - } + this.pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendCumulativeAck(txnID, position).thenAccept(v -> { + if (log.isDebugEnabled()) { + log.debug("[{}] cumulativeAcknowledgeMessage position: [{}], " + + "txnID:[{}], subName: [{}].", topicName, txnID.toString(), position, subName); + } - if (this.cumulativeAckOfTransaction == null) { - this.cumulativeAckOfTransaction = MutablePair.of(txnID, position); - } else if (this.cumulativeAckOfTransaction.getKey().equals(txnID) - && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) > 0) { - this.cumulativeAckOfTransaction.setValue(position); + if (position.compareTo((PositionImpl) persistentSubscription.getCursor() + .getMarkDeletedPosition()) <= 0) { + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to cumulative ack position: " + position + " within range of cursor's " + + "markDeletePosition: " + + persistentSubscription.getCursor().getMarkDeletedPosition(); + log.error(errorMsg); + completableFuture.completeExceptionally(new TransactionConflictException(errorMsg)); + return; + } - } else { - String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID - + " try to cumulative batch ack position: " + position + " within range of current " - + "currentPosition: " + this.cumulativeAckOfTransaction.getValue(); - log.error(errorMsg); - return FutureUtil.failedFuture(new TransactionConflictException(errorMsg)); - } - return CompletableFuture.completedFuture(null); + if (cumulativeAckOfTransaction != null && (!cumulativeAckOfTransaction.getKey().equals(txnID) + || compareToWithAckSet(position, cumulativeAckOfTransaction.getValue()) <= 0)) { + String errorMsg = "[" + topicName + "][" + subName + "] Transaction:" + txnID + + " try to cumulative batch ack position: " + position + " within range of current " + + "currentPosition: " + cumulativeAckOfTransaction.getValue(); + log.error(errorMsg); + completableFuture.completeExceptionally(new TransactionConflictException(errorMsg)); + return; + } + + handleCumulativeAck(txnID, position); + completableFuture.complete(null); + }).exceptionally(e -> { + //we also modify the in memory state when append fail, because we don't know the persistent + // state, when wereplay it, it will produce the wrong operation. so we append fail, we should + // wait tc time out or client abort this transaction. + handleCumulativeAck(txnID, position); + completableFuture.completeExceptionally(e.getCause()); + return null; + }) + ).exceptionally(e -> { + completableFuture.completeExceptionally(e); + return null; + }); + return completableFuture; } @Override public synchronized CompletableFuture commitTxn(TxnID txnID, Map properties, long lowWaterMark) { + if (!checkIfReady()) { + return FutureUtil.failedFuture(new ServiceUnitNotReadyException("PendingAckHandle not replay complete!")); + } CompletableFuture commitFuture = new CompletableFuture<>(); + // It's valid to create transaction then commit without doing any operation, which will cause // pendingAckMessagesMap to be null. if (this.cumulativeAckOfTransaction != null) { if (cumulativeAckOfTransaction.getKey().equals(txnID)) { - persistentSubscription.acknowledgeMessage(Collections - .singletonList(this.cumulativeAckOfTransaction.getValue()), AckType.Cumulative, properties); - this.cumulativeAckOfTransaction = null; + pendingAckStoreFuture.thenAccept(pendingAckStore -> pendingAckStore + .appendCommitMark(txnID, AckType.Cumulative).thenAccept(v -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Transaction pending ack store commit txnId : [{}] " + + "success! subName: [{}]", topicName, txnID, subName); + } + persistentSubscription.acknowledgeMessage( + Collections.singletonList(cumulativeAckOfTransaction.getValue()), + AckType.Cumulative, properties); + cumulativeAckOfTransaction = null; + commitFuture.complete(null); + }).exceptionally(e -> { + log.error("[{}] Transaction pending ack store commit txnId : [{}] fail!", + topicName, txnID, e); + commitFuture.completeExceptionally(e); + return null; + })).exceptionally(e -> { + commitFuture.completeExceptionally(e); + return null; + }); + } else { + commitFuture.complete(null); } } else { - if (individualAckOfTransaction != null && individualAckOfTransaction.containsKey(txnID)) { - HashMap pendingAckMessageForCurrentTxn = - individualAckOfTransaction.remove(txnID); - if (pendingAckMessageForCurrentTxn != null) { - persistentSubscription.acknowledgeMessage(new ArrayList<>(pendingAckMessageForCurrentTxn.values()), - AckType.Individual, properties); - } - } + pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendCommitMark(txnID, AckType.Individual).thenAccept(v -> { + synchronized (PendingAckHandleImpl.this) { + if (individualAckOfTransaction != null && individualAckOfTransaction.containsKey(txnID)) { + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.get(txnID); + if (log.isDebugEnabled()) { + log.debug("[{}] Transaction pending ack store commit txnId : " + + "[{}] success! subName: [{}]", topicName, txnID, subName); + } + individualAckCommitCommon(txnID, pendingAckMessageForCurrentTxn, properties); + commitFuture.complete(null); + handleLowWaterMark(txnID, lowWaterMark); + } else { + commitFuture.complete(null); + } + } + }).exceptionally(e -> { + log.error("[{}] Transaction pending ack store commit txnId : [{}] fail!", + topicName, txnID, e); + commitFuture.completeExceptionally(e.getCause()); + return null; + })).exceptionally(e -> { + commitFuture.completeExceptionally(e); + return null; + }); } - handleLowWaterMark(txnID, lowWaterMark); - commitFuture.complete(null); return commitFuture; } @Override public synchronized CompletableFuture abortTxn(TxnID txnId, Consumer consumer, long lowWaterMark) { + if (!checkIfReady()) { + return FutureUtil.failedFuture(new ServiceUnitNotReadyException("PendingAckHandle not replay complete!")); + } CompletableFuture abortFuture = new CompletableFuture<>(); if (this.cumulativeAckOfTransaction != null) { - if (this.cumulativeAckOfTransaction.getKey().equals(txnId)) { - this.cumulativeAckOfTransaction = null; - } - this.persistentSubscription.redeliverUnacknowledgedMessages(consumer); - } else if (this.individualAckOfTransaction != null){ - HashMap pendingAckMessageForCurrentTxn = - individualAckOfTransaction.remove(txnId); - if (pendingAckMessageForCurrentTxn != null) { - for (Entry entry : pendingAckMessageForCurrentTxn.entrySet()) { - if (entry.getValue().hasAckSet() && individualAckPositions.containsKey(entry.getValue())) { - BitSetRecyclable thisBitSet = BitSetRecyclable.valueOf(entry.getValue().getAckSet()); - thisBitSet.flip(0, individualAckPositions.get(entry.getValue()).right); - BitSetRecyclable otherBitSet = - BitSetRecyclable.valueOf(individualAckPositions.get(entry.getValue()).left.getAckSet()); - otherBitSet.or(thisBitSet); - individualAckPositions.get(entry.getKey()).left.setAckSet(otherBitSet.toLongArray()); - otherBitSet.recycle(); - thisBitSet.recycle(); - } else { - individualAckPositions.remove(entry.getValue()); - } - } - this.persistentSubscription.redeliverUnacknowledgedMessages(consumer, - new ArrayList<>(pendingAckMessageForCurrentTxn.values())); - } + pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendAbortMark(txnId, AckType.Cumulative).thenAccept(v -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Transaction pending ack store abort txnId : [{}] success! subName: [{}]", + topicName, txnId, subName); + } + if (cumulativeAckOfTransaction.getKey().equals(txnId)) { + cumulativeAckOfTransaction = null; + } + persistentSubscription.redeliverUnacknowledgedMessages(consumer); + abortFuture.complete(null); + }).exceptionally(e -> { + log.error("[{}] Transaction pending ack store abort txnId : [{}] fail!", + topicName, txnId, e); + abortFuture.completeExceptionally(e); + return null; + }) + ).exceptionally(e -> { + abortFuture.completeExceptionally(e); + return null; + }); + } else if (this.individualAckOfTransaction != null) { + pendingAckStoreFuture.thenAccept(pendingAckStore -> + pendingAckStore.appendAbortMark(txnId, AckType.Individual).thenAccept(v -> { + synchronized (PendingAckHandleImpl.this) { + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.get(txnId); + if (pendingAckMessageForCurrentTxn != null) { + if (log.isDebugEnabled()) { + log.debug("[{}] Transaction pending ack store abort txnId : [{}] success! " + + "subName: [{}]", topicName, txnId, subName); + } + individualAckAbortCommon(txnId, pendingAckMessageForCurrentTxn); + persistentSubscription.redeliverUnacknowledgedMessages(consumer, + new ArrayList<>(pendingAckMessageForCurrentTxn.values())); + abortFuture.complete(null); + handleLowWaterMark(txnId, lowWaterMark); + } else { + abortFuture.complete(null); + } + } + }).exceptionally(e -> { + log.error("[{}] Transaction pending ack store abort txnId : [{}] fail!", + topicName, txnId, e); + abortFuture.completeExceptionally(e); + return null; + }) + ).exceptionally(e -> { + log.error("[{}] abortTxn", txnId, e); + abortFuture.completeExceptionally(e); + return null; + }); + } else { + abortFuture.complete(null); } - handleLowWaterMark(txnId, lowWaterMark); - abortFuture.complete(null); return abortFuture; } @@ -318,8 +435,22 @@ private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { if (firstTxn.getMostSigBits() == txnID.getMostSigBits() && firstTxn.getLeastSigBits() <= lowWaterMark) { - individualAckOfTransaction.remove(firstTxn); - handleLowWaterMark(txnID, lowWaterMark); + this.pendingAckStoreFuture.whenComplete((pendingAckStore, throwable) -> { + if (throwable == null) { + pendingAckStore.appendAbortMark(txnID, AckType.Individual).thenAccept(v -> { + synchronized (PendingAckHandleImpl.this) { + log.warn("[{}] Transaction pending ack handle low water mark success! txnId : [{}], " + + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark); + individualAckOfTransaction.remove(firstTxn); + handleLowWaterMark(txnID, lowWaterMark); + } + }).exceptionally(e -> { + log.warn("[{}] Transaction pending ack handle low water mark fail! txnId : [{}], " + + "lowWaterMark : [{}]", topicName, txnID, lowWaterMark); + return null; + }); + } + }); } } } @@ -329,8 +460,8 @@ public synchronized void syncBatchPositionAckSetForTransaction(PositionImpl posi if (individualAckPositions == null) { individualAckPositions = new ConcurrentSkipListMap<>(); } - //sync don't carry the batch size - //when one position is ack by transaction the batch size is for `and` operation. + // sync don't carry the batch size + // when one position is ack by transaction the batch size is for `and` operation. if (!individualAckPositions.containsKey(position)) { this.individualAckPositions.put(position, new MutablePair<>(position, 0)); } else { @@ -339,7 +470,7 @@ public synchronized void syncBatchPositionAckSetForTransaction(PositionImpl posi } @Override - public boolean checkIsCanDeleteConsumerPendingAck(PositionImpl position) { + public synchronized boolean checkIsCanDeleteConsumerPendingAck(PositionImpl position) { if (!individualAckPositions.containsKey(position)) { return true; } else { @@ -359,24 +490,217 @@ public boolean checkIsCanDeleteConsumerPendingAck(PositionImpl position) { } } + protected void handleAbort(TxnID txnID, AckType ackType) { + if (ackType == AckType.Cumulative) { + this.cumulativeAckOfTransaction = null; + } else { + if (this.individualAckOfTransaction != null) { + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.get(txnID); + if (pendingAckMessageForCurrentTxn != null) { + individualAckAbortCommon(txnID, pendingAckMessageForCurrentTxn); + } + } + } + } + + private void individualAckAbortCommon(TxnID txnID, HashMap currentTxn) { + for (Map.Entry entry : + currentTxn.entrySet()) { + if (entry.getValue().hasAckSet() + && individualAckPositions.containsKey(entry.getValue())) { + BitSetRecyclable thisBitSet = + BitSetRecyclable.valueOf(entry.getValue().getAckSet()); + thisBitSet.flip(0, individualAckPositions.get(entry.getValue()).right); + BitSetRecyclable otherBitSet = + BitSetRecyclable.valueOf(individualAckPositions + .get(entry.getValue()).left.getAckSet()); + otherBitSet.or(thisBitSet); + individualAckPositions.get(entry.getKey()) + .left.setAckSet(otherBitSet.toLongArray()); + otherBitSet.recycle(); + thisBitSet.recycle(); + } else { + individualAckPositions.remove(entry.getValue()); + } + } + individualAckOfTransaction.remove(txnID); + } + + protected void handleCommit(TxnID txnID, AckType ackType, Map properties) { + if (ackType == AckType.Cumulative) { + if (this.cumulativeAckOfTransaction != null) { + persistentSubscription.acknowledgeMessage( + Collections.singletonList(this.cumulativeAckOfTransaction.getValue()), + AckType.Cumulative, properties); + } + this.cumulativeAckOfTransaction = null; + } else { + if (this.individualAckOfTransaction != null) { + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.get(txnID); + if (pendingAckMessageForCurrentTxn != null) { + individualAckCommitCommon(txnID, pendingAckMessageForCurrentTxn, null); + } + } + } + } + + private void individualAckCommitCommon(TxnID txnID, + HashMap currentTxn, + Map properties) { + if (currentTxn != null) { + persistentSubscription.acknowledgeMessage(new ArrayList<>(currentTxn.values()), + AckType.Individual, properties); + individualAckOfTransaction.remove(txnID); + } + } + + private void handleIndividualAck(TxnID txnID, List> positions) { + for (int i = 0; i < positions.size(); i++) { + if (log.isDebugEnabled()) { + log.debug("[{}][{}] TxnID:[{}] Individual acks on {}", topicName, + subName, txnID.toString(), positions); + } + if (individualAckOfTransaction == null) { + individualAckOfTransaction = new LinkedMap<>(); + } + + if (individualAckPositions == null) { + individualAckPositions = new ConcurrentSkipListMap<>(); + } + + PositionImpl position = positions.get(i).left; + + if (position.hasAckSet()) { + + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); + + if (pendingAckMessageForCurrentTxn.containsKey(position)) { + andAckSet(pendingAckMessageForCurrentTxn.get(position), position); + } else { + pendingAckMessageForCurrentTxn.put(position, position); + } + + if (!individualAckPositions.containsKey(position)) { + this.individualAckPositions.put(position, positions.get(i)); + } else { + MutablePair positionPair = + this.individualAckPositions.get(position); + positionPair.setRight(positions.get(i).right); + andAckSet(positionPair.getLeft(), position); + } + + } else { + HashMap pendingAckMessageForCurrentTxn = + individualAckOfTransaction.computeIfAbsent(txnID, txn -> new HashMap<>()); + pendingAckMessageForCurrentTxn.put(position, position); + this.individualAckPositions.putIfAbsent(position, positions.get(i)); + } + } + } + + private void handleCumulativeAck(TxnID txnID, PositionImpl position) { + if (this.cumulativeAckOfTransaction == null) { + this.cumulativeAckOfTransaction = MutablePair.of(txnID, position); + } else if (this.cumulativeAckOfTransaction.getKey().equals(txnID) + && compareToWithAckSet(position, this.cumulativeAckOfTransaction.getValue()) > 0) { + this.cumulativeAckOfTransaction.setValue(position); + } + } + + protected void handleCumulativeAckRecover(TxnID txnID, PositionImpl position) { + if ((position.compareTo((PositionImpl) persistentSubscription.getCursor() + .getMarkDeletedPosition()) > 0) && (cumulativeAckOfTransaction == null + || (cumulativeAckOfTransaction.getKey().equals(txnID) + && compareToWithAckSet(position, cumulativeAckOfTransaction.getValue()) > 0))) { + handleCumulativeAck(txnID, position); + } + } + + protected void handleIndividualAckRecover(TxnID txnID, List> positions) { + for (MutablePair positionIntegerMutablePair : positions) { + PositionImpl position = positionIntegerMutablePair.left; + + // If try to ack message already acked by committed transaction or + // normal acknowledge,throw exception. + if (((ManagedCursorImpl) persistentSubscription.getCursor()) + .isMessageDeleted(position)) { + return; + } + + if (position.hasAckSet()) { + //in order to jude the bit set is over lap, so set the covering + // the batch size bit to 1,should know the two + // bit set don't have the same point is 0 + BitSetRecyclable bitSetRecyclable = + BitSetRecyclable.valueOf(position.getAckSet()); + if (positionIntegerMutablePair.right > bitSetRecyclable.size()) { + bitSetRecyclable.set(positionIntegerMutablePair.right); + } + bitSetRecyclable.set(positionIntegerMutablePair.right, bitSetRecyclable.size()); + long[] ackSetOverlap = bitSetRecyclable.toLongArray(); + bitSetRecyclable.recycle(); + if (isAckSetOverlap(ackSetOverlap, + ((ManagedCursorImpl) persistentSubscription.getCursor()) + .getBatchPositionAckSet(position))) { + return; + } + + if (individualAckPositions != null + && individualAckPositions.containsKey(position) + && isAckSetOverlap(individualAckPositions + .get(position).getLeft().getAckSet(), ackSetOverlap)) { + return; + } + } else { + if (individualAckPositions != null + && individualAckPositions.containsKey(position)) { + return; + } + } + } + handleIndividualAck(txnID, positions); + } + + public String getTopicName() { + return topicName; + } + + public String getSubName() { + return subName; + } + @Override - public void clearIndividualPosition(Position position) { + public synchronized void clearIndividualPosition(Position position) { if (individualAckPositions == null) { return; } if (position instanceof PositionImpl) { individualAckPositions.remove(position); - for (PositionImpl individualAckPosition : individualAckPositions.keySet()) { - // individualAckPositions is currentSkipListMap, delete the position form individualAckPositions which - // is smaller than can delete position - if (individualAckPosition.compareTo((PositionImpl) position) <= 0) { - individualAckPositions.remove(individualAckPosition); - } else { - return; - } - } } + + individualAckPositions.forEach((persistentPosition, positionIntegerMutablePair) -> { + if (persistentPosition.compareTo((PositionImpl) persistentSubscription + .getCursor().getMarkDeletedPosition()) < 0) { + individualAckPositions.remove(persistentPosition); + } + }); } -} + @Override + public CompletableFuture pendingAckHandleFuture() { + return pendingAckHandleCompletableFuture; + } + + public void completeHandleFuture() { + this.pendingAckHandleCompletableFuture.complete(PendingAckHandleImpl.this); + } + + @Override + public CompletableFuture close() { + return this.pendingAckStoreFuture.thenAccept(PendingAckStore::closeAsync); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java new file mode 100644 index 0000000000000..352962c33fa84 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleState.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack.impl; + +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; + +/** + * The implement of transaction pending ack store. + */ +public abstract class PendingAckHandleState { + + /** + * The state of the pending ack handle {@link PendingAckHandleState}. + */ + public enum State { + None, + Initializing, + Ready, + Close + } + + private static final AtomicReferenceFieldUpdater STATE_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(PendingAckHandleState.class, State.class, "state"); + + @SuppressWarnings("unused") + private volatile State state = null; + + public PendingAckHandleState(State state) { + STATE_UPDATER.set(this, state); + + } + + protected boolean changeToReadyState() { + return (STATE_UPDATER.compareAndSet(this, State.Initializing, State.Ready)); + } + + protected boolean changeToInitializingState() { + return STATE_UPDATER.compareAndSet(this, State.None, State.Initializing); + } + + protected boolean changeToCloseState() { + return (STATE_UPDATER.compareAndSet(this, State.Ready, State.Close) + || STATE_UPDATER.compareAndSet(this, State.None, State.Close) + || STATE_UPDATER.compareAndSet(this, State.Initializing, State.Close)); + } + + public boolean checkIfReady() { + return STATE_UPDATER.get(this) == State.Ready; + } + + public State getState() { + return STATE_UPDATER.get(this); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/main/proto/TransactionPendingAck.proto b/pulsar-broker/src/main/proto/TransactionPendingAck.proto new file mode 100644 index 0000000000000..fd9bb6a991405 --- /dev/null +++ b/pulsar-broker/src/main/proto/TransactionPendingAck.proto @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +syntax = "proto2"; + +import "pulsar-common/src/main/proto/PulsarApi.proto"; +package pulsar.proto; +option java_package = "org.apache.pulsar.broker.transaction.pendingack.proto"; +option optimize_for = LITE_RUNTIME; + +enum PendingAckOp { + ACK = 1; + COMMIT = 2; + ABORT = 3; +} + +message PendingAckMetadata { + required uint64 ledgerId = 1; + required uint64 entryId = 2; + repeated int64 ack_set = 3; + optional int32 batch_size = 4; +} + +message PendingAckMetadataEntry { + optional PendingAckOp pending_ack_op = 1; + optional pulsar.proto.CommandAck.AckType ack_type = 2; + optional uint64 txnid_least_bits = 3; + optional uint64 txnid_most_bits = 4; + repeated PendingAckMetadata pending_ack_metadata = 5; +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 2fa43b77cf5cc..e3a8161ce653b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -41,7 +41,6 @@ import static org.testng.Assert.fail; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -574,8 +573,8 @@ public void testMaxSameAddressProducers() throws Exception { PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); - InetAddress address1 = InetAddress.getLoopbackAddress(); - InetAddress address2 = InetAddress.getLocalHost(); + InetAddress address1 = InetAddress.getByName("127.0.0.1"); + InetAddress address2 = InetAddress.getByName("0.0.0.0"); String ipAddress1 = address1.getHostAddress(); String ipAddress2 = address2.getHostAddress(); @@ -748,10 +747,10 @@ public void testAddRemoveConsumer() throws Exception { // 2. duplicate add consumer try { - sub.addConsumer(consumer); + sub.addConsumer(consumer).get(); fail("Should fail with ConsumerBusyException"); - } catch (BrokerServiceException e) { - assertTrue(e instanceof BrokerServiceException.ConsumerBusyException); + } catch (Exception e) { + assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); } // 3. simple remove consumer @@ -827,9 +826,9 @@ private void testMaxConsumersShared() throws Exception { Consumer consumer3 = new Consumer(sub, SubType.Shared, topic.getName(), 3 /* consumer id */, 0, "Cons3"/* consumer name */, 50000, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, InitialPosition.Latest, null); - addConsumerToSubscription.invoke(topic, sub, consumer3); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub, consumer3)).get(); fail("should have failed"); - } catch (InvocationTargetException e) { + } catch (ExecutionException e) { assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); } @@ -851,9 +850,9 @@ private void testMaxConsumersShared() throws Exception { Consumer consumer5 = new Consumer(sub2, SubType.Shared, topic.getName(), 5 /* consumer id */, 0, "Cons5"/* consumer name */, 50000, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, InitialPosition.Latest, null); - addConsumerToSubscription.invoke(topic, sub2, consumer5); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer5)).get(); fail("should have failed"); - } catch (InvocationTargetException e) { + } catch (ExecutionException e) { assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); } } @@ -922,9 +921,9 @@ private void testMaxConsumersFailover() throws Exception { Consumer consumer3 = new Consumer(sub, SubType.Failover, topic.getName(), 3 /* consumer id */, 0, "Cons3"/* consumer name */, 50000, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, InitialPosition.Latest, null); - addConsumerToSubscription.invoke(topic, sub, consumer3); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub, consumer3)).get(); fail("should have failed"); - } catch (InvocationTargetException e) { + } catch (ExecutionException e) { assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); } @@ -946,9 +945,9 @@ private void testMaxConsumersFailover() throws Exception { Consumer consumer5 = new Consumer(sub2, SubType.Failover, topic.getName(), 5 /* consumer id */, 0, "Cons5"/* consumer name */, 50000, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, InitialPosition.Latest, null); - addConsumerToSubscription.invoke(topic, sub2, consumer5); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer5)).get(); fail("should have failed"); - } catch (InvocationTargetException e) { + } catch (ExecutionException e) { assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); } } @@ -1007,8 +1006,8 @@ public void testMaxSameAddressConsumers() throws Exception { PersistentSubscription sub1 = new PersistentSubscription(topic, "sub1", cursorMock, false); PersistentSubscription sub2 = new PersistentSubscription(topic, "sub2", cursorMock, false); - InetAddress address1 = InetAddress.getLoopbackAddress(); - InetAddress address2 = InetAddress.getLocalHost(); + InetAddress address1 = InetAddress.getByName("127.0.0.1"); + InetAddress address2 = InetAddress.getByName("0.0.0.0"); String ipAddress1 = address1.getHostAddress(); String ipAddress2 = address2.getHostAddress(); @@ -1026,14 +1025,14 @@ public void testMaxSameAddressConsumers() throws Exception { // 1. add consumer1 with ipAddress1 to sub1 Consumer consumer1 = getMockedConsumerWithSpecificAddress(topic, sub1, 1, address1); - addConsumerToSubscription.invoke(topic, sub1, consumer1); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub1, consumer1)).get(); assertEquals(topic.getNumberOfConsumers(), 1); assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress1), 1); assertEquals(sub1.getNumberOfSameAddressConsumers(ipAddress1), 1); // 2. add consumer2 with ipAddress1 to sub2 Consumer consumer2 = getMockedConsumerWithSpecificAddress(topic, sub2, 2, address1); - addConsumerToSubscription.invoke(topic, sub2, consumer2); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer2)).get(); assertEquals(topic.getNumberOfConsumers(), 2); assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress1), 2); assertEquals(sub1.getNumberOfSameAddressConsumers(ipAddress1), 1); @@ -1041,7 +1040,7 @@ public void testMaxSameAddressConsumers() throws Exception { // 3. add consumer3 with ipAddress2 to sub1 Consumer consumer3 = getMockedConsumerWithSpecificAddress(topic, sub1, 3, address2); - addConsumerToSubscription.invoke(topic, sub1, consumer3); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub1, consumer3)).get(); assertEquals(topic.getNumberOfConsumers(), 3); assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress1), 2); assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress2), 1); @@ -1050,7 +1049,7 @@ public void testMaxSameAddressConsumers() throws Exception { // 4. add consumer4 with ipAddress2 to sub2 Consumer consumer4 = getMockedConsumerWithSpecificAddress(topic, sub2, 4, address2); - addConsumerToSubscription.invoke(topic, sub2, consumer4); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer4)).get(); assertEquals(topic.getNumberOfConsumers(), 4); assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress1), 2); assertEquals(topic.getNumberOfSameAddressConsumers(ipAddress2), 2); @@ -1060,9 +1059,10 @@ public void testMaxSameAddressConsumers() throws Exception { // 5. add consumer5 with ipAddress1 to sub1 but reach maxSameAddressConsumersPerTopic try { Consumer consumer5 = getMockedConsumerWithSpecificAddress(topic, sub1, 5, address1); - addConsumerToSubscription.invoke(topic, sub1, consumer5); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub1, consumer5)).get(); + fail("should have failed"); - } catch (InvocationTargetException e) { + } catch (ExecutionException e) { assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); } assertEquals(topic.getNumberOfConsumers(), 4); @@ -1072,9 +1072,9 @@ public void testMaxSameAddressConsumers() throws Exception { // 6. add consumer6 with ipAddress2 to sub2 but reach maxSameAddressConsumersPerTopic try { Consumer consumer6 = getMockedConsumerWithSpecificAddress(topic, sub2, 6, address2); - addConsumerToSubscription.invoke(topic, sub2, consumer6); + ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer6)).get(); fail("should have failed"); - } catch (InvocationTargetException e) { + } catch (ExecutionException e) { assertTrue(e.getCause() instanceof BrokerServiceException.ConsumerBusyException); } assertEquals(topic.getNumberOfConsumers(), 4); @@ -1122,10 +1122,12 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { try { Thread.sleep(10); /* delay to ensure that the ubsubscribe gets executed first */ - new Consumer(sub, SubType.Exclusive, topic.getName(), 2 /* consumer id */, 0, "Cons2"/* consumer name */, - 50000, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, InitialPosition.Latest, null); - } catch (BrokerServiceException e) { - assertTrue(e instanceof BrokerServiceException.SubscriptionFencedException); + sub.addConsumer(new Consumer(sub, SubType.Exclusive, topic.getName(), 2 /* consumer id */, + 0, "Cons2"/* consumer name */, 50000, serverCnx, + "myrole-1", Collections.emptyMap(), false /* read compacted */, InitialPosition.Latest, null)).get(); + fail(); + } catch (Exception e) { + assertTrue(e.getCause() instanceof BrokerServiceException.SubscriptionFencedException); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java index 5a17eadeab51a..65b6bd945a32d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/TransactionMarkerDeleteTest.java @@ -38,6 +38,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.api.proto.MarkersMessageIdData; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.Markers; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -70,6 +71,7 @@ public void testTransactionMarkerDelete() throws Exception { doReturn(configuration).when(pulsarService).getConfig(); doReturn(true).when(configuration).isTransactionCoordinatorEnabled(); doReturn(managedLedger).when(topic).getManagedLedger(); + doReturn(TopicName.TRANSACTION_COORDINATOR_ASSIGN.getLocalName()).when(topic).getName(); ManagedCursor cursor = managedLedger.openCursor("test"); PersistentSubscription persistentSubscription = new PersistentSubscription(topic, "test", managedLedger.openCursor("test"), false); @@ -100,7 +102,7 @@ public void testMarkerDeleteTimes() throws Exception { doReturn(brokerService).when(topic).getBrokerService(); doReturn(pulsarService).when(brokerService).getPulsar(); doReturn(configuration).when(pulsarService).getConfig(); - doReturn(true).when(configuration).isTransactionCoordinatorEnabled(); + doReturn(false).when(configuration).isTransactionCoordinatorEnabled(); doReturn(managedLedger).when(topic).getManagedLedger(); ManagedCursor cursor = managedLedger.openCursor("test"); PersistentSubscription persistentSubscription = spy(new PersistentSubscription(topic, "test", diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java index bfa678ae104d3..ba4e14d7422f0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentSubscriptionTest.java @@ -31,6 +31,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -40,7 +41,8 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; - +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; @@ -60,7 +62,10 @@ import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.PersistentTopicTest; import org.apache.pulsar.broker.transaction.buffer.impl.InMemTransactionBufferProvider; -import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferProvider; +import org.apache.pulsar.broker.transaction.pendingack.PendingAckStore; +import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; +import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; +import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleState; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.proto.CommandAck.AckType; import org.apache.pulsar.common.api.proto.CommandSubscribe; @@ -71,6 +76,7 @@ import org.apache.pulsar.zookeeper.ZooKeeperCache; import org.apache.pulsar.zookeeper.ZooKeeperDataCache; import org.apache.zookeeper.ZooKeeper; +import org.awaitility.Awaitility; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.slf4j.Logger; @@ -114,6 +120,48 @@ public void setup() throws Exception { svcConfig.setTransactionCoordinatorEnabled(true); pulsarMock = spy(new PulsarService(svcConfig)); doReturn(new InMemTransactionBufferProvider()).when(pulsarMock).getTransactionBufferProvider(); + doReturn(new TransactionPendingAckStoreProvider() { + @Override + public CompletableFuture newPendingAckStore(PersistentSubscription subscription) { + return CompletableFuture.completedFuture(new PendingAckStore() { + @Override + public void replayAsync(PendingAckHandleImpl pendingAckHandle, ScheduledExecutorService executorService) { + try { + Field field = PendingAckHandleState.class.getDeclaredField("state"); + field.setAccessible(true); + field.set(pendingAckHandle, PendingAckHandleState.State.Ready); + } catch (NoSuchFieldException | IllegalAccessException e) { + fail(); + } + } + + @Override + public CompletableFuture closeAsync() { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendIndividualAck(TxnID txnID, List> positions) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendCumulativeAck(TxnID txnID, PositionImpl position) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendCommitMark(TxnID txnID, AckType ackType) { + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture appendAbortMark(TxnID txnID, AckType ackType) { + return CompletableFuture.completedFuture(null); + } + }); + } + }).when(pulsarMock).getTransactionPendingAckStoreProvider(); doReturn(svcConfig).when(pulsarMock).getConfiguration(); doReturn(mock(Compactor.class)).when(pulsarMock).getCompactor(); @@ -236,8 +284,14 @@ public void testCanAcknowledgeAndAbortForTransaction() throws BrokerServiceExcep }).when(cursorMock).asyncDelete(any(List.class), any(AsyncCallbacks.DeleteCallback.class), any()); doReturn(CommandSubscribe.SubType.Exclusive).when(consumerMock).subType(); - - persistentSubscription.addConsumer(consumerMock); + Awaitility.await().until(() -> { + try { + persistentSubscription.addConsumer(consumerMock); + return true; + } catch (Exception e) { + return false; + } + }); // Single ack for txn1 persistentSubscription.transactionIndividualAcknowledge(txnID1, positionsPair); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java index f1dd27fe6ebcb..644ea9e3326dc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionConsumeTest.java @@ -45,6 +45,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.protocol.Commands; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -105,6 +106,9 @@ public void noSortedTest() throws Exception { .subscriptionType(SubscriptionType.Shared) .subscribe(); + Awaitility.await().until(exclusiveConsumer::isConnected); + Awaitility.await().until(sharedConsumer::isConnected); + long mostSigBits = 2L; long leastSigBits = 5L; TxnID txnID = new TxnID(mostSigBits, leastSigBits); @@ -180,6 +184,8 @@ public void sortedTest() throws Exception { .subscriptionName("shared-test") .subscriptionType(SubscriptionType.Shared) .subscribe(); + Awaitility.await().until(exclusiveConsumer::isConnected); + Awaitility.await().until(sharedConsumer::isConnected); long mostSigBits = 2L; long leastSigBits = 5L; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java index e2836c6cb72ae..a5bae35d6ec48 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java @@ -59,6 +59,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.protocol.Commands; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -279,6 +280,8 @@ public void ackCommitTest() throws Exception { .subscriptionType(SubscriptionType.Shared) .subscribe(); + Awaitility.await().until(consumer::isConnected); + for (int i = 0; i < incomingMessageCnt; i++) { Message message = consumer.receive(); log.info("receive messageId: {}", message.getMessageId()); @@ -340,6 +343,7 @@ public void ackAbortTest() throws Exception { .enableBatchIndexAcknowledgment(true) .subscriptionType(SubscriptionType.Shared) .subscribe(); + Awaitility.await().until(consumer::isConnected); for (int i = 0; i < incomingMessageCnt; i++) { Message message = consumer.receive(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java index cf55cf956dadc..db7d1a81da81e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckInMemoryDeleteTest.java @@ -33,6 +33,7 @@ import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.SubscriptionType; @@ -215,9 +216,6 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { .subscriptionName(subscriptionName) .enableBatchIndexAcknowledgment(true) .subscriptionType(SubscriptionType.Shared) - .isAckReceiptEnabled(true) - .ackTimeout(2, TimeUnit.SECONDS) - .acknowledgmentGroupTime(0, TimeUnit.MICROSECONDS) .subscribe(); @Cleanup @@ -227,15 +225,12 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { .batchingMaxMessages(200) .create(); - PersistentSubscription persistentSubscription = null; PendingAckHandleImpl pendingAckHandle = null; LinkedMap> individualAckOfTransaction = null; ManagedCursorImpl managedCursor = null; - ConcurrentSkipListMap batchDeletedIndexes = null; - - Message[] messages = new Message[2]; + MessageId[] messageIds = new MessageId[2]; for (int retryCnt = 0; retryCnt < 2; retryCnt++) { int messageCnt = 1000; @@ -249,15 +244,20 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { Transaction commitTxn = getTxn(); //send 1000 and ack 999, and test the consumer pending ack has already clear 999 messages - for (int i = 0; i < messageCnt - 1; i++) { + for (int i = 0; i < messageCnt; i++) { message = consumer.receive(2, TimeUnit.SECONDS); Assert.assertNotNull(message); - if (i % 2 == 0) { - consumer.acknowledgeAsync(message.getMessageId(), commitTxn).get(); - log.info("txn receive msgId: {}, count: {}", message.getMessageId(), i); + // in order to free up 2 position to judge the consumer pending ack delete + if (i != 500) { + if (i % 2 == 0) { + consumer.acknowledgeAsync(message.getMessageId(), commitTxn).get(); + log.info("txn receive msgId: {}, count: {}", message.getMessageId(), i); + } else { + consumer.acknowledge(message.getMessageId()); + log.info("normal receive msgId: {}, count: {}", message.getMessageId(), i); + } } else { - consumer.acknowledge(message.getMessageId()); - log.info("normal receive msgId: {}, count: {}", message.getMessageId(), i); + messageIds[retryCnt] = message.getMessageId(); } } @@ -273,30 +273,54 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { if (completableFuture != null) { Optional topic = completableFuture.get(); if (topic.isPresent()) { - persistentSubscription = (PersistentSubscription) topic.get().getSubscription(subscriptionName); + PersistentSubscription testPersistentSubscription = + (PersistentSubscription) topic.get().getSubscription(subscriptionName); field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); field.setAccessible(true); - pendingAckHandle = (PendingAckHandleImpl) field.get(persistentSubscription); + pendingAckHandle = (PendingAckHandleImpl) field.get(testPersistentSubscription); field = PendingAckHandleImpl.class.getDeclaredField("individualAckOfTransaction"); field.setAccessible(true); individualAckOfTransaction = (LinkedMap>) field.get(pendingAckHandle); assertTrue(individualAckOfTransaction.isEmpty()); - managedCursor = (ManagedCursorImpl) persistentSubscription.getCursor(); + managedCursor = (ManagedCursorImpl) testPersistentSubscription.getCursor(); field = ManagedCursorImpl.class.getDeclaredField("batchDeletedIndexes"); field.setAccessible(true); - batchDeletedIndexes = + final ConcurrentSkipListMap batchDeletedIndexes = (ConcurrentSkipListMap) field.get(managedCursor); if (retryCnt == 0) { //one message are not ack + Awaitility.await().until(() -> { + return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 1; + }); + assertEquals(batchDeletedIndexes.size(), 1); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1); - messages[0] = consumer.receive(); + assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1); } else { //two message are not ack - assertEquals(batchDeletedIndexes.size(), 2); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 2); - messages[1] = consumer.receive(); + Awaitility.await().until(() -> { + return testPersistentSubscription.getConsumers().get(0).getPendingAcks().size() == 2; + }); + + Transaction commitTwice = getTxn(); + + //this message is in one batch point + consumer.acknowledge(messageIds[0]); + Awaitility.await().until(() -> { + return batchDeletedIndexes.size() == 1; + }); + assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1); + + // this test is for the last message has been cleared in this consumer pending acks + // and it won't clear the last message in cursor batch index ack set + consumer.acknowledgeAsync(messageIds[1], commitTwice).get(); + assertEquals(batchDeletedIndexes.size(), 1); + assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0); + + // the messages has been produced were all acked, the memory in broker for the messages has been cleared. + commitTwice.commit().get(); + assertEquals(batchDeletedIndexes.size(), 0); + assertEquals(testPersistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0); } count++; } @@ -304,24 +328,6 @@ public void txnAckTestBatchAndSharedSubMemoryDeleteTest() throws Exception { } assertEquals(count, 1); } - - Transaction commitTwice = getTxn(); - - //this message is in one batch point - consumer.acknowledge(messages[0].getMessageId()); - assertEquals(batchDeletedIndexes.size(), 1); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 1); - - // this test is for the last message has been cleared in this consumer pending acks - // and it won't clear the last message in cursor batch index ack set - consumer.acknowledgeAsync(messages[1].getMessageId(), commitTwice).get(); - assertEquals(batchDeletedIndexes.size(), 1); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0); - - // the messages has been produced were all acked, the memory in broker for the messages has been cleared. - commitTwice.commit().get(); - assertEquals(batchDeletedIndexes.size(), 0); - assertEquals(persistentSubscription.getConsumers().get(0).getPendingAcks().size(), 0); } private Transaction getTxn() throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java new file mode 100644 index 0000000000000..da2442675b5d5 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -0,0 +1,301 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.pendingack; + +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; +import com.google.common.collect.Sets; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.TransactionTestBase; +import org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore; +import org.apache.pulsar.broker.transaction.pendingack.impl.PendingAckHandleImpl; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Test for consuming transaction messages. + */ +@Slf4j +public class PendingAckPersistentTest extends TransactionTestBase { + + private final static String PENDING_ACK_REPLAY_TOPIC = "persistent://public/txn/pending-ack-replay"; + + @BeforeMethod + public void setup() throws Exception { + setBrokerCount(1); + super.internalSetup(); + + String[] brokerServiceUrlArr = getPulsarServiceList().get(0).getBrokerServiceUrl().split(":"); + String webServicePort = brokerServiceUrlArr[brokerServiceUrlArr.length -1]; + admin.clusters().createCluster(CLUSTER_NAME, new ClusterData("http://localhost:" + webServicePort)); + admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), + new TenantInfo(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); + admin.topics().createPartitionedTopic(TopicName.TRANSACTION_COORDINATOR_ASSIGN.toString(), 16); + admin.tenants().createTenant("public", + new TenantInfo(Sets.newHashSet(), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace("public/txn", 10); + admin.topics().createNonPartitionedTopic(PENDING_ACK_REPLAY_TOPIC); + + pulsarClient = PulsarClient.builder() + .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) + .statsInterval(0, TimeUnit.SECONDS) + .enableTransaction(true) + .build(); + + Thread.sleep(1000 * 3); + } + + @AfterMethod(alwaysRun = true) + protected void cleanup() { + super.internalCleanup(); + } + + @Test + public void individualPendingAckReplayTest() throws Exception { + int messageCount = 1000; + String subName = "individual-test"; + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(PENDING_ACK_REPLAY_TOPIC) + .enableBatching(true) + .batchingMaxMessages(200) + .create(); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(PENDING_ACK_REPLAY_TOPIC) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Shared) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + Transaction abortTxn = pulsarClient.newTransaction() + .withTransactionTimeout(30, TimeUnit.SECONDS).build().get(); + + List pendingAckMessageIds = new ArrayList<>(); + List normalAckMessageIds = new ArrayList<>(); + for (int i = 0; i < messageCount; i++) { + producer.send("Hello Pulsar!".getBytes()); + Message message = consumer.receive(); + if (i % 2 == 0) { + consumer.acknowledgeAsync(message.getMessageId(), abortTxn).get(); + pendingAckMessageIds.add(message.getMessageId()); + } else { + normalAckMessageIds.add(message.getMessageId()); + } + } + + //in order to test pending ack replay + admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); + Awaitility.await().until(consumer::isConnected); + Transaction commitTxn = pulsarClient.newTransaction() + .withTransactionTimeout(30, TimeUnit.SECONDS).build().get(); + + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(30, TimeUnit.SECONDS).build().get(); + + // this messageIds are ack by transaction + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + try { + consumer.acknowledgeAsync(pendingAckMessageIds.get(i), txn).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + } + // this messageIds are not ack by transaction + for (int i = 0; i < normalAckMessageIds.size(); i++) { + consumer.acknowledgeAsync(normalAckMessageIds.get(i), commitTxn).get(); + } + + txn.abort().get(); + // commit this txn , normalAckMessageIds are in pending ack state + commitTxn.commit().get(); + // abort this txn, pendingAckMessageIds are delete from pending ack state + abortTxn.abort().get(); + + // replay this pending ack + admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); + Awaitility.await().until(consumer::isConnected); + + abortTxn = pulsarClient.newTransaction() + .withTransactionTimeout(30, TimeUnit.SECONDS).build().get(); + + commitTxn = pulsarClient.newTransaction() + .withTransactionTimeout(30, TimeUnit.SECONDS).build().get(); + // normalAckMessageIds are ack and then commit, so ack fail + for (int i = 0; i < normalAckMessageIds.size(); i++) { + try { + consumer.acknowledgeAsync(normalAckMessageIds.get(i), abortTxn).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + } + + // pendingAckMessageIds are all abort, so can ack again + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + consumer.acknowledgeAsync(pendingAckMessageIds.get(i), commitTxn).get(); + } + + abortTxn.abort().get(); + commitTxn.commit().get(); + + PersistentTopic topic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() + .getTopic(TopicName.get(PENDING_ACK_REPLAY_TOPIC).toString(), false).get().get(); + Field field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); + field.setAccessible(true); + PendingAckHandleImpl pendingAckHandle = + (PendingAckHandleImpl) field.get(topic.getSubscription(subName)); + field = PendingAckHandleImpl.class.getDeclaredField("pendingAckStoreFuture"); + field.setAccessible(true); + CompletableFuture pendingAckStoreCompletableFuture = + (CompletableFuture) field.get(pendingAckHandle); + pendingAckStoreCompletableFuture.get(); + + field = MLPendingAckStore.class.getDeclaredField("cursor"); + field.setAccessible(true); + + ManagedCursor managedCursor = (ManagedCursor) field.get(pendingAckStoreCompletableFuture.get()); + + // in order to check out the pending ack cursor is clear whether or not. + Awaitility.await() + .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition()) + .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == -1); + } + + @Test + public void cumulativePendingAckReplayTest() throws Exception { + int messageCount = 1000; + String subName = "cumulative-test"; + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(PENDING_ACK_REPLAY_TOPIC) + .enableBatching(true) + .batchingMaxMessages(200) + .create(); + + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(PENDING_ACK_REPLAY_TOPIC) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Failover) + .enableBatchIndexAcknowledgment(true) + .subscribe(); + + Transaction abortTxn = pulsarClient.newTransaction() + .withTransactionTimeout(30, TimeUnit.SECONDS).build().get(); + + List pendingAckMessageIds = new ArrayList<>(); + for (int i = 0; i < messageCount; i++) { + producer.send("Hello Pulsar!".getBytes()); + } + + for (int i = 0; i < messageCount; i++) { + Message message = consumer.receive(); + pendingAckMessageIds.add(message.getMessageId()); + consumer.acknowledgeCumulativeAsync(message.getMessageId(), abortTxn).get(); + } + + admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(30, TimeUnit.SECONDS).build().get(); + + Awaitility.await().until(consumer::isConnected); + + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + try { + consumer.acknowledgeCumulativeAsync(pendingAckMessageIds.get(i), txn).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + } + Transaction commitTxn = pulsarClient.newTransaction() + .withTransactionTimeout(30, TimeUnit.SECONDS).build().get(); + abortTxn.abort().get(); + + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + consumer.acknowledgeCumulativeAsync(pendingAckMessageIds.get(i), commitTxn).get(); + } + commitTxn.commit().get(); + + admin.topics().unload(PENDING_ACK_REPLAY_TOPIC); + Awaitility.await().until(consumer::isConnected); + + for (int i = 0; i < pendingAckMessageIds.size(); i++) { + try { + consumer.acknowledgeCumulativeAsync(pendingAckMessageIds.get(i), txn).get(); + fail(); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof PulsarClientException.TransactionConflictException); + } + } + + PersistentTopic topic = (PersistentTopic) getPulsarServiceList().get(0).getBrokerService() + .getTopic(TopicName.get(PENDING_ACK_REPLAY_TOPIC).toString(), false).get().get(); + Field field = PersistentSubscription.class.getDeclaredField("pendingAckHandle"); + field.setAccessible(true); + PendingAckHandleImpl pendingAckHandle = + (PendingAckHandleImpl) field.get(topic.getSubscription(subName)); + field = PendingAckHandleImpl.class.getDeclaredField("pendingAckStoreFuture"); + field.setAccessible(true); + CompletableFuture pendingAckStoreCompletableFuture = + (CompletableFuture) field.get(pendingAckHandle); + pendingAckStoreCompletableFuture.get(); + + field = MLPendingAckStore.class.getDeclaredField("cursor"); + field.setAccessible(true); + + ManagedCursor managedCursor = (ManagedCursor) field.get(pendingAckStoreCompletableFuture.get()); + + // in order to check out the pending ack cursor is clear whether or not. + Awaitility.await() + .until(() -> ((PositionImpl) managedCursor.getMarkDeletedPosition()) + .compareTo((PositionImpl) managedCursor.getManagedLedger().getLastConfirmedEntry()) == -1); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 1053efd512f62..58d87f6db6943 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -46,7 +46,6 @@ import org.apache.pulsar.broker.transaction.TransactionTestBase; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.PulsarClient; @@ -142,6 +141,7 @@ private void produceCommitTest(boolean enableBatch) throws Exception { .subscriptionName("test") .enableBatchIndexAcknowledgment(true) .subscribe(); + Awaitility.await().until(consumer::isConnected); ProducerBuilder producerBuilder = pulsarClient .newProducer() @@ -215,6 +215,7 @@ public void produceAbortTest() throws Exception { .subscriptionName(subName) .enableBatchIndexAcknowledgment(true) .subscribe(); + Awaitility.await().until(consumer::isConnected); // Can't receive transaction messages before abort. Message message = consumer.receive(2, TimeUnit.SECONDS); @@ -305,6 +306,7 @@ private void txnAckTest(boolean batchEnable, int maxBatchSize, .enableBatchIndexAcknowledgment(true) .subscriptionType(subscriptionType) .subscribe(); + Awaitility.await().until(consumer::isConnected); @Cleanup Producer producer = pulsarClient.newProducer() @@ -380,6 +382,7 @@ public void txnMessageAckTest() throws Exception { .enableBatchIndexAcknowledgment(true) .acknowledgmentGroupTime(0, TimeUnit.MILLISECONDS) .subscribe(); + Awaitility.await().until(consumer::isConnected); @Cleanup Producer producer = pulsarClient @@ -503,6 +506,7 @@ private void txnCumulativeAckTest(boolean batchEnable, int maxBatchSize, Subscri .subscriptionType(subscriptionType) .ackTimeout(1, TimeUnit.MINUTES) .subscribe(); + Awaitility.await().until(consumer::isConnected); @Cleanup Producer producer = pulsarClient.newProducer() @@ -649,6 +653,7 @@ public void txnMetadataHandlerRecoverTest() throws Exception { .subscriptionName("test") .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscribe(); + Awaitility.await().until(consumer::isConnected); for (int i = 0; i < txnCnt * messageCnt; i++) { Message message = consumer.receive(); @@ -665,6 +670,7 @@ public void produceTxnMessageOrderTest() throws Exception { .topic(topic) .subscriptionName("test") .subscribe(); + Awaitility.await().until(consumer::isConnected); @Cleanup Producer producer = pulsarClient.newProducer() diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java index 3cc05be8d99a6..20110cd544ae5 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/transaction/TransactionTest.java @@ -30,6 +30,7 @@ import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; +import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.Test; @@ -99,6 +100,7 @@ public void transferNormalTest(Supplier serviceUrl) throws Exception { .subscriptionType(SubscriptionType.Shared) .enableBatchIndexAcknowledgment(true) .subscribe(); + Awaitility.await().until(transferConsumer::isConnected); log.info("transfer consumer create finished"); @Cleanup @@ -114,6 +116,7 @@ public void transferNormalTest(Supplier serviceUrl) throws Exception { .subscriptionName("integration-test") .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) .subscribe(); + Awaitility.await().until(balanceUpdateConsumer::isConnected); log.info("balance update consumer create finished"); while (true) {