From eaf5d25cf8df8aef27f2bf1e561b7a44e96578ea Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Tue, 25 Oct 2022 00:44:10 +0800 Subject: [PATCH 01/32] Add Processor for snapshotSegment --- .../buffer/AbortedTxnProcessor.java | 49 +++ ...SingleSnapshotAbortedTxnProcessorImpl.java | 4 + ...napshotSegmentAbortedTxnProcessorImpl.java | 408 ++++++++++++++++++ 3 files changed, 461 insertions(+) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java new file mode 100644 index 0000000000000..a7e9654c5667d --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -0,0 +1,49 @@ +package org.apache.pulsar.broker.transaction.buffer; + +import java.util.concurrent.CompletableFuture; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferRecoverCallBack; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; + +public interface AbortedTxnProcessor { + + /** + * After the transaction buffer writes a transaction aborted mark to the topic, + * the transaction buffer will add the aborted transaction ID to AbortedTxnProcessor. + * @param abortedTxnId aborted transaction ID. + */ + void appendAbortedTxn(TxnIDData abortedTxnId, PositionImpl position); + + /** + * After the transaction buffer writes a transaction aborted mark to the topic, + * the transaction buffer will update max read position in AbortedTxnProcessor + * @param maxReadPosition the Max read position after the transaction is aborted. + */ + void updateMaxReadPosition(Position maxReadPosition); + + + /** + * Pulsar has a configuration for ledger retention time. + * If the transaction aborted mark position has been deleted, the transaction is valid and can be clear. + * In the old implementation we clear the invalid aborted txn ID one by one. + * In the new implementation, we adopt snapshot segments. And then we clear invalid segment by its max read position. + */ + void trimSnapshotSegments(); + + /** + * Check whether the transaction ID is an aborted transaction ID. + * @param txnID the transaction ID that needs to be checked. + * @param readPosition the read position of the transaction message, can be used to find the segment. + * @return a boolean, whether the transaction ID is an aborted transaction ID. + */ + boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition); + + /** + * Recover transaction buffer by transaction buffer snapshot. + * @return a pair consists of a Boolean if the transaction buffer needs to recover and a Position (startReadCursorPosition) determiner where to start to recover in the original topic. + */ + + CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack); + +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java new file mode 100644 index 0000000000000..76c617bd0af1b --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -0,0 +1,4 @@ +package org.apache.pulsar.broker.transaction.buffer.impl; + +public class SingleSnapshotAbortedTxnProcessorImpl { +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java new file mode 100644 index 0000000000000..325ad876a1ef0 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -0,0 +1,408 @@ +package org.apache.pulsar.broker.transaction.buffer.impl; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.util.Timeout; +import io.netty.util.Timer; +import io.netty.util.TimerTask; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.systopic.SystemTopicClient; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.apache.pulsar.common.events.EventType; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.FutureUtil; + +@Slf4j +public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor, TimerTask { + private final AtomicLong sequenceID = new AtomicLong(0); + + //TODO: recover this at recover processor. + private final ConcurrentSkipListMap> aborts + = new ConcurrentSkipListMap<>(); + private final ConcurrentSkipListMap> snapshotSegmentQueue + = new ConcurrentSkipListMap<>(); + + private final ConcurrentSkipListMap indexes + = new ConcurrentSkipListMap<>(); + + private TransactionBufferSnapshotIndexes theLatestSnapshotIndexes; + private final Timer timer; + private PositionImpl maxReadPosition; + + private final PersistentTopic topic; + + private CopyOnWriteArrayList lastAbortedTxnIDs = new CopyOnWriteArrayList<>(); + + //When add abort or change max read position, the count will +1. Take snapshot will set 0 into it. + private final AtomicLong changeMaxReadPositionAndAddAbortTimes = new AtomicLong(); + + private volatile long lastSnapshotTimestamps; + + private final int takeSnapshotIntervalNumber; + + private final int takeSnapshotIntervalTime; + + private final int transactionBufferMaxAbortedTxnsOfSnapshotSegment; + + private final Semaphore segmentUpdateSemaphore = new Semaphore(1); + + private final CompletableFuture> + snapshotSegmentsWriterFuture; + private final CompletableFuture> + snapshotIndexWriterFuture; + + + public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic, int takeSnapshotIntervalNumber, int takeSnapshotIntervalTime, + int transactionBufferMaxAbortedTxnsOfSnapshotSegment) { + this.topic = topic; + this.takeSnapshotIntervalTime = takeSnapshotIntervalTime; + this.takeSnapshotIntervalNumber = takeSnapshotIntervalNumber; + this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = transactionBufferMaxAbortedTxnsOfSnapshotSegment; + snapshotSegmentsWriterFuture = this.topic.getBrokerService().getPulsar() + .getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName())); + snapshotIndexWriterFuture = this.topic.getBrokerService().getPulsar() + .getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName())); + + this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); + } + + @Override + public void appendAbortedTxn(TxnIDData abortedTxnId, PositionImpl position) { + lastAbortedTxnIDs.add(abortedTxnId); + //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. + if (lastAbortedTxnIDs.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { + aborts.put(position, lastAbortedTxnIDs); + //Guarantee the order of the segments. + snapshotSegmentQueue.put(position, lastAbortedTxnIDs); + takeSnapshotSegment(); + lastAbortedTxnIDs = new CopyOnWriteArrayList<>(); + } + } + + private void takeSnapshotSegment() { + //Only one segment can be written at the same time. + if (segmentUpdateSemaphore.tryAcquire()) { + CopyOnWriteArrayList abortedTxns = + (CopyOnWriteArrayList) snapshotSegmentQueue.firstEntry().getValue(); + PositionImpl maxReadPosition = snapshotSegmentQueue.firstKey(); + + takeSnapshotSegmentAsync(abortedTxns, maxReadPosition).thenRun(() -> { + if (log.isDebugEnabled()) { + log.debug("Successes to take snapshot segment [{}] at maxReadPosition [{}] " + + "for the topic [{}], and the size of the segment is [{}]", + sequenceID, maxReadPosition, topic.getName(), abortedTxns.size()); + } + sequenceID.getAndIncrement(); + }).exceptionally(e -> { + //Just log the error, and the processor will try to take snapshot again when the transactionBuffer + //append aborted txn nex time. + log.error("Failed to take snapshot segment [{}] at maxReadPosition [{}] " + + "for the topic [{}], and the size of the segment is [{}]", + sequenceID, maxReadPosition, topic.getName(), abortedTxns.size(), e); + //Try again + timer.newTimeout((ignore) -> takeSnapshotSegment(), takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); + return null; + }); + } + } + + + @Override + public void updateMaxReadPosition(Position position) { + this.maxReadPosition = (PositionImpl) position; + updateSnapshotMetadataByChangeTimes(); + } + + @Override + public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { + List txnIDSet = aborts.ceilingEntry((PositionImpl) readPosition).getValue(); + if (txnIDSet == null) { + return lastAbortedTxnIDs.contains(txnID); + } else { + return txnIDSet.contains(txnID); + } + } + + @Override + public void trimSnapshotSegments() { + //Checking whether there are some segment expired. + while (!aborts.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) + .ledgerExists(aborts.firstKey().getLedgerId())) { + if (log.isDebugEnabled()) { + log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", + topic.getName(), aborts.firstKey()); + } + PositionImpl positionNeedToDelete = aborts.firstKey(); + long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); + snapshotSegmentsWriterFuture.thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) + .thenRun(() -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Successes to delete the snapshot segment, " + + "whose sequenceId is [{}] and maxReadPosition is [{}]", + this.topic.getName(), this.sequenceID, this.maxReadPosition); + } + aborts.remove(positionNeedToDelete); + indexes.remove(positionNeedToDelete); + //TODO: check whether the snapshot segment is null, and update index. + updateSnapshotIndex(); + }).exceptionally(e -> { + log.error("[{}] Failed to delete the snapshot segment, " + + "whose sequenceId is [{}] and maxReadPosition is [{}]", + this.topic.getName(), this.sequenceID, this.maxReadPosition); + return null; + }); + } + } + + private String buildKey(long sequenceId) { + return "multiple-" + sequenceId + this.topic.getName(); + } + + private void updateSnapshotMetadataByChangeTimes() { + if (this.changeMaxReadPositionAndAddAbortTimes.incrementAndGet() == takeSnapshotIntervalNumber) { + changeMaxReadPositionAndAddAbortTimes.set(0); + updateIndexMetadataForTheLastSnapshot(); + } + } + + private void takeSnapshotByTimeout() { + if (changeMaxReadPositionAndAddAbortTimes.get() > 0) { + updateIndexMetadataForTheLastSnapshot(); + } + timer.newTimeout(SnapshotSegmentAbortedTxnProcessorImpl.this, + takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); + } + + @Override + public void run(Timeout timeout) { + //TODO: Run the processor after transaction buffer ready. + takeSnapshotByTimeout(); + } + + private CompletableFuture takeSnapshotSegmentAsync(List segment, PositionImpl maxReadPosition) { + TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = new TransactionBufferSnapshotSegment(); + transactionBufferSnapshotSegment.setAborts(segment); + transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); + transactionBufferSnapshotSegment.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); + transactionBufferSnapshotSegment.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); + + return snapshotSegmentsWriterFuture.thenCompose(segmentWriter -> { + transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get()); + return segmentWriter.writeAsync(buildKey(this.sequenceID.get()), transactionBufferSnapshotSegment); + }).thenCompose((messageId) -> { + //Build index for this segment + TransactionBufferSnapshotIndex index = new TransactionBufferSnapshotIndex(); + index.setSequenceID(transactionBufferSnapshotSegment.getSequenceId()); + index.setMaxReadPositionLedgerID(maxReadPosition.getLedgerId()); + index.setMaxReadPositionEntryID(maxReadPosition.getEntryId()); + index.setPersistentPositionLedgerID(((MessageIdImpl) messageId).getLedgerId()); + index.setPersistentPositionEntryID(((MessageIdImpl) messageId).getEntryId()); + + indexes.put(maxReadPosition, index); + //update snapshot segment index. + return updateSnapshotIndex(); + }); + } + + //Update the indexes in the transactionBufferSnapshotIndexe. + //Concurrency control is performed by snapshotIndexWriterFuture. + private CompletableFuture updateSnapshotIndex() { + TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); + return snapshotIndexWriterFuture + .thenCompose((indexesWriter) -> { + snapshotIndexes.setIndexList(indexes.values().stream().toList()); + //Only update the index in indexes and keep the metadata in indexes unchanged. + snapshotIndexes.setSnapshot(theLatestSnapshotIndexes.getSnapshot()); + return indexesWriter.writeAsync(snapshotIndexes.getTopicName(), snapshotIndexes); + }) + .thenRun(() -> { + theLatestSnapshotIndexes.setIndexList(snapshotIndexes.getIndexList()); + }) + .exceptionally(e -> { + log.error("[{}] Failed to update snapshot segment index", snapshotIndexes.getTopicName(), e); + return null; + }); + } + + //Update the metadata in the transactionBufferSnapshotIndexes. + //Concurrency control is performed by snapshotIndexWriterFuture. + private void updateIndexMetadataForTheLastSnapshot() { + TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); + snapshotIndexWriterFuture + .thenCompose((indexesWriter) -> { + //Store the latest metadata + TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = + new TransactionBufferSnapshotSegment(); + transactionBufferSnapshotSegment.setAborts(lastAbortedTxnIDs); + indexes.setSnapshot(transactionBufferSnapshotSegment); + //Only update the metadata in indexes and keep the index in indexes unchanged. + indexes.setIndexList(theLatestSnapshotIndexes.getIndexList()); + return indexesWriter.writeAsync(indexes.getTopicName(), indexes); + }) + .thenRun(() -> { + theLatestSnapshotIndexes.setSnapshot(indexes.getSnapshot()); + }) + .exceptionally(e -> { + log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); + return null; + }); + } + + @Override + public CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack) { + return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotIndexService() + .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { + PositionImpl startReadCursorPosition = null; + CompletableFuture recoverSnapshotSegmentFuture = new CompletableFuture<>(); + try { + boolean hasIndex = false; + //Read Index to recover the sequenceID, indexes, lastAbortedTxns and maxReadPosition. + while (reader.hasMoreEvents()) { + Message message = reader.readNext(); + if (topic.getName().equals(message.getKey())) { + TransactionBufferSnapshotIndexes transactionBufferSnapshotIndexes = message.getValue(); + if (transactionBufferSnapshotIndexes != null) { + hasIndex = true; + this.theLatestSnapshotIndexes = transactionBufferSnapshotIndexes; + //TODO:take a snapshot when create producer + startReadCursorPosition = PositionImpl.get( + transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionLedgerId(), + transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); + } + } + } + closeReader(reader); + if (!hasIndex) { + callBack.noNeedToRecover(); + return null; + } else { + theLatestSnapshotIndexes.getIndexList().forEach(transactionBufferSnapshotIndex -> { + indexes.put(new PositionImpl(transactionBufferSnapshotIndex.persistentPositionLedgerID, + transactionBufferSnapshotIndex.persistentPositionEntryID), + transactionBufferSnapshotIndex); + }); + this.lastAbortedTxnIDs = (CopyOnWriteArrayList) theLatestSnapshotIndexes + .getSnapshot().getAborts(); + this.maxReadPosition = new PositionImpl(theLatestSnapshotIndexes + .getSnapshot().getMaxReadPositionLedgerId(), + theLatestSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); + sequenceID.set(indexes.lastEntry().getValue().sequenceID + 1); + } + //Read snapshot segment to recover aborts. + LinkedList> completableFutures = new LinkedList<>(); + AtomicLong invalidIndex = new AtomicLong(0); + AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks + .OpenReadOnlyManagedLedgerCallback() { + @Override + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { + theLatestSnapshotIndexes.getIndexList().forEach(index -> { + CompletableFuture completableFuture1 = new CompletableFuture<>(); + completableFutures.add(completableFuture1); + readOnlyManagedLedger.asyncReadEntry( + new PositionImpl(index.getPersistentPositionLedgerID(), + index.getPersistentPositionEntryID()), + new AsyncCallbacks.ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + //Remove invalid index + if (entry == null) { + indexes.remove(new PositionImpl( + index.getMaxReadPositionLedgerID(), + index.getMaxReadPositionEntryID())); + completableFuture1.complete(null); + invalidIndex.getAndIncrement(); + return; + } + handleSnapshotSegmentEntry(entry); + completableFuture1.complete(null); + } + + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + completableFuture1.completeExceptionally(exception); + } + }, null); + }); + } + + @Override + public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx) { + // + } + }; + + TopicName snapshotIndexTopicName = TopicName.get(TopicDomain.persistent.toString(), + TopicName.get(topic.getName()).getNamespaceObject(), + EventType.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS.toString()); + this.topic.getBrokerService().getPulsar().getManagedLedgerFactory() + .asyncOpenReadOnlyManagedLedger(snapshotIndexTopicName + .getPersistenceNamingEncoding(), callback, + topic.getManagedLedger().getConfig(), + null); + //Wait the processor recover completely and the allow TB to recover the messages + // after the startReadCursorPosition. + FutureUtil.waitForAll(completableFutures).get(); + if (invalidIndex.get() != 0) { + updateSnapshotIndex(); + } + return CompletableFuture.completedFuture(startReadCursorPosition); + } catch (Exception ex) { + log.error("[{}] Transaction buffer recover fail when read " + + "transactionBufferSnapshot!", topic.getName(), ex); + callBack.recoverExceptionally(ex); + closeReader(reader); + return null; + } + + }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this)); + } + + private void handleSnapshotSegmentEntry(Entry entry) { + //decode snapshot from entry + ByteBuf headersAndPayload = entry.getDataBuffer(); + //skip metadata + MessageMetadata msgMetadata = Commands.parseMessageMetadata(headersAndPayload); + TransactionBufferSnapshotSegment snapshotSegment = Schema.AVRO(TransactionBufferSnapshotSegment.class) + .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer()); + aborts.put(new PositionImpl(snapshotSegment.getMaxReadPositionLedgerId(), + snapshotSegment.getMaxReadPositionEntryId()), snapshotSegment.getAborts()); + + } + + private void closeReader(SystemTopicClient.Reader reader) { + reader.closeAsync().exceptionally(e -> { + log.error("[{}]Transaction buffer snapshot reader close error!", topic.getName(), e); + return null; + }); + } +} \ No newline at end of file From b097c21584156988abdf4430344a36c948a234fe Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Tue, 25 Oct 2022 08:04:26 +0800 Subject: [PATCH 02/32] optimize Processor for snapshotSegment --- ...napshotSegmentAbortedTxnProcessorImpl.java | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 325ad876a1ef0..22ce4baf8b7d5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -31,7 +31,6 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.MessageIdImpl; -import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; @@ -243,9 +242,7 @@ private CompletableFuture updateSnapshotIndex() { snapshotIndexes.setSnapshot(theLatestSnapshotIndexes.getSnapshot()); return indexesWriter.writeAsync(snapshotIndexes.getTopicName(), snapshotIndexes); }) - .thenRun(() -> { - theLatestSnapshotIndexes.setIndexList(snapshotIndexes.getIndexList()); - }) + .thenRun(() -> theLatestSnapshotIndexes.setIndexList(snapshotIndexes.getIndexList())) .exceptionally(e -> { log.error("[{}] Failed to update snapshot segment index", snapshotIndexes.getTopicName(), e); return null; @@ -267,9 +264,7 @@ private void updateIndexMetadataForTheLastSnapshot() { indexes.setIndexList(theLatestSnapshotIndexes.getIndexList()); return indexesWriter.writeAsync(indexes.getTopicName(), indexes); }) - .thenRun(() -> { - theLatestSnapshotIndexes.setSnapshot(indexes.getSnapshot()); - }) + .thenRun(() -> theLatestSnapshotIndexes.setSnapshot(indexes.getSnapshot())) .exceptionally(e -> { log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); return null; @@ -282,7 +277,6 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecov .getTxnBufferSnapshotIndexService() .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { PositionImpl startReadCursorPosition = null; - CompletableFuture recoverSnapshotSegmentFuture = new CompletableFuture<>(); try { boolean hasIndex = false; //Read Index to recover the sequenceID, indexes, lastAbortedTxns and maxReadPosition. @@ -305,11 +299,12 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecov callBack.noNeedToRecover(); return null; } else { - theLatestSnapshotIndexes.getIndexList().forEach(transactionBufferSnapshotIndex -> { - indexes.put(new PositionImpl(transactionBufferSnapshotIndex.persistentPositionLedgerID, - transactionBufferSnapshotIndex.persistentPositionEntryID), - transactionBufferSnapshotIndex); - }); + theLatestSnapshotIndexes.getIndexList() + .forEach(transactionBufferSnapshotIndex -> + indexes.put(new PositionImpl( + transactionBufferSnapshotIndex.persistentPositionLedgerID, + transactionBufferSnapshotIndex.persistentPositionEntryID), + transactionBufferSnapshotIndex)); this.lastAbortedTxnIDs = (CopyOnWriteArrayList) theLatestSnapshotIndexes .getSnapshot().getAborts(); this.maxReadPosition = new PositionImpl(theLatestSnapshotIndexes @@ -391,7 +386,7 @@ private void handleSnapshotSegmentEntry(Entry entry) { //decode snapshot from entry ByteBuf headersAndPayload = entry.getDataBuffer(); //skip metadata - MessageMetadata msgMetadata = Commands.parseMessageMetadata(headersAndPayload); + Commands.parseMessageMetadata(headersAndPayload); TransactionBufferSnapshotSegment snapshotSegment = Schema.AVRO(TransactionBufferSnapshotSegment.class) .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer()); aborts.put(new PositionImpl(snapshotSegment.getMaxReadPositionLedgerId(), From ce6d1efd0aa1e133d4e1b5373ec2e86630157a0b Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Tue, 25 Oct 2022 09:02:15 +0800 Subject: [PATCH 03/32] Add Processor for singleSnapshot --- .../buffer/AbortedTxnProcessor.java | 12 +- ...SingleSnapshotAbortedTxnProcessorImpl.java | 202 +++++++++++++++++- ...napshotSegmentAbortedTxnProcessorImpl.java | 10 +- 3 files changed, 213 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index a7e9654c5667d..80e07da3f2179 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -4,16 +4,16 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferRecoverCallBack; -import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; -public interface AbortedTxnProcessor { + +public interface AbortedTxnProcessor { /** * After the transaction buffer writes a transaction aborted mark to the topic, * the transaction buffer will add the aborted transaction ID to AbortedTxnProcessor. - * @param abortedTxnId aborted transaction ID. + * @param txnID aborted transaction ID. */ - void appendAbortedTxn(TxnIDData abortedTxnId, PositionImpl position); + void appendAbortedTxn(T txnID, PositionImpl position); /** * After the transaction buffer writes a transaction aborted mark to the topic, @@ -29,7 +29,7 @@ public interface AbortedTxnProcessor { * In the old implementation we clear the invalid aborted txn ID one by one. * In the new implementation, we adopt snapshot segments. And then we clear invalid segment by its max read position. */ - void trimSnapshotSegments(); + void trimExpiredTxnIDDataOrSnapshotSegments(); /** * Check whether the transaction ID is an aborted transaction ID. @@ -37,7 +37,7 @@ public interface AbortedTxnProcessor { * @param readPosition the read position of the transaction message, can be used to find the segment. * @return a boolean, whether the transaction ID is an aborted transaction ID. */ - boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition); + boolean checkAbortedTransaction(T txnID, Position readPosition); /** * Recover transaction buffer by transaction buffer snapshot. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 76c617bd0af1b..eacb4b7ade23e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -1,4 +1,204 @@ package org.apache.pulsar.broker.transaction.buffer.impl; -public class SingleSnapshotAbortedTxnProcessorImpl { +import io.netty.util.Timeout; +import io.netty.util.Timer; +import io.netty.util.TimerTask; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.collections4.map.LinkedMap; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.systopic.SystemTopicClient; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; +import org.apache.pulsar.broker.transaction.buffer.metadata.AbortTxnMetadata; +import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.naming.TopicName; + +@Slf4j +public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcessor, TimerTask { + private final PersistentTopic topic; + private final CompletableFuture> takeSnapshotWriter; + private volatile PositionImpl maxReadPosition; + + private final Timer timer; + + /** + * Aborts, map for jude message is aborted, linked for remove abort txn in memory when this + * position have been deleted. + */ + private final LinkedMap aborts = new LinkedMap<>(); + + private final int takeSnapshotIntervalNumber; + + private final int takeSnapshotIntervalTime; + + + // when add abort or change max read position, the count will +1. Take snapshot will set 0 into it. + private final AtomicLong changeMaxReadPositionAndAddAbortTimes = new AtomicLong(); + + + public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { + this.topic = topic; + this.takeSnapshotWriter = this.topic.getBrokerService().getPulsar() + .getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName())); + this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); + this.takeSnapshotIntervalNumber = topic.getBrokerService().getPulsar() + .getConfiguration().getTransactionBufferSnapshotMaxTransactionCount(); + this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() + .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); + this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + } + + @Override + public void appendAbortedTxn(TxnID abortedTxnId, PositionImpl position) { + aborts.put(abortedTxnId, position); + } + + @Override + public void updateMaxReadPosition(Position maxReadPosition) { + if (this.maxReadPosition != maxReadPosition) { + this.maxReadPosition = (PositionImpl) maxReadPosition; + takeSnapshotByChangeTimes(); + } + } + + @Override + public void trimExpiredTxnIDDataOrSnapshotSegments() { + while (!aborts.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) + .ledgerExists(aborts.get(aborts.firstKey()).getLedgerId())) { + if (log.isDebugEnabled()) { + aborts.firstKey(); + log.debug("[{}] Topic transaction buffer clear aborted transaction, TxnId : {}, Position : {}", + topic.getName(), aborts.firstKey(), aborts.get(aborts.firstKey())); + } + aborts.remove(aborts.firstKey()); + } + } + + @Override + public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { + return aborts.containsKey(txnID); + } + + + @Override + public CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack) { + return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotService() + .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { + PositionImpl startReadCursorPosition = null; + try { + boolean hasSnapshot = false; + while (reader.hasMoreEvents()) { + Message message = reader.readNext(); + if (topic.getName().equals(message.getKey())) { + TransactionBufferSnapshot transactionBufferSnapshot = message.getValue(); + if (transactionBufferSnapshot != null) { + hasSnapshot = true; + handleSnapshot(transactionBufferSnapshot); + startReadCursorPosition = PositionImpl.get( + transactionBufferSnapshot.getMaxReadPositionLedgerId(), + transactionBufferSnapshot.getMaxReadPositionEntryId()); + } + } + } + closeReader(reader); + if (!hasSnapshot) { + callBack.noNeedToRecover(); + return null; + } + return CompletableFuture.completedFuture(startReadCursorPosition); + } catch (Exception ex) { + log.error("[{}] Transaction buffer recover fail when read " + + "transactionBufferSnapshot!", topic.getName(), ex); + callBack.recoverExceptionally(ex); + closeReader(reader); + return null; + } + + }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this)); + } + + private void closeReader(SystemTopicClient.Reader reader) { + reader.closeAsync().exceptionally(e -> { + log.error("[{}]Transaction buffer reader close error!", topic.getName(), e); + return null; + }); + } + + private void takeSnapshotByChangeTimes() { + if (changeMaxReadPositionAndAddAbortTimes.incrementAndGet() >= takeSnapshotIntervalNumber) { + takeSnapshot(); + } + } + + private void takeSnapshotByTimeout() { + if (changeMaxReadPositionAndAddAbortTimes.get() > 0) { + takeSnapshot(); + } + this.timer.newTimeout(SingleSnapshotAbortedTxnProcessorImpl.this, + takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); + } + + private void handleSnapshot(TransactionBufferSnapshot snapshot) { + maxReadPosition = PositionImpl.get(snapshot.getMaxReadPositionLedgerId(), + snapshot.getMaxReadPositionEntryId()); + if (snapshot.getAborts() != null) { + snapshot.getAborts().forEach(abortTxnMetadata -> + aborts.put(new TxnID(abortTxnMetadata.getTxnIdMostBits(), + abortTxnMetadata.getTxnIdLeastBits()), + PositionImpl.get(abortTxnMetadata.getLedgerId(), + abortTxnMetadata.getEntryId()))); + } + } + + private CompletableFuture takeSnapshot() { + changeMaxReadPositionAndAddAbortTimes.set(0); + return takeSnapshotWriter.thenCompose(writer -> { + TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); + synchronized (SingleSnapshotAbortedTxnProcessorImpl.this) { + snapshot.setTopicName(topic.getName()); + snapshot.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); + snapshot.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); + List list = new ArrayList<>(); + aborts.forEach((k, v) -> { + AbortTxnMetadata abortTxnMetadata = new AbortTxnMetadata(); + abortTxnMetadata.setTxnIdMostBits(k.getMostSigBits()); + abortTxnMetadata.setTxnIdLeastBits(k.getLeastSigBits()); + abortTxnMetadata.setLedgerId(v.getLedgerId()); + abortTxnMetadata.setEntryId(v.getEntryId()); + list.add(abortTxnMetadata); + }); + snapshot.setAborts(list); + } + return writer.writeAsync(snapshot.getTopicName(), snapshot).thenAccept(messageId-> { + //TODO: do record this in TB +// this.lastSnapshotTimestamps = System.currentTimeMillis(); + if (log.isDebugEnabled()) { + log.debug("[{}]Transaction buffer take snapshot success! " + + "messageId : {}", topic.getName(), messageId); + } + }).exceptionally(e -> { + log.warn("[{}]Transaction buffer take snapshot fail! ", topic.getName(), e); + return null; + }); + }); + } + + @Override + public void run(Timeout timeout) { + // TODO: Start to do this after TB and processor ready + takeSnapshotByTimeout(); + } + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 22ce4baf8b7d5..1aac9d5075ded 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -38,7 +38,7 @@ import org.apache.pulsar.common.util.FutureUtil; @Slf4j -public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor, TimerTask { +public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor, TimerTask { private final AtomicLong sequenceID = new AtomicLong(0); //TODO: recover this at recover processor. @@ -136,8 +136,10 @@ private void takeSnapshotSegment() { @Override public void updateMaxReadPosition(Position position) { - this.maxReadPosition = (PositionImpl) position; - updateSnapshotMetadataByChangeTimes(); + if (position != this.maxReadPosition) { + this.maxReadPosition = (PositionImpl) position; + updateSnapshotMetadataByChangeTimes(); + } } @Override @@ -151,7 +153,7 @@ public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { } @Override - public void trimSnapshotSegments() { + public void trimExpiredTxnIDDataOrSnapshotSegments() { //Checking whether there are some segment expired. while (!aborts.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) .ledgerExists(aborts.firstKey().getLedgerId())) { From 4a22a6ac2a09b6de0a5fa1714cbe6f002da2fe1f Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Tue, 25 Oct 2022 11:58:34 +0800 Subject: [PATCH 04/32] modify the TransactionBuffer to use AbortedTxnProcessor --- .../buffer/AbortedTxnProcessor.java | 16 +- ...SingleSnapshotAbortedTxnProcessorImpl.java | 45 ++- ...napshotSegmentAbortedTxnProcessorImpl.java | 122 +++++-- .../buffer/impl/TopicTransactionBuffer.java | 304 ++++++------------ ...TopicTransactionBufferRecoverCallBack.java | 7 - 5 files changed, 233 insertions(+), 261 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index 80e07da3f2179..45a15f1ba7b33 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -1,19 +1,21 @@ package org.apache.pulsar.broker.transaction.buffer; +import io.netty.util.TimerTask; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferRecoverCallBack; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; -public interface AbortedTxnProcessor { +public interface AbortedTxnProcessor extends TimerTask { /** * After the transaction buffer writes a transaction aborted mark to the topic, * the transaction buffer will add the aborted transaction ID to AbortedTxnProcessor. * @param txnID aborted transaction ID. */ - void appendAbortedTxn(T txnID, PositionImpl position); + void appendAbortedTxn(TxnIDData txnID, PositionImpl position); /** * After the transaction buffer writes a transaction aborted mark to the topic, @@ -37,13 +39,19 @@ public interface AbortedTxnProcessor { * @param readPosition the read position of the transaction message, can be used to find the segment. * @return a boolean, whether the transaction ID is an aborted transaction ID. */ - boolean checkAbortedTransaction(T txnID, Position readPosition); + boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition); /** * Recover transaction buffer by transaction buffer snapshot. * @return a pair consists of a Boolean if the transaction buffer needs to recover and a Position (startReadCursorPosition) determiner where to start to recover in the original topic. */ - CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack); + CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack); + + public CompletableFuture clearSnapshot(); + public CompletableFuture takesFirstSnapshot(); + public PositionImpl getMaxReadPosition(); + + public long getLastSnapshotTimestamps(); } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index eacb4b7ade23e..154f997ccd016 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -2,7 +2,6 @@ import io.netty.util.Timeout; import io.netty.util.Timer; -import io.netty.util.TimerTask; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -18,12 +17,12 @@ import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.metadata.AbortTxnMetadata; import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.naming.TopicName; @Slf4j -public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcessor, TimerTask { +public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcessor { private final PersistentTopic topic; private final CompletableFuture> takeSnapshotWriter; private volatile PositionImpl maxReadPosition; @@ -34,8 +33,9 @@ public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcesso * Aborts, map for jude message is aborted, linked for remove abort txn in memory when this * position have been deleted. */ - private final LinkedMap aborts = new LinkedMap<>(); + private final LinkedMap aborts = new LinkedMap<>(); + private volatile long lastSnapshotTimestamps; private final int takeSnapshotIntervalNumber; private final int takeSnapshotIntervalTime; @@ -47,6 +47,7 @@ public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcesso public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; + this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); this.takeSnapshotWriter = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName())); @@ -59,7 +60,7 @@ public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { } @Override - public void appendAbortedTxn(TxnID abortedTxnId, PositionImpl position) { + public void appendAbortedTxn(TxnIDData abortedTxnId, PositionImpl position) { aborts.put(abortedTxnId, position); } @@ -85,13 +86,13 @@ public void trimExpiredTxnIDDataOrSnapshotSegments() { } @Override - public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { + public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { return aborts.containsKey(txnID); } @Override - public CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack) { + public CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack) { return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotService() .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { @@ -129,6 +130,30 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecov .getExecutor(this)); } + @Override + public CompletableFuture clearSnapshot() { + return this.takeSnapshotWriter.thenCompose(writer -> { + TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); + snapshot.setTopicName(topic.getName()); + return writer.deleteAsync(snapshot.getTopicName(), snapshot); + }).thenCompose(__ -> CompletableFuture.completedFuture(null)); + } + + @Override + public CompletableFuture takesFirstSnapshot() { + return takeSnapshot(); + } + + @Override + public PositionImpl getMaxReadPosition() { + return maxReadPosition; + } + + @Override + public long getLastSnapshotTimestamps() { + return this.lastSnapshotTimestamps; + } + private void closeReader(SystemTopicClient.Reader reader) { reader.closeAsync().exceptionally(e -> { log.error("[{}]Transaction buffer reader close error!", topic.getName(), e); @@ -155,7 +180,7 @@ private void handleSnapshot(TransactionBufferSnapshot snapshot) { snapshot.getMaxReadPositionEntryId()); if (snapshot.getAborts() != null) { snapshot.getAborts().forEach(abortTxnMetadata -> - aborts.put(new TxnID(abortTxnMetadata.getTxnIdMostBits(), + aborts.put(new TxnIDData(abortTxnMetadata.getTxnIdMostBits(), abortTxnMetadata.getTxnIdLeastBits()), PositionImpl.get(abortTxnMetadata.getLedgerId(), abortTxnMetadata.getEntryId()))); @@ -181,9 +206,9 @@ private CompletableFuture takeSnapshot() { }); snapshot.setAborts(list); } - return writer.writeAsync(snapshot.getTopicName(), snapshot).thenAccept(messageId-> { + return writer.writeAsync(snapshot.getTopicName(), snapshot).thenAccept(messageId -> { //TODO: do record this in TB -// this.lastSnapshotTimestamps = System.currentTimeMillis(); + this.lastSnapshotTimestamps = System.currentTimeMillis(); if (log.isDebugEnabled()) { log.debug("[{}]Transaction buffer take snapshot success! " + "messageId : {}", topic.getName(), messageId); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 1aac9d5075ded..1c59391226519 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -5,6 +5,7 @@ import io.netty.util.Timeout; import io.netty.util.Timer; import io.netty.util.TimerTask; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -38,7 +39,7 @@ import org.apache.pulsar.common.util.FutureUtil; @Slf4j -public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor, TimerTask { +public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { private final AtomicLong sequenceID = new AtomicLong(0); //TODO: recover this at recover processor. @@ -50,7 +51,7 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess private final ConcurrentSkipListMap indexes = new ConcurrentSkipListMap<>(); - private TransactionBufferSnapshotIndexes theLatestSnapshotIndexes; + private TransactionBufferSnapshotIndexes theLatestSnapshotIndexes = new TransactionBufferSnapshotIndexes(); private final Timer timer; private PositionImpl maxReadPosition; @@ -76,13 +77,15 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess private final CompletableFuture> snapshotIndexWriterFuture; - - public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic, int takeSnapshotIntervalNumber, int takeSnapshotIntervalTime, - int transactionBufferMaxAbortedTxnsOfSnapshotSegment) { + public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; - this.takeSnapshotIntervalTime = takeSnapshotIntervalTime; - this.takeSnapshotIntervalNumber = takeSnapshotIntervalNumber; - this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = transactionBufferMaxAbortedTxnsOfSnapshotSegment; + this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + this.takeSnapshotIntervalNumber = topic.getBrokerService().getPulsar() + .getConfiguration().getTransactionBufferSnapshotMaxTransactionCount(); + this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() + .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); + this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = topic.getBrokerService().getPulsar() + .getConfiguration().getTransactionBufferSnapshotSegmentSize(); snapshotSegmentsWriterFuture = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName())); @@ -98,6 +101,7 @@ public void appendAbortedTxn(TxnIDData abortedTxnId, PositionImpl position) { lastAbortedTxnIDs.add(abortedTxnId); //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. if (lastAbortedTxnIDs.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { + changeMaxReadPositionAndAddAbortTimes.set(0); aborts.put(position, lastAbortedTxnIDs); //Guarantee the order of the segments. snapshotSegmentQueue.put(position, lastAbortedTxnIDs); @@ -144,6 +148,9 @@ public void updateMaxReadPosition(Position position) { @Override public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { + if (readPosition == null) { + return aborts.values().stream().anyMatch(list -> list.contains(txnID)) || lastAbortedTxnIDs.contains(txnID); + } List txnIDSet = aborts.ceilingEntry((PositionImpl) readPosition).getValue(); if (txnIDSet == null) { return lastAbortedTxnIDs.contains(txnID); @@ -162,24 +169,7 @@ public void trimExpiredTxnIDDataOrSnapshotSegments() { topic.getName(), aborts.firstKey()); } PositionImpl positionNeedToDelete = aborts.firstKey(); - long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); - snapshotSegmentsWriterFuture.thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) - .thenRun(() -> { - if (log.isDebugEnabled()) { - log.debug("[{}] Successes to delete the snapshot segment, " - + "whose sequenceId is [{}] and maxReadPosition is [{}]", - this.topic.getName(), this.sequenceID, this.maxReadPosition); - } - aborts.remove(positionNeedToDelete); - indexes.remove(positionNeedToDelete); - //TODO: check whether the snapshot segment is null, and update index. - updateSnapshotIndex(); - }).exceptionally(e -> { - log.error("[{}] Failed to delete the snapshot segment, " - + "whose sequenceId is [{}] and maxReadPosition is [{}]", - this.topic.getName(), this.sequenceID, this.maxReadPosition); - return null; - }); + clearSnapshotSegment(positionNeedToDelete); } } @@ -242,9 +232,12 @@ private CompletableFuture updateSnapshotIndex() { snapshotIndexes.setIndexList(indexes.values().stream().toList()); //Only update the index in indexes and keep the metadata in indexes unchanged. snapshotIndexes.setSnapshot(theLatestSnapshotIndexes.getSnapshot()); - return indexesWriter.writeAsync(snapshotIndexes.getTopicName(), snapshotIndexes); + return indexesWriter.writeAsync(topic.getName(), snapshotIndexes); + }) + .thenRun(() -> { + theLatestSnapshotIndexes.setIndexList(snapshotIndexes.getIndexList()); + this.lastSnapshotTimestamps = System.currentTimeMillis(); }) - .thenRun(() -> theLatestSnapshotIndexes.setIndexList(snapshotIndexes.getIndexList())) .exceptionally(e -> { log.error("[{}] Failed to update snapshot segment index", snapshotIndexes.getTopicName(), e); return null; @@ -253,9 +246,9 @@ private CompletableFuture updateSnapshotIndex() { //Update the metadata in the transactionBufferSnapshotIndexes. //Concurrency control is performed by snapshotIndexWriterFuture. - private void updateIndexMetadataForTheLastSnapshot() { + private CompletableFuture updateIndexMetadataForTheLastSnapshot() { TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); - snapshotIndexWriterFuture + return snapshotIndexWriterFuture .thenCompose((indexesWriter) -> { //Store the latest metadata TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = @@ -264,9 +257,12 @@ private void updateIndexMetadataForTheLastSnapshot() { indexes.setSnapshot(transactionBufferSnapshotSegment); //Only update the metadata in indexes and keep the index in indexes unchanged. indexes.setIndexList(theLatestSnapshotIndexes.getIndexList()); - return indexesWriter.writeAsync(indexes.getTopicName(), indexes); + return indexesWriter.writeAsync(topic.getName(), indexes); + }) + .thenRun(() -> { + theLatestSnapshotIndexes.setSnapshot(indexes.getSnapshot()); + this.lastSnapshotTimestamps = System.currentTimeMillis(); }) - .thenRun(() -> theLatestSnapshotIndexes.setSnapshot(indexes.getSnapshot())) .exceptionally(e -> { log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); return null; @@ -274,7 +270,7 @@ private void updateIndexMetadataForTheLastSnapshot() { } @Override - public CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack) { + public CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack) { return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotIndexService() .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { @@ -384,6 +380,66 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob .getExecutor(this)); } + @Override + public CompletableFuture clearSnapshot() { + ArrayList> completableFutures = new ArrayList<>(); + while (!aborts.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", + topic.getName(), aborts.firstKey()); + } + PositionImpl positionNeedToDelete = aborts.firstKey(); + completableFutures.add(clearSnapshotSegment(positionNeedToDelete)); + } + return FutureUtil.waitForAll(completableFutures) + .thenCompose((ignore) -> snapshotIndexWriterFuture + .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) + .thenRun(() -> { + log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", topic.getName()); + }).exceptionally(e -> { + log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", + topic.getName(), e); + return null; + }); + } + + private CompletableFuture clearSnapshotSegment(PositionImpl positionNeedToDelete) { + + long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); + return snapshotSegmentsWriterFuture.thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) + .thenRun(() -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Successes to delete the snapshot segment, " + + "whose sequenceId is [{}] and maxReadPosition is [{}]", + this.topic.getName(), this.sequenceID, this.maxReadPosition); + } + aborts.remove(positionNeedToDelete); + indexes.remove(positionNeedToDelete); + //The process will check whether the snapshot segment is null, and update index when recovered. + updateSnapshotIndex(); + }).exceptionally(e -> { + log.error("[{}] Failed to delete the snapshot segment, " + + "whose sequenceId is [{}] and maxReadPosition is [{}]", + this.topic.getName(), this.sequenceID, this.maxReadPosition); + return null; + }); + } + + @Override + public CompletableFuture takesFirstSnapshot() { + return updateIndexMetadataForTheLastSnapshot(); + } + + @Override + public PositionImpl getMaxReadPosition() { + return this.maxReadPosition; + } + + @Override + public long getLastSnapshotTimestamps() { + return this.lastSnapshotTimestamps; + } + private void handleSnapshotSegmentEntry(Entry entry) { //decode snapshot from entry ByteBuf headersAndPayload = entry.getDataBuffer(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 1245c7d8129aa..8b06c64aa1e43 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -20,10 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; -import io.netty.util.Timeout; import io.netty.util.Timer; -import io.netty.util.TimerTask; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -38,19 +35,18 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.map.LinkedMap; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.SystemTopicClient; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.TransactionBufferReader; import org.apache.pulsar.broker.transaction.buffer.TransactionMeta; -import org.apache.pulsar.broker.transaction.buffer.metadata.AbortTxnMetadata; import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; -import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.proto.MessageMetadata; @@ -68,40 +64,25 @@ * Transaction buffer based on normal persistent topic. */ @Slf4j -public class TopicTransactionBuffer extends TopicTransactionBufferState implements TransactionBuffer, TimerTask { +public class TopicTransactionBuffer extends TopicTransactionBufferState implements TransactionBuffer { private final PersistentTopic topic; - private volatile PositionImpl maxReadPosition; - /** * Ongoing transaction, map for remove txn stable position, linked for find max read position. */ private final LinkedMap ongoingTxns = new LinkedMap<>(); - /** - * Aborts, map for jude message is aborted, linked for remove abort txn in memory when this - * position have been deleted. - */ - private final LinkedMap aborts = new LinkedMap<>(); - private final CompletableFuture> takeSnapshotWriter; - // when add abort or change max read position, the count will +1. Take snapshot will set 0 into it. - private final AtomicLong changeMaxReadPositionAndAddAbortTimes = new AtomicLong(); - private final LongAdder txnCommittedCounter = new LongAdder(); private final LongAdder txnAbortedCounter = new LongAdder(); private final Timer timer; - private final int takeSnapshotIntervalNumber; - private final int takeSnapshotIntervalTime; - private volatile long lastSnapshotTimestamps; - private final CompletableFuture transactionBufferFuture = new CompletableFuture<>(); /** @@ -113,6 +94,8 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen private final Semaphore handleLowWaterMark = new Semaphore(1); + private final AbortedTxnProcessor snapshotAbortedTxnProcessor; + public TopicTransactionBuffer(PersistentTopic topic) { super(State.None); this.topic = topic; @@ -120,11 +103,14 @@ public TopicTransactionBuffer(PersistentTopic topic) { .getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName())); this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); - this.takeSnapshotIntervalNumber = topic.getBrokerService().getPulsar() - .getConfiguration().getTransactionBufferSnapshotMaxTransactionCount(); this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); - this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + + if (topic.getBrokerService().getPulsar().getConfiguration().isTransactionBufferSegmentedSnapshotEnabled()) { + snapshotAbortedTxnProcessor = new SnapshotSegmentAbortedTxnProcessorImpl(topic); + } else { + snapshotAbortedTxnProcessor = new SingleSnapshotAbortedTxnProcessorImpl(topic); + } this.recover(); } @@ -139,7 +125,8 @@ public void recoverComplete() { // completely the normal message have been sent to broker and state is // not Ready can't sync maxReadPosition when no ongoing transactions if (ongoingTxns.isEmpty()) { - maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + snapshotAbortedTxnProcessor + .updateMaxReadPosition(topic.getManagedLedger().getLastConfirmedEntry()); } if (!changeToReadyState()) { log.error("[{}]Transaction buffer recover fail, current state: {}", @@ -149,7 +136,7 @@ public void recoverComplete() { "Transaction buffer recover failed to change the status to Ready," + "current state is: " + getState())); } else { - timer.newTimeout(TopicTransactionBuffer.this, + timer.newTimeout(snapshotAbortedTxnProcessor, takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); transactionBufferFuture.complete(null); recoverTime.setRecoverEndTime(System.currentTimeMillis()); @@ -163,7 +150,9 @@ public void noNeedToRecover() { // sync maxReadPosition change to LAC when TopicTransaction buffer have not recover // completely the normal message have been sent to broker and state is // not NoSnapshot can't sync maxReadPosition - maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + snapshotAbortedTxnProcessor + .updateMaxReadPosition(topic.getManagedLedger().getLastConfirmedEntry()); + if (!changeToNoSnapshotState()) { log.error("[{}]Transaction buffer recover fail", topic.getName()); } else { @@ -172,20 +161,6 @@ public void noNeedToRecover() { } } } - - @Override - public void handleSnapshot(TransactionBufferSnapshot snapshot) { - maxReadPosition = PositionImpl.get(snapshot.getMaxReadPositionLedgerId(), - snapshot.getMaxReadPositionEntryId()); - if (snapshot.getAborts() != null) { - snapshot.getAborts().forEach(abortTxnMetadata -> - aborts.put(new TxnID(abortTxnMetadata.getTxnIdMostBits(), - abortTxnMetadata.getTxnIdLeastBits()), - PositionImpl.get(abortTxnMetadata.getLedgerId(), - abortTxnMetadata.getEntryId()))); - } - } - @Override public void handleTxnEntry(Entry entry) { ByteBuf metadataAndPayload = entry.getDataBuffer(); @@ -197,7 +172,8 @@ public void handleTxnEntry(Entry entry) { PositionImpl position = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); if (Markers.isTxnMarker(msgMetadata)) { if (Markers.isTxnAbortMarker(msgMetadata)) { - aborts.put(txnID, position); + snapshotAbortedTxnProcessor.appendAbortedTxn( + new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits()), position); } updateMaxReadPosition(txnID); } else { @@ -225,7 +201,8 @@ public void recoverExceptionally(Throwable e) { recoverTime.setRecoverEndTime(System.currentTimeMillis()); topic.close(true); } - }, this.topic, this, takeSnapshotWriter)); + }, this.topic, + this, takeSnapshotWriter, snapshotAbortedTxnProcessor)); } @Override @@ -241,9 +218,9 @@ public CompletableFuture checkIfTBRecoverCompletely(boolean isTxnEnabled) CompletableFuture completableFuture = new CompletableFuture<>(); transactionBufferFuture.thenRun(() -> { if (checkIfNoSnapshot()) { - takeSnapshot().thenRun(() -> { + snapshotAbortedTxnProcessor.takesFirstSnapshot().thenRun(() -> { if (changeToReadyStateFromNoSnapshot()) { - timer.newTimeout(TopicTransactionBuffer.this, + timer.newTimeout(snapshotAbortedTxnProcessor, takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); } completableFuture.complete(null); @@ -308,15 +285,16 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { } private void handleTransactionMessage(TxnID txnId, Position position) { - if (!ongoingTxns.containsKey(txnId) && !aborts.containsKey(txnId)) { + if (!ongoingTxns.containsKey(txnId) && !this.snapshotAbortedTxnProcessor.checkAbortedTransaction( + new TxnIDData(txnId.getMostSigBits(), txnId.getLeastSigBits()), position)) { ongoingTxns.put(txnId, (PositionImpl) position); PositionImpl firstPosition = ongoingTxns.get(ongoingTxns.firstKey()); //max read position is less than first ongoing transaction message position, so entryId -1 - maxReadPosition = PositionImpl.get(firstPosition.getLedgerId(), firstPosition.getEntryId() - 1); + snapshotAbortedTxnProcessor.updateMaxReadPosition(PositionImpl.get(firstPosition.getLedgerId(), + firstPosition.getEntryId() - 1)); } } - @Override public CompletableFuture openTransactionBufferReader(TxnID txnID, long startSequenceId) { return null; @@ -339,8 +317,9 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { updateMaxReadPosition(txnID); handleLowWaterMark(txnID, lowWaterMark); - clearAbortedTransactions(); - takeSnapshotByChangeTimes(); + snapshotAbortedTxnProcessor.trimExpiredTxnIDDataOrSnapshotSegments(); + snapshotAbortedTxnProcessor.appendAbortedTxn(new TxnIDData(txnID.getMostSigBits(), + txnID.getLeastSigBits()), (PositionImpl) position); } txnCommittedCounter.increment(); completableFuture.complete(null); @@ -383,11 +362,10 @@ public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { - aborts.put(txnID, (PositionImpl) position); + snapshotAbortedTxnProcessor.appendAbortedTxn(new TxnIDData(txnID.getMostSigBits(), + txnID.getLeastSigBits()), (PositionImpl) position); updateMaxReadPosition(txnID); - changeMaxReadPositionAndAddAbortTimes.getAndIncrement(); - clearAbortedTransactions(); - takeSnapshotByChangeTimes(); + snapshotAbortedTxnProcessor.trimExpiredTxnIDDataOrSnapshotSegments(); } txnAbortedCounter.increment(); completableFuture.complete(null); @@ -452,65 +430,9 @@ private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { } } - private void takeSnapshotByChangeTimes() { - if (changeMaxReadPositionAndAddAbortTimes.get() >= takeSnapshotIntervalNumber) { - takeSnapshot(); - } - } - - private void takeSnapshotByTimeout() { - if (changeMaxReadPositionAndAddAbortTimes.get() > 0) { - takeSnapshot(); - } - this.timer.newTimeout(TopicTransactionBuffer.this, - takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); - } - - private CompletableFuture takeSnapshot() { - changeMaxReadPositionAndAddAbortTimes.set(0); - return takeSnapshotWriter.thenCompose(writer -> { - TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); - synchronized (TopicTransactionBuffer.this) { - snapshot.setTopicName(topic.getName()); - snapshot.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); - snapshot.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); - List list = new ArrayList<>(); - aborts.forEach((k, v) -> { - AbortTxnMetadata abortTxnMetadata = new AbortTxnMetadata(); - abortTxnMetadata.setTxnIdMostBits(k.getMostSigBits()); - abortTxnMetadata.setTxnIdLeastBits(k.getLeastSigBits()); - abortTxnMetadata.setLedgerId(v.getLedgerId()); - abortTxnMetadata.setEntryId(v.getEntryId()); - list.add(abortTxnMetadata); - }); - snapshot.setAborts(list); - } - return writer.writeAsync(snapshot.getTopicName(), snapshot).thenAccept(messageId-> { - this.lastSnapshotTimestamps = System.currentTimeMillis(); - if (log.isDebugEnabled()) { - log.debug("[{}]Transaction buffer take snapshot success! " - + "messageId : {}", topic.getName(), messageId); - } - }).exceptionally(e -> { - log.warn("[{}]Transaction buffer take snapshot fail! ", topic.getName(), e); - return null; - }); - }); - } - private void clearAbortedTransactions() { - while (!aborts.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) - .ledgerExists(aborts.get(aborts.firstKey()).getLedgerId())) { - if (log.isDebugEnabled()) { - aborts.firstKey(); - log.debug("[{}] Topic transaction buffer clear aborted transaction, TxnId : {}, Position : {}", - topic.getName(), aborts.firstKey(), aborts.get(aborts.firstKey())); - } - aborts.remove(aborts.firstKey()); - } - } void updateMaxReadPosition(TxnID txnID) { - PositionImpl preMaxReadPosition = this.maxReadPosition; ongoingTxns.remove(txnID); + PositionImpl maxReadPosition; if (!ongoingTxns.isEmpty()) { PositionImpl position = ongoingTxns.get(ongoingTxns.firstKey()); //max read position is less than first ongoing transaction message position, so entryId -1 @@ -518,9 +440,7 @@ void updateMaxReadPosition(TxnID txnID) { } else { maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); } - if (preMaxReadPosition.compareTo(this.maxReadPosition) != 0) { - this.changeMaxReadPositionAndAddAbortTimes.getAndIncrement(); - } + snapshotAbortedTxnProcessor.updateMaxReadPosition(maxReadPosition); } @Override @@ -530,11 +450,8 @@ public CompletableFuture purgeTxns(List dataLedgers) { @Override public CompletableFuture clearSnapshot() { - return this.takeSnapshotWriter.thenCompose(writer -> { - TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); - snapshot.setTopicName(topic.getName()); - return writer.deleteAsync(snapshot.getTopicName(), snapshot); - }).thenCompose(__ -> CompletableFuture.completedFuture(null)); + return snapshotAbortedTxnProcessor.clearSnapshot(); + } @Override @@ -545,7 +462,8 @@ public CompletableFuture closeAsync() { @Override public boolean isTxnAborted(TxnID txnID) { - return aborts.containsKey(txnID); + return snapshotAbortedTxnProcessor.checkAbortedTransaction( + new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits()), null); } @Override @@ -554,11 +472,11 @@ public void syncMaxReadPositionForNormalPublish(PositionImpl position) { // thread is the same tread, in this time the lastAddConfirm don't content transaction message. synchronized (TopicTransactionBuffer.this) { if (checkIfNoSnapshot()) { - maxReadPosition = position; + //TODO:The changes time here should not be changed. + snapshotAbortedTxnProcessor.updateMaxReadPosition(position); } else if (checkIfReady()) { if (ongoingTxns.isEmpty()) { - maxReadPosition = position; - changeMaxReadPositionAndAddAbortTimes.incrementAndGet(); + snapshotAbortedTxnProcessor.updateMaxReadPosition(position); } } } @@ -567,7 +485,7 @@ public void syncMaxReadPositionForNormalPublish(PositionImpl position) { @Override public PositionImpl getMaxReadPosition() { if (checkIfReady() || checkIfNoSnapshot()) { - return this.maxReadPosition; + return this.snapshotAbortedTxnProcessor.getMaxReadPosition(); } else { return PositionImpl.EARLIEST; } @@ -586,9 +504,9 @@ public TransactionInBufferStats getTransactionInBufferStats(TxnID txnID) { @Override public TransactionBufferStats getStats(boolean lowWaterMarks) { TransactionBufferStats transactionBufferStats = new TransactionBufferStats(); - transactionBufferStats.lastSnapshotTimestamps = this.lastSnapshotTimestamps; + transactionBufferStats.lastSnapshotTimestamps = this.snapshotAbortedTxnProcessor.getLastSnapshotTimestamps(); transactionBufferStats.state = this.getState().name(); - transactionBufferStats.maxReadPosition = this.maxReadPosition.toString(); + transactionBufferStats.maxReadPosition = this.snapshotAbortedTxnProcessor.getMaxReadPosition().toString(); if (lowWaterMarks) { transactionBufferStats.lowWaterMarks = this.lowWaterMarks; } @@ -599,13 +517,6 @@ public TransactionBufferStats getStats(boolean lowWaterMarks) { return transactionBufferStats; } - @Override - public void run(Timeout timeout) { - if (checkIfReady()) { - takeSnapshotByTimeout(); - } - } - // we store the maxReadPosition from snapshot then open the non-durable cursor by this topic's manageLedger. // the non-durable cursor will read to lastConfirmedEntry. @VisibleForTesting @@ -627,14 +538,18 @@ public static class TopicTransactionBufferRecover implements Runnable { private final CompletableFuture> takeSnapshotWriter; + private final AbortedTxnProcessor abortedTxnProcessor; + private TopicTransactionBufferRecover(TopicTransactionBufferRecoverCallBack callBack, PersistentTopic topic, TopicTransactionBuffer transactionBuffer, CompletableFuture< - SystemTopicClient.Writer> takeSnapshotWriter) { + SystemTopicClient.Writer> takeSnapshotWriter, + AbortedTxnProcessor abortedTxnProcessor) { this.topic = topic; this.callBack = callBack; this.entryQueue = new SpscArrayQueue<>(2000); this.topicTransactionBuffer = transactionBuffer; this.takeSnapshotWriter = takeSnapshotWriter; + this.abortedTxnProcessor = abortedTxnProcessor; } @SneakyThrows @@ -646,82 +561,57 @@ public void run() { this, topic.getName()); return; } - topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotService().createReader(TopicName.get(topic.getName())) - .thenAcceptAsync(reader -> { - try { - boolean hasSnapshot = false; - while (reader.hasMoreEvents()) { - Message message = reader.readNext(); - if (topic.getName().equals(message.getKey())) { - TransactionBufferSnapshot transactionBufferSnapshot = message.getValue(); - if (transactionBufferSnapshot != null) { - hasSnapshot = true; - callBack.handleSnapshot(transactionBufferSnapshot); - this.startReadCursorPosition = PositionImpl.get( - transactionBufferSnapshot.getMaxReadPositionLedgerId(), - transactionBufferSnapshot.getMaxReadPositionEntryId()); - } - } - } - if (!hasSnapshot) { - closeReader(reader); - callBack.noNeedToRecover(); - return; + abortedTxnProcessor.recoverFromSnapshot(callBack).thenAcceptAsync(startReadCursorPosition -> { + //Transaction is not enable for this topic, so just make maxReadPosition as LAC. + if (startReadCursorPosition == null) { + return; + } else { + this.startReadCursorPosition = startReadCursorPosition; + } + ManagedCursor managedCursor; + try { + managedCursor = topic.getManagedLedger() + .newNonDurableCursor(this.startReadCursorPosition, SUBSCRIPTION_NAME); + } catch (ManagedLedgerException e) { + callBack.recoverExceptionally(e); + log.error("[{}]Transaction buffer recover fail when open cursor!", topic.getName(), e); + return; + } + PositionImpl lastConfirmedEntry = + (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + PositionImpl currentLoadPosition = (PositionImpl) this.startReadCursorPosition; + FillEntryQueueCallback fillEntryQueueCallback = new FillEntryQueueCallback(entryQueue, + managedCursor, TopicTransactionBufferRecover.this); + if (lastConfirmedEntry.getEntryId() != -1) { + while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0 + && fillEntryQueueCallback.fillQueue()) { + Entry entry = entryQueue.poll(); + if (entry != null) { + try { + currentLoadPosition = PositionImpl.get(entry.getLedgerId(), + entry.getEntryId()); + callBack.handleTxnEntry(entry); + } finally { + entry.release(); } - } catch (Exception ex) { - log.error("[{}] Transaction buffer recover fail when read " - + "transactionBufferSnapshot!", topic.getName(), ex); - callBack.recoverExceptionally(ex); - closeReader(reader); - return; - } - closeReader(reader); - - ManagedCursor managedCursor; - try { - managedCursor = topic.getManagedLedger() - .newNonDurableCursor(this.startReadCursorPosition, SUBSCRIPTION_NAME); - } catch (ManagedLedgerException e) { - callBack.recoverExceptionally(e); - log.error("[{}]Transaction buffer recover fail when open cursor!", topic.getName(), e); - return; - } - PositionImpl lastConfirmedEntry = - (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); - PositionImpl currentLoadPosition = (PositionImpl) this.startReadCursorPosition; - FillEntryQueueCallback fillEntryQueueCallback = new FillEntryQueueCallback(entryQueue, - managedCursor, TopicTransactionBufferRecover.this); - if (lastConfirmedEntry.getEntryId() != -1) { - while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0 - && fillEntryQueueCallback.fillQueue()) { - Entry entry = entryQueue.poll(); - if (entry != null) { - try { - currentLoadPosition = PositionImpl.get(entry.getLedgerId(), - entry.getEntryId()); - callBack.handleTxnEntry(entry); - } finally { - entry.release(); - } - } else { - try { - Thread.sleep(1); - } catch (InterruptedException e) { - //no-op - } - } + } else { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + //no-op } } + } + } - closeCursor(SUBSCRIPTION_NAME); - callBack.recoverComplete(); - }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() - .getExecutor(this)).exceptionally(e -> { - callBack.recoverExceptionally(e.getCause()); - log.error("[{}]Transaction buffer new snapshot reader fail!", topic.getName(), e); - return null; - }); + closeCursor(SUBSCRIPTION_NAME); + callBack.recoverComplete(); + }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this)).exceptionally(e -> { + callBack.recoverExceptionally(e.getCause()); + log.error("[{}]Transaction buffer new snapshot reader fail!", topic.getName(), e); + return null; + }); }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() .getExecutor(this)).exceptionally(e -> { callBack.recoverExceptionally(e.getCause()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoverCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoverCallBack.java index d229fbb8f5dd3..324806296b750 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoverCallBack.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoverCallBack.java @@ -34,13 +34,6 @@ public interface TopicTransactionBufferRecoverCallBack { */ void noNeedToRecover(); - /** - * Handle transactionBufferSnapshot. - * - * @param snapshot the transaction buffer snapshot - */ - void handleSnapshot(TransactionBufferSnapshot snapshot); - /** * Handle transaction entry beyond the snapshot. * From 28380bb826ca303279944f56eb9b5bc72a3b6fb9 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Tue, 25 Oct 2022 12:03:09 +0800 Subject: [PATCH 05/32] delete TODO --- .../buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java | 2 -- .../buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java | 3 --- 2 files changed, 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 154f997ccd016..c8c105629507d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -207,7 +207,6 @@ private CompletableFuture takeSnapshot() { snapshot.setAborts(list); } return writer.writeAsync(snapshot.getTopicName(), snapshot).thenAccept(messageId -> { - //TODO: do record this in TB this.lastSnapshotTimestamps = System.currentTimeMillis(); if (log.isDebugEnabled()) { log.debug("[{}]Transaction buffer take snapshot success! " @@ -222,7 +221,6 @@ private CompletableFuture takeSnapshot() { @Override public void run(Timeout timeout) { - // TODO: Start to do this after TB and processor ready takeSnapshotByTimeout(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 1c59391226519..3f7684447e6d8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -42,7 +42,6 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { private final AtomicLong sequenceID = new AtomicLong(0); - //TODO: recover this at recover processor. private final ConcurrentSkipListMap> aborts = new ConcurrentSkipListMap<>(); private final ConcurrentSkipListMap> snapshotSegmentQueue @@ -194,7 +193,6 @@ private void takeSnapshotByTimeout() { @Override public void run(Timeout timeout) { - //TODO: Run the processor after transaction buffer ready. takeSnapshotByTimeout(); } @@ -285,7 +283,6 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe if (transactionBufferSnapshotIndexes != null) { hasIndex = true; this.theLatestSnapshotIndexes = transactionBufferSnapshotIndexes; - //TODO:take a snapshot when create producer startReadCursorPosition = PositionImpl.get( transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionLedgerId(), transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); From 15f721d5bfceea5c3211b4ba133a68eaadec20b5 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Tue, 25 Oct 2022 12:05:11 +0800 Subject: [PATCH 06/32] add License --- .../buffer/AbortedTxnProcessor.java | 18 +++++++++++++ ...SingleSnapshotAbortedTxnProcessorImpl.java | 18 +++++++++++++ ...napshotSegmentAbortedTxnProcessorImpl.java | 25 ++++++++++++++++--- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index 45a15f1ba7b33..4de3e2b803213 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.pulsar.broker.transaction.buffer; import io.netty.util.TimerTask; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index c8c105629507d..33269c5de02c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.pulsar.broker.transaction.buffer.impl; import io.netty.util.Timeout; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 3f7684447e6d8..5fd8f4a388683 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -1,10 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.pulsar.broker.transaction.buffer.impl; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.util.Timeout; import io.netty.util.Timer; -import io.netty.util.TimerTask; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; @@ -391,9 +408,9 @@ public CompletableFuture clearSnapshot() { return FutureUtil.waitForAll(completableFutures) .thenCompose((ignore) -> snapshotIndexWriterFuture .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) - .thenRun(() -> { - log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", topic.getName()); - }).exceptionally(e -> { + .thenRun(() -> log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", + topic.getName())) + .exceptionally(e -> { log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", topic.getName(), e); return null; From a4cb1123e515227911aac7eb02e97dec019fc6ed Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Tue, 25 Oct 2022 12:11:06 +0800 Subject: [PATCH 07/32] add description for interface --- .../buffer/AbortedTxnProcessor.java | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index 4de3e2b803213..9df1cb366807a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -66,10 +66,28 @@ public interface AbortedTxnProcessor extends TimerTask { CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack); - public CompletableFuture clearSnapshot(); - public CompletableFuture takesFirstSnapshot(); - public PositionImpl getMaxReadPosition(); + /** + * Clear the snapshot/snapshot segment and index for this topic. + * @return a completableFuture. + */ + CompletableFuture clearSnapshot(); + + /** + * Take the frist snapshot if the topic has no snapshot before. + * @return a completableFuture. + */ + CompletableFuture takesFirstSnapshot(); - public long getLastSnapshotTimestamps(); + /** + * Get the max read position. + * @return the maxReadPosition. + */ + PositionImpl getMaxReadPosition(); + + /** + * Get the lastSnapshotTimestamps. + * @return the lastSnapshotTimestamps. + */ + long getLastSnapshotTimestamps(); } \ No newline at end of file From 280f0d3bc4a7d0d439ca635f64c7b566de6e60cd Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Tue, 25 Oct 2022 15:51:45 +0800 Subject: [PATCH 08/32] optimize and fix some test. --- .../buffer/AbortedTxnProcessor.java | 7 ++++- ...SingleSnapshotAbortedTxnProcessorImpl.java | 7 ++++- ...napshotSegmentAbortedTxnProcessorImpl.java | 13 ++++++--- .../buffer/impl/TopicTransactionBuffer.java | 7 ++--- .../TopicTransactionBufferRecoverTest.java | 20 +++++++++---- .../broker/transaction/TransactionTest.java | 29 +++++++++++++------ 6 files changed, 58 insertions(+), 25 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index 9df1cb366807a..8bf62af172ed2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -38,10 +38,15 @@ public interface AbortedTxnProcessor extends TimerTask { /** * After the transaction buffer writes a transaction aborted mark to the topic, * the transaction buffer will update max read position in AbortedTxnProcessor - * @param maxReadPosition the Max read position after the transaction is aborted. + * @param maxReadPosition the max read position after the transaction is aborted. */ void updateMaxReadPosition(Position maxReadPosition); + /** + * This method is used to updated max read position for the topic which nerver used transaction send message. + * @param maxReadPosition the max read position after the transaction is aborted. + */ + void updateMaxReadPositionNotIncreaseChangeTimes(Position maxReadPosition); /** * Pulsar has a configuration for ledger retention time. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 33269c5de02c8..5d81dade55258 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -90,6 +90,11 @@ public void updateMaxReadPosition(Position maxReadPosition) { } } + @Override + public void updateMaxReadPositionNotIncreaseChangeTimes(Position maxReadPosition) { + this.maxReadPosition = (PositionImpl) maxReadPosition; + } + @Override public void trimExpiredTxnIDDataOrSnapshotSegments() { while (!aborts.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) @@ -133,7 +138,7 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe closeReader(reader); if (!hasSnapshot) { callBack.noNeedToRecover(); - return null; + return CompletableFuture.completedFuture(startReadCursorPosition); } return CompletableFuture.completedFuture(startReadCursorPosition); } catch (Exception ex) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 5fd8f4a388683..f52ff4209c52d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -73,7 +73,7 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess private final PersistentTopic topic; - private CopyOnWriteArrayList lastAbortedTxnIDs = new CopyOnWriteArrayList<>(); + private LinkedList lastAbortedTxnIDs = new LinkedList<>(); //When add abort or change max read position, the count will +1. Take snapshot will set 0 into it. private final AtomicLong changeMaxReadPositionAndAddAbortTimes = new AtomicLong(); @@ -122,7 +122,7 @@ public void appendAbortedTxn(TxnIDData abortedTxnId, PositionImpl position) { //Guarantee the order of the segments. snapshotSegmentQueue.put(position, lastAbortedTxnIDs); takeSnapshotSegment(); - lastAbortedTxnIDs = new CopyOnWriteArrayList<>(); + lastAbortedTxnIDs = new LinkedList<>(); } } @@ -161,6 +161,11 @@ public void updateMaxReadPosition(Position position) { updateSnapshotMetadataByChangeTimes(); } } + @Override + public void updateMaxReadPositionNotIncreaseChangeTimes(Position maxReadPosition) { + this.maxReadPosition = (PositionImpl) maxReadPosition; + } + @Override public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { @@ -309,7 +314,7 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe closeReader(reader); if (!hasIndex) { callBack.noNeedToRecover(); - return null; + return CompletableFuture.completedFuture(startReadCursorPosition); } else { theLatestSnapshotIndexes.getIndexList() .forEach(transactionBufferSnapshotIndex -> @@ -317,7 +322,7 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe transactionBufferSnapshotIndex.persistentPositionLedgerID, transactionBufferSnapshotIndex.persistentPositionEntryID), transactionBufferSnapshotIndex)); - this.lastAbortedTxnIDs = (CopyOnWriteArrayList) theLatestSnapshotIndexes + this.lastAbortedTxnIDs = (LinkedList) theLatestSnapshotIndexes .getSnapshot().getAborts(); this.maxReadPosition = new PositionImpl(theLatestSnapshotIndexes .getSnapshot().getMaxReadPositionLedgerId(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 8b06c64aa1e43..3fcfdf02b8552 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -151,7 +151,8 @@ public void noNeedToRecover() { // completely the normal message have been sent to broker and state is // not NoSnapshot can't sync maxReadPosition snapshotAbortedTxnProcessor - .updateMaxReadPosition(topic.getManagedLedger().getLastConfirmedEntry()); + .updateMaxReadPositionNotIncreaseChangeTimes(topic.getManagedLedger() + .getLastConfirmedEntry()); if (!changeToNoSnapshotState()) { log.error("[{}]Transaction buffer recover fail", topic.getName()); @@ -318,8 +319,6 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { updateMaxReadPosition(txnID); handleLowWaterMark(txnID, lowWaterMark); snapshotAbortedTxnProcessor.trimExpiredTxnIDDataOrSnapshotSegments(); - snapshotAbortedTxnProcessor.appendAbortedTxn(new TxnIDData(txnID.getMostSigBits(), - txnID.getLeastSigBits()), (PositionImpl) position); } txnCommittedCounter.increment(); completableFuture.complete(null); @@ -473,7 +472,7 @@ public void syncMaxReadPositionForNormalPublish(PositionImpl position) { synchronized (TopicTransactionBuffer.this) { if (checkIfNoSnapshot()) { //TODO:The changes time here should not be changed. - snapshotAbortedTxnProcessor.updateMaxReadPosition(position); + snapshotAbortedTxnProcessor.updateMaxReadPositionNotIncreaseChangeTimes(position); } else if (checkIfReady()) { if (ongoingTxns.isEmpty()) { snapshotAbortedTxnProcessor.updateMaxReadPosition(position); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index c1f6ff16e77ab..5f221cbfbbb24 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -61,6 +61,8 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.systopic.SystemTopicClient; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; +import org.apache.pulsar.broker.transaction.buffer.impl.SingleSnapshotAbortedTxnProcessorImpl; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex; @@ -169,7 +171,7 @@ private void recoverTest(String testTopic) throws Exception { Message message = consumer.receive(2, TimeUnit.SECONDS); assertNull(message); - tnx1.commit(); + tnx1.commit().get(); // only can receive message 1 message = consumer.receive(2, TimeUnit.SECONDS); @@ -390,10 +392,14 @@ private void testTopicTransactionBufferDeleteAbort() throws Exception { field.setAccessible(true); TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) field.get(persistentTopic); - field = TopicTransactionBuffer.class.getDeclaredField("aborts"); + field = TopicTransactionBuffer.class.getDeclaredField("snapshotAbortedTxnProcessor"); field.setAccessible(true); + AbortedTxnProcessor abortedTxnProcessor = (AbortedTxnProcessor) field.get(topicTransactionBuffer); + Field abortsField = SingleSnapshotAbortedTxnProcessorImpl.class.getDeclaredField("aborts"); + abortsField.setAccessible(true); + LinkedMap linkedMap = - (LinkedMap) field.get(topicTransactionBuffer); + (LinkedMap) abortsField.get(abortedTxnProcessor); assertEquals(linkedMap.size(), 1); assertEquals(linkedMap.get(linkedMap.firstKey()).getLedgerId(), ((MessageIdImpl) message.getMessageId()).getLedgerId()); @@ -426,9 +432,11 @@ public void clearTransactionBufferSnapshotTest() throws Exception { PersistentTopic originalTopic = (PersistentTopic) getPulsarServiceList().get(0) .getBrokerService().getTopic(TopicName.get(topic).toString(), false).get().get(); TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) originalTopic.getTransactionBuffer(); - Method takeSnapshotMethod = TopicTransactionBuffer.class.getDeclaredMethod("takeSnapshot"); - takeSnapshotMethod.setAccessible(true); - takeSnapshotMethod.invoke(topicTransactionBuffer); + Field abortedTxnProcessorField = TopicTransactionBuffer.class.getDeclaredField("snapshotAbortedTxnProcessor"); + abortedTxnProcessorField.setAccessible(true); + AbortedTxnProcessor abortedTxnProcessor = + (AbortedTxnProcessor) abortedTxnProcessorField.get(topicTransactionBuffer); + abortedTxnProcessor.takesFirstSnapshot(); TopicName transactionBufferTopicName = NamespaceEventsSystemTopicFactory.getSystemTopicName( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 67e7fe268a644..924eae54e16ae 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -91,7 +91,9 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.systopic.SystemTopicClient; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; +import org.apache.pulsar.broker.transaction.buffer.impl.SingleSnapshotAbortedTxnProcessorImpl; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferProvider; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferRecoverCallBack; @@ -661,11 +663,13 @@ public void testMaxReadPositionForNormalPublish() throws Exception { Field field = transactionBufferStateClass.getDeclaredField("state"); field.setAccessible(true); Class topicTransactionBufferClass = TopicTransactionBuffer.class; - Field maxReadPositionField = topicTransactionBufferClass.getDeclaredField("maxReadPosition"); - maxReadPositionField.setAccessible(true); + Field processorField = topicTransactionBufferClass.getDeclaredField("snapshotAbortedTxnProcessor"); + processorField.setAccessible(true); + AbortedTxnProcessor abortedTxnProcessor = (AbortedTxnProcessor) processorField.get(topicTransactionBuffer); + field.set(topicTransactionBuffer, TopicTransactionBufferState.State.Initializing); MessageIdImpl messageId5 = (MessageIdImpl) normalProducer.newMessage().value("normal message").send(); - PositionImpl position5 = (PositionImpl) maxReadPositionField.get(topicTransactionBuffer); + PositionImpl position5 = abortedTxnProcessor.getMaxReadPosition(); Assert.assertEquals(position5.getLedgerId(), messageId4.getLedgerId()); Assert.assertEquals(position5.getEntryId(), messageId4.getEntryId()); } @@ -942,9 +946,10 @@ public void testNoEntryCanBeReadWhenRecovery() throws Exception { filed1.set(persistentTopic, managedLedger); TopicTransactionBuffer topicTransactionBuffer = (TopicTransactionBuffer) field2.get(persistentTopic); - Method method = TopicTransactionBuffer.class.getDeclaredMethod("takeSnapshot"); - method.setAccessible(true); - CompletableFuture completableFuture = (CompletableFuture) method.invoke(topicTransactionBuffer); + Field processorField = TopicTransactionBuffer.class.getDeclaredField("snapshotAbortedTxnProcessor"); + processorField.setAccessible(true); + AbortedTxnProcessor abortedTxnProcessor = (AbortedTxnProcessor) processorField.get(topicTransactionBuffer); + CompletableFuture completableFuture = abortedTxnProcessor.takesFirstSnapshot(); completableFuture.get(); doReturn(PositionImpl.LATEST).when(managedLedger).getLastConfirmedEntry(); @@ -1026,9 +1031,15 @@ public void testNotChangeMaxReadPositionAndAddAbortTimesWhenCheckIfNoSnapshot() .getTopic(NAMESPACE1 + "/changeMaxReadPositionAndAddAbortTimes" + UUID.randomUUID(), true) .get().get(); TransactionBuffer buffer = persistentTopic.getTransactionBuffer(); - Field field = TopicTransactionBuffer.class.getDeclaredField("changeMaxReadPositionAndAddAbortTimes"); - field.setAccessible(true); - AtomicLong changeMaxReadPositionAndAddAbortTimes = (AtomicLong) field.get(buffer); + Field processorField = TopicTransactionBuffer.class.getDeclaredField("snapshotAbortedTxnProcessor"); + processorField.setAccessible(true); + + AbortedTxnProcessor abortedTxnProcessor = (AbortedTxnProcessor) processorField.get(buffer); + Field changeTimeField = SingleSnapshotAbortedTxnProcessorImpl + .class.getDeclaredField("changeMaxReadPositionAndAddAbortTimes"); + changeTimeField.setAccessible(true); + AtomicLong changeMaxReadPositionAndAddAbortTimes = (AtomicLong) changeTimeField.get(abortedTxnProcessor); + Field field1 = TopicTransactionBufferState.class.getDeclaredField("state"); field1.setAccessible(true); From 5b2699cf3400d12d314b13527544718f9ad7429f Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Wed, 26 Oct 2022 14:55:27 +0800 Subject: [PATCH 09/32] optimize and fix some test. --- ...napshotSegmentAbortedTxnProcessorImpl.java | 155 ++++++++++++------ .../TopicTransactionBufferRecoverTest.java | 49 ++++-- 2 files changed, 135 insertions(+), 69 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index f52ff4209c52d..576724ef55180 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -23,11 +23,10 @@ import io.netty.util.Timeout; import io.netty.util.Timer; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -59,35 +58,42 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { private final AtomicLong sequenceID = new AtomicLong(0); - private final ConcurrentSkipListMap> aborts + //Store the lastest aborted transaction IDs and the latest max read position. + private PositionImpl maxReadPosition; + private ArrayList lastAbortedTxnIDs = new ArrayList<>(); + //Store the fixed aborted transaction segment + private final ConcurrentSkipListMap> abortTxnSegment = new ConcurrentSkipListMap<>(); - private final ConcurrentSkipListMap> snapshotSegmentQueue + + //The queue of snapshot segment, the writer of snapshot segment will write segment in the order of the queue. + private final ConcurrentSkipListMap> snapshotSegmentQueue = new ConcurrentSkipListMap<>(); + //Only one segment can be written at the same time. + //Remove the segment from the queue only when Persistenting successfully. + private final Semaphore segmentUpdateSemaphore = new Semaphore(1); private final ConcurrentSkipListMap indexes = new ConcurrentSkipListMap<>(); - + //The latest persistent snapshot index. This is used to combine new segment indexes with the latest metadata and + // indexes. private TransactionBufferSnapshotIndexes theLatestSnapshotIndexes = new TransactionBufferSnapshotIndexes(); private final Timer timer; - private PositionImpl maxReadPosition; private final PersistentTopic topic; - private LinkedList lastAbortedTxnIDs = new LinkedList<>(); - //When add abort or change max read position, the count will +1. Take snapshot will set 0 into it. private final AtomicLong changeMaxReadPositionAndAddAbortTimes = new AtomicLong(); private volatile long lastSnapshotTimestamps; + //Configurations private final int takeSnapshotIntervalNumber; private final int takeSnapshotIntervalTime; private final int transactionBufferMaxAbortedTxnsOfSnapshotSegment; - private final Semaphore segmentUpdateSemaphore = new Semaphore(1); - + //Persistent snapshot segment and index at the single thread. private final CompletableFuture> snapshotSegmentsWriterFuture; private final CompletableFuture> @@ -118,19 +124,18 @@ public void appendAbortedTxn(TxnIDData abortedTxnId, PositionImpl position) { //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. if (lastAbortedTxnIDs.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { changeMaxReadPositionAndAddAbortTimes.set(0); - aborts.put(position, lastAbortedTxnIDs); + abortTxnSegment.put(position, lastAbortedTxnIDs); //Guarantee the order of the segments. snapshotSegmentQueue.put(position, lastAbortedTxnIDs); takeSnapshotSegment(); - lastAbortedTxnIDs = new LinkedList<>(); + lastAbortedTxnIDs = new ArrayList<>(); } } private void takeSnapshotSegment() { //Only one segment can be written at the same time. if (segmentUpdateSemaphore.tryAcquire()) { - CopyOnWriteArrayList abortedTxns = - (CopyOnWriteArrayList) snapshotSegmentQueue.firstEntry().getValue(); + ArrayList abortedTxns = snapshotSegmentQueue.firstEntry().getValue(); PositionImpl maxReadPosition = snapshotSegmentQueue.firstKey(); takeSnapshotSegmentAsync(abortedTxns, maxReadPosition).thenRun(() -> { @@ -170,26 +175,28 @@ public void updateMaxReadPositionNotIncreaseChangeTimes(Position maxReadPosition @Override public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { if (readPosition == null) { - return aborts.values().stream().anyMatch(list -> list.contains(txnID)) || lastAbortedTxnIDs.contains(txnID); + return abortTxnSegment.values().stream() + .anyMatch(list -> list.contains(txnID)) || lastAbortedTxnIDs.contains(txnID); } - List txnIDSet = aborts.ceilingEntry((PositionImpl) readPosition).getValue(); - if (txnIDSet == null) { + Map.Entry> ceilingEntry = abortTxnSegment + .ceilingEntry((PositionImpl) readPosition); + if (ceilingEntry == null) { return lastAbortedTxnIDs.contains(txnID); } else { - return txnIDSet.contains(txnID); + return ceilingEntry.getValue().contains(txnID); } } @Override public void trimExpiredTxnIDDataOrSnapshotSegments() { //Checking whether there are some segment expired. - while (!aborts.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) - .ledgerExists(aborts.firstKey().getLedgerId())) { + while (!abortTxnSegment.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) + .ledgerExists(abortTxnSegment.firstKey().getLedgerId()) && segmentUpdateSemaphore.tryAcquire()) { if (log.isDebugEnabled()) { log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", - topic.getName(), aborts.firstKey()); + topic.getName(), abortTxnSegment.firstKey()); } - PositionImpl positionNeedToDelete = aborts.firstKey(); + PositionImpl positionNeedToDelete = abortTxnSegment.firstKey(); clearSnapshotSegment(positionNeedToDelete); } } @@ -257,18 +264,21 @@ private CompletableFuture updateSnapshotIndex() { .thenRun(() -> { theLatestSnapshotIndexes.setIndexList(snapshotIndexes.getIndexList()); this.lastSnapshotTimestamps = System.currentTimeMillis(); + this.segmentUpdateSemaphore.release(); }) .exceptionally(e -> { log.error("[{}] Failed to update snapshot segment index", snapshotIndexes.getTopicName(), e); + this.segmentUpdateSemaphore.release(); return null; }); } //Update the metadata in the transactionBufferSnapshotIndexes. //Concurrency control is performed by snapshotIndexWriterFuture. - private CompletableFuture updateIndexMetadataForTheLastSnapshot() { - TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); - return snapshotIndexWriterFuture + private void updateIndexMetadataForTheLastSnapshot() { + if (segmentUpdateSemaphore.tryAcquire()) { + TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); + snapshotIndexWriterFuture .thenCompose((indexesWriter) -> { //Store the latest metadata TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = @@ -281,14 +291,50 @@ private CompletableFuture updateIndexMetadataForTheLastSnapshot() { }) .thenRun(() -> { theLatestSnapshotIndexes.setSnapshot(indexes.getSnapshot()); + this.segmentUpdateSemaphore.release(); this.lastSnapshotTimestamps = System.currentTimeMillis(); }) .exceptionally(e -> { + this.segmentUpdateSemaphore.release(); log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); return null; }); + } } + @Override + public CompletableFuture takesFirstSnapshot() { + //TODO: change to acquire? + if (segmentUpdateSemaphore.tryAcquire()) { + TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); + return snapshotIndexWriterFuture + .thenCompose((indexesWriter) -> { + TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = + new TransactionBufferSnapshotSegment(); + transactionBufferSnapshotSegment.setAborts(lastAbortedTxnIDs); + transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); + transactionBufferSnapshotSegment.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); + transactionBufferSnapshotSegment.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); + indexes.setSnapshot(transactionBufferSnapshotSegment); + indexes.setIndexList(new ArrayList<>()); + indexes.setTopicName(this.topic.getName()); + return indexesWriter.writeAsync(topic.getName(), indexes); + }) + .thenRun(() -> { + theLatestSnapshotIndexes.setSnapshot(indexes.getSnapshot()); + indexes.setIndexList(new ArrayList<>()); + indexes.setTopicName(this.topic.getName()); + this.lastSnapshotTimestamps = System.currentTimeMillis(); + }) + .exceptionally(e -> { + log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); + return null; + }); + } + return CompletableFuture.completedFuture(null); + } + + @Override public CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack) { return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() @@ -314,7 +360,7 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe closeReader(reader); if (!hasIndex) { callBack.noNeedToRecover(); - return CompletableFuture.completedFuture(startReadCursorPosition); + return CompletableFuture.completedFuture(null); } else { theLatestSnapshotIndexes.getIndexList() .forEach(transactionBufferSnapshotIndex -> @@ -322,15 +368,17 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe transactionBufferSnapshotIndex.persistentPositionLedgerID, transactionBufferSnapshotIndex.persistentPositionEntryID), transactionBufferSnapshotIndex)); - this.lastAbortedTxnIDs = (LinkedList) theLatestSnapshotIndexes + this.lastAbortedTxnIDs = (ArrayList) theLatestSnapshotIndexes .getSnapshot().getAborts(); this.maxReadPosition = new PositionImpl(theLatestSnapshotIndexes .getSnapshot().getMaxReadPositionLedgerId(), theLatestSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); - sequenceID.set(indexes.lastEntry().getValue().sequenceID + 1); + if (indexes.size() != 0) { + sequenceID.set(indexes.lastEntry().getValue().sequenceID + 1); + } } //Read snapshot segment to recover aborts. - LinkedList> completableFutures = new LinkedList<>(); + ArrayList> completableFutures = new ArrayList<>(); AtomicLong invalidIndex = new AtomicLong(0); AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks .OpenReadOnlyManagedLedgerCallback() { @@ -383,7 +431,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob //Wait the processor recover completely and the allow TB to recover the messages // after the startReadCursorPosition. FutureUtil.waitForAll(completableFutures).get(); - if (invalidIndex.get() != 0) { + if (invalidIndex.get() != 0 && segmentUpdateSemaphore.tryAcquire()) { updateSnapshotIndex(); } return CompletableFuture.completedFuture(startReadCursorPosition); @@ -402,24 +450,29 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob @Override public CompletableFuture clearSnapshot() { ArrayList> completableFutures = new ArrayList<>(); - while (!aborts.isEmpty()) { - if (log.isDebugEnabled()) { - log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", - topic.getName(), aborts.firstKey()); + try { + segmentUpdateSemaphore.acquire(); + while (!abortTxnSegment.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", + topic.getName(), abortTxnSegment.firstKey()); + } + PositionImpl positionNeedToDelete = abortTxnSegment.firstKey(); + completableFutures.add(clearSnapshotSegment(positionNeedToDelete)); } - PositionImpl positionNeedToDelete = aborts.firstKey(); - completableFutures.add(clearSnapshotSegment(positionNeedToDelete)); + return FutureUtil.waitForAll(completableFutures) + .thenCompose((ignore) -> snapshotIndexWriterFuture + .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) + .thenRun(() -> log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", + topic.getName())) + .exceptionally(e -> { + log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", + topic.getName(), e); + return null; + }); + } catch (InterruptedException e) { + return FutureUtil.failedFuture(e); } - return FutureUtil.waitForAll(completableFutures) - .thenCompose((ignore) -> snapshotIndexWriterFuture - .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) - .thenRun(() -> log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", - topic.getName())) - .exceptionally(e -> { - log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", - topic.getName(), e); - return null; - }); } private CompletableFuture clearSnapshotSegment(PositionImpl positionNeedToDelete) { @@ -432,7 +485,7 @@ private CompletableFuture clearSnapshotSegment(PositionImpl positionNeedTo + "whose sequenceId is [{}] and maxReadPosition is [{}]", this.topic.getName(), this.sequenceID, this.maxReadPosition); } - aborts.remove(positionNeedToDelete); + abortTxnSegment.remove(positionNeedToDelete); indexes.remove(positionNeedToDelete); //The process will check whether the snapshot segment is null, and update index when recovered. updateSnapshotIndex(); @@ -444,10 +497,6 @@ private CompletableFuture clearSnapshotSegment(PositionImpl positionNeedTo }); } - @Override - public CompletableFuture takesFirstSnapshot() { - return updateIndexMetadataForTheLastSnapshot(); - } @Override public PositionImpl getMaxReadPosition() { @@ -466,8 +515,8 @@ private void handleSnapshotSegmentEntry(Entry entry) { Commands.parseMessageMetadata(headersAndPayload); TransactionBufferSnapshotSegment snapshotSegment = Schema.AVRO(TransactionBufferSnapshotSegment.class) .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer()); - aborts.put(new PositionImpl(snapshotSegment.getMaxReadPositionLedgerId(), - snapshotSegment.getMaxReadPositionEntryId()), snapshotSegment.getAborts()); + abortTxnSegment.put(new PositionImpl(snapshotSegment.getMaxReadPositionLedgerId(), + snapshotSegment.getMaxReadPositionEntryId()), (ArrayList) snapshotSegment.getAborts()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 5f221cbfbbb24..82667f533eda4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -63,6 +64,7 @@ import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.impl.SingleSnapshotAbortedTxnProcessorImpl; +import org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex; @@ -130,6 +132,14 @@ public Object[] testTopic() { }; } + @DataProvider(name = "enableSnapshotSegment") + public Object[] testSnapshot() { + return new Boolean[] { + true, + false + }; + } + @Test(dataProvider = "testTopic") private void recoverTest(String testTopic) throws Exception { PulsarClient pulsarClient = this.pulsarClient; @@ -244,9 +254,7 @@ private void recoverTest(String testTopic) throws Exception { } - @Test - private void testTakeSnapshot() throws IOException, ExecutionException, InterruptedException { - + private void testTakeSnapshot() throws Exception { @Cleanup Producer producer = pulsarClient .newProducer(Schema.STRING) @@ -316,8 +324,9 @@ private void testTakeSnapshot() throws IOException, ExecutionException, Interrup } - @Test - private void testTopicTransactionBufferDeleteAbort() throws Exception { + @Test(dataProvider = "enableSnapshotSegment") + private void testTopicTransactionBufferDeleteAbort(Boolean enableSnapshotSegment) throws Exception { + getPulsarServiceList().get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(enableSnapshotSegment); @Cleanup Producer producer = pulsarClient .newProducer(Schema.STRING) @@ -395,23 +404,31 @@ private void testTopicTransactionBufferDeleteAbort() throws Exception { field = TopicTransactionBuffer.class.getDeclaredField("snapshotAbortedTxnProcessor"); field.setAccessible(true); AbortedTxnProcessor abortedTxnProcessor = (AbortedTxnProcessor) field.get(topicTransactionBuffer); - Field abortsField = SingleSnapshotAbortedTxnProcessorImpl.class.getDeclaredField("aborts"); - abortsField.setAccessible(true); - - LinkedMap linkedMap = - (LinkedMap) abortsField.get(abortedTxnProcessor); - assertEquals(linkedMap.size(), 1); - assertEquals(linkedMap.get(linkedMap.firstKey()).getLedgerId(), - ((MessageIdImpl) message.getMessageId()).getLedgerId()); - exist = true; + + if (enableSnapshotSegment) { + //TODO: + exist = true; + } else { + Field abortsField = SingleSnapshotAbortedTxnProcessorImpl.class.getDeclaredField("aborts"); + abortsField.setAccessible(true); + + LinkedMap linkedMap = + (LinkedMap) abortsField.get(abortedTxnProcessor); + assertEquals(linkedMap.size(), 1); + assertEquals(linkedMap.get(linkedMap.firstKey()).getLedgerId(), + ((MessageIdImpl) message.getMessageId()).getLedgerId()); + exist = true; + } + } } } assertTrue(exist); } - @Test - public void clearTransactionBufferSnapshotTest() throws Exception { + @Test(dataProvider = "enableSnapshotSegment") + public void clearTransactionBufferSnapshotTest(Boolean enableSnapshotSegment) throws Exception { + getPulsarServiceList().get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(enableSnapshotSegment); String topic = NAMESPACE1 + "/tb-snapshot-delete-" + RandomUtils.nextInt(); Producer producer = pulsarClient From 04741728518a201bc7353d84f90695498dc020e5 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Wed, 26 Oct 2022 20:48:10 +0800 Subject: [PATCH 10/32] chenge semaphore to state and some optimize --- .../service/AbstractBaseDispatcher.java | 3 +- .../service/persistent/PersistentTopic.java | 4 +- .../transaction/buffer/TransactionBuffer.java | 3 +- .../buffer/impl/InMemTransactionBuffer.java | 2 +- ...SingleSnapshotAbortedTxnProcessorImpl.java | 2 +- ...napshotSegmentAbortedTxnProcessorImpl.java | 248 ++++++++++-------- .../buffer/impl/TopicTransactionBuffer.java | 6 +- .../buffer/impl/TransactionBufferDisable.java | 2 +- .../v2/TransactionBufferSnapshotIndexes.java | 2 +- ...nsactionBufferSnapshotIndexesMetadata.java | 33 +++ .../service/AbstractBaseDispatcherTest.java | 2 +- 11 files changed, 188 insertions(+), 119 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index 5069f7cd44000..4a3e34439683c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -164,7 +164,8 @@ public int filterEntriesForConsumer(Optional optMetadataArray entry.release(); continue; } else if (((PersistentTopic) subscription.getTopic()) - .isTxnAborted(new TxnID(msgMetadata.getTxnidMostBits(), msgMetadata.getTxnidLeastBits()))) { + .isTxnAborted(new TxnID(msgMetadata.getTxnidMostBits(), msgMetadata.getTxnidLeastBits()), + (PositionImpl) entry.getPosition())) { individualAcknowledgeMessageIfNeeded(entry.getPosition(), Collections.emptyMap()); entries.set(i, null); entry.release(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 68cb51f41bbc1..7b993c3231e08 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3328,8 +3328,8 @@ public PositionImpl getMaxReadPosition() { return this.transactionBuffer.getMaxReadPosition(); } - public boolean isTxnAborted(TxnID txnID) { - return this.transactionBuffer.isTxnAborted(txnID); + public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { + return this.transactionBuffer.isTxnAborted(txnID, readPosition); } public TransactionInBufferStats getTransactionInBufferStats(TxnID txnID) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java index ab1270ef0e67f..da799d4aebc7d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/TransactionBuffer.java @@ -141,9 +141,10 @@ public interface TransactionBuffer { /** * Close the buffer asynchronously. * @param txnID {@link TxnID} txnId. + * @param readPosition the persitent position of the txn message. * @return the txnId is aborted. */ - boolean isTxnAborted(TxnID txnID); + boolean isTxnAborted(TxnID txnID, PositionImpl readPosition); /** * Sync max read position for normal publish. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java index c4a9fc2b77407..e984e3ab58e25 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/InMemTransactionBuffer.java @@ -360,7 +360,7 @@ public CompletableFuture closeAsync() { } @Override - public boolean isTxnAborted(TxnID txnID) { + public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { return false; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 5d81dade55258..9cbb98b6f931e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -138,7 +138,7 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe closeReader(reader); if (!hasSnapshot) { callBack.noNeedToRecover(); - return CompletableFuture.completedFuture(startReadCursorPosition); + return CompletableFuture.completedFuture(null); } return CompletableFuture.completedFuture(startReadCursorPosition); } catch (Exception ex) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 576724ef55180..e829ebd29b344 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -27,9 +27,9 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; @@ -43,6 +43,7 @@ import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexesMetadata; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; import org.apache.pulsar.client.api.Message; @@ -56,27 +57,44 @@ @Slf4j public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { + + public enum OperationState { + None, + UpdatingIndex, + TakingSegment, + DeletingSegment, + Closing, + Closed + } + + private volatile OperationState operationState = OperationState.None; private final AtomicLong sequenceID = new AtomicLong(0); - //Store the lastest aborted transaction IDs and the latest max read position. + //Store the latest aborted transaction IDs and the latest max read position. private PositionImpl maxReadPosition; - private ArrayList lastAbortedTxnIDs = new ArrayList<>(); + private ArrayList unsealedAbortedTxnIdSegment = new ArrayList<>(); + //Store the fixed aborted transaction segment - private final ConcurrentSkipListMap> abortTxnSegment + private final ConcurrentSkipListMap> abortTxnSegments = new ConcurrentSkipListMap<>(); //The queue of snapshot segment, the writer of snapshot segment will write segment in the order of the queue. + //Only one segment can be written at the same time. + //Remove the segment from the queue only when persistent successfully. private final ConcurrentSkipListMap> snapshotSegmentQueue = new ConcurrentSkipListMap<>(); - //Only one segment can be written at the same time. - //Remove the segment from the queue only when Persistenting successfully. - private final Semaphore segmentUpdateSemaphore = new Semaphore(1); + + private static final AtomicReferenceFieldUpdater STATE_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(SnapshotSegmentAbortedTxnProcessorImpl.class, + SnapshotSegmentAbortedTxnProcessorImpl.OperationState.class, "operationState"); private final ConcurrentSkipListMap indexes = new ConcurrentSkipListMap<>(); //The latest persistent snapshot index. This is used to combine new segment indexes with the latest metadata and // indexes. - private TransactionBufferSnapshotIndexes theLatestSnapshotIndexes = new TransactionBufferSnapshotIndexes(); + private TransactionBufferSnapshotIndexes persistentSnapshotIndexes = new TransactionBufferSnapshotIndexes(); + private final Timer timer; private final PersistentTopic topic; @@ -120,21 +138,21 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { @Override public void appendAbortedTxn(TxnIDData abortedTxnId, PositionImpl position) { - lastAbortedTxnIDs.add(abortedTxnId); + unsealedAbortedTxnIdSegment.add(abortedTxnId); //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. - if (lastAbortedTxnIDs.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { + if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { changeMaxReadPositionAndAddAbortTimes.set(0); - abortTxnSegment.put(position, lastAbortedTxnIDs); + abortTxnSegments.put(position, unsealedAbortedTxnIdSegment); //Guarantee the order of the segments. - snapshotSegmentQueue.put(position, lastAbortedTxnIDs); + snapshotSegmentQueue.put(position, unsealedAbortedTxnIdSegment); takeSnapshotSegment(); - lastAbortedTxnIDs = new ArrayList<>(); + unsealedAbortedTxnIdSegment = new ArrayList<>(); } } private void takeSnapshotSegment() { //Only one segment can be written at the same time. - if (segmentUpdateSemaphore.tryAcquire()) { + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.TakingSegment)) { ArrayList abortedTxns = snapshotSegmentQueue.firstEntry().getValue(); PositionImpl maxReadPosition = snapshotSegmentQueue.firstKey(); @@ -144,6 +162,7 @@ private void takeSnapshotSegment() { + "for the topic [{}], and the size of the segment is [{}]", sequenceID, maxReadPosition, topic.getName(), abortedTxns.size()); } + snapshotSegmentQueue.remove(maxReadPosition); sequenceID.getAndIncrement(); }).exceptionally(e -> { //Just log the error, and the processor will try to take snapshot again when the transactionBuffer @@ -163,7 +182,7 @@ private void takeSnapshotSegment() { public void updateMaxReadPosition(Position position) { if (position != this.maxReadPosition) { this.maxReadPosition = (PositionImpl) position; - updateSnapshotMetadataByChangeTimes(); + updateSnapshotIndexMetadataByChangeTimes(); } } @Override @@ -175,13 +194,13 @@ public void updateMaxReadPositionNotIncreaseChangeTimes(Position maxReadPosition @Override public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { if (readPosition == null) { - return abortTxnSegment.values().stream() - .anyMatch(list -> list.contains(txnID)) || lastAbortedTxnIDs.contains(txnID); + return abortTxnSegments.values().stream() + .anyMatch(list -> list.contains(txnID)) || unsealedAbortedTxnIdSegment.contains(txnID); } - Map.Entry> ceilingEntry = abortTxnSegment + Map.Entry> ceilingEntry = abortTxnSegments .ceilingEntry((PositionImpl) readPosition); if (ceilingEntry == null) { - return lastAbortedTxnIDs.contains(txnID); + return unsealedAbortedTxnIdSegment.contains(txnID); } else { return ceilingEntry.getValue().contains(txnID); } @@ -190,14 +209,15 @@ public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { @Override public void trimExpiredTxnIDDataOrSnapshotSegments() { //Checking whether there are some segment expired. - while (!abortTxnSegment.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) - .ledgerExists(abortTxnSegment.firstKey().getLedgerId()) && segmentUpdateSemaphore.tryAcquire()) { + while (!abortTxnSegments.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) + .ledgerExists(abortTxnSegments.firstKey().getLedgerId()) + && STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.DeletingSegment)) { if (log.isDebugEnabled()) { log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", - topic.getName(), abortTxnSegment.firstKey()); + topic.getName(), abortTxnSegments.firstKey()); } - PositionImpl positionNeedToDelete = abortTxnSegment.firstKey(); - clearSnapshotSegment(positionNeedToDelete); + PositionImpl positionNeedToDelete = abortTxnSegments.firstKey(); + deleteSnapshotSegment(positionNeedToDelete); } } @@ -205,16 +225,29 @@ private String buildKey(long sequenceId) { return "multiple-" + sequenceId + this.topic.getName(); } - private void updateSnapshotMetadataByChangeTimes() { + private void updateSnapshotIndexMetadataByChangeTimes() { if (this.changeMaxReadPositionAndAddAbortTimes.incrementAndGet() == takeSnapshotIntervalNumber) { - changeMaxReadPositionAndAddAbortTimes.set(0); - updateIndexMetadataForTheLastSnapshot(); + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.UpdatingIndex)) { + changeMaxReadPositionAndAddAbortTimes.set(0); + if (snapshotSegmentQueue.isEmpty()) { + updateIndexMetadataForTheLastSnapshot(); + } else { + takeSnapshotSegment(); + } + } } } private void takeSnapshotByTimeout() { if (changeMaxReadPositionAndAddAbortTimes.get() > 0) { - updateIndexMetadataForTheLastSnapshot(); + changeMaxReadPositionAndAddAbortTimes.set(0); + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.UpdatingIndex)) { + if (snapshotSegmentQueue.isEmpty()) { + updateIndexMetadataForTheLastSnapshot(); + } else { + takeSnapshotSegment(); + } + } } timer.newTimeout(SnapshotSegmentAbortedTxnProcessorImpl.this, takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); @@ -246,82 +279,82 @@ private CompletableFuture takeSnapshotSegmentAsync(List segment indexes.put(maxReadPosition, index); //update snapshot segment index. - return updateSnapshotIndex(); + return updateSnapshotIndex(maxReadPosition, new ArrayList<>()); }); } - //Update the indexes in the transactionBufferSnapshotIndexe. + //Update the indexes and metadata in the transactionBufferSnapshotIndexe. //Concurrency control is performed by snapshotIndexWriterFuture. - private CompletableFuture updateSnapshotIndex() { + private CompletableFuture updateSnapshotIndex(PositionImpl maxReadPosition, + ArrayList unsealedAbortedTxnIdSegment) { TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); return snapshotIndexWriterFuture .thenCompose((indexesWriter) -> { snapshotIndexes.setIndexList(indexes.values().stream().toList()); - //Only update the index in indexes and keep the metadata in indexes unchanged. - snapshotIndexes.setSnapshot(theLatestSnapshotIndexes.getSnapshot()); + //update the metadata in the idnexes. + snapshotIndexes.setSnapshot(new TransactionBufferSnapshotIndexesMetadata( + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), unsealedAbortedTxnIdSegment)); return indexesWriter.writeAsync(topic.getName(), snapshotIndexes); }) .thenRun(() -> { - theLatestSnapshotIndexes.setIndexList(snapshotIndexes.getIndexList()); + persistentSnapshotIndexes.setIndexList(snapshotIndexes.getIndexList()); this.lastSnapshotTimestamps = System.currentTimeMillis(); - this.segmentUpdateSemaphore.release(); + STATE_UPDATER.set(this, OperationState.None); }) .exceptionally(e -> { log.error("[{}] Failed to update snapshot segment index", snapshotIndexes.getTopicName(), e); - this.segmentUpdateSemaphore.release(); + STATE_UPDATER.set(this, OperationState.None); return null; }); } - //Update the metadata in the transactionBufferSnapshotIndexes. + //Only update the metadata in the transactionBufferSnapshotIndexes. //Concurrency control is performed by snapshotIndexWriterFuture. private void updateIndexMetadataForTheLastSnapshot() { - if (segmentUpdateSemaphore.tryAcquire()) { - TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); - snapshotIndexWriterFuture - .thenCompose((indexesWriter) -> { - //Store the latest metadata - TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = - new TransactionBufferSnapshotSegment(); - transactionBufferSnapshotSegment.setAborts(lastAbortedTxnIDs); - indexes.setSnapshot(transactionBufferSnapshotSegment); - //Only update the metadata in indexes and keep the index in indexes unchanged. - indexes.setIndexList(theLatestSnapshotIndexes.getIndexList()); - return indexesWriter.writeAsync(topic.getName(), indexes); - }) - .thenRun(() -> { - theLatestSnapshotIndexes.setSnapshot(indexes.getSnapshot()); - this.segmentUpdateSemaphore.release(); - this.lastSnapshotTimestamps = System.currentTimeMillis(); - }) - .exceptionally(e -> { - this.segmentUpdateSemaphore.release(); - log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); - return null; - }); - } + TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); + snapshotIndexWriterFuture + .thenCompose((indexesWriter) -> { + //Store the latest metadata + TransactionBufferSnapshotIndexesMetadata transactionBufferSnapshotSegment = + new TransactionBufferSnapshotIndexesMetadata(); + transactionBufferSnapshotSegment.setAborts(unsealedAbortedTxnIdSegment); + indexes.setSnapshot(transactionBufferSnapshotSegment); + //Only update the metadata in indexes and keep the index in indexes unchanged. + indexes.setIndexList(persistentSnapshotIndexes.getIndexList()); + return indexesWriter.writeAsync(topic.getName(), indexes); + }) + .thenRun(() -> { + persistentSnapshotIndexes.setSnapshot(indexes.getSnapshot()); + STATE_UPDATER.set(this, OperationState.None); + this.lastSnapshotTimestamps = System.currentTimeMillis(); + }) + .exceptionally(e -> { + STATE_UPDATER.set(this, OperationState.None); + log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); + return null; + }); + } @Override public CompletableFuture takesFirstSnapshot() { - //TODO: change to acquire? - if (segmentUpdateSemaphore.tryAcquire()) { + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.UpdatingIndex)) { TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); return snapshotIndexWriterFuture .thenCompose((indexesWriter) -> { - TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = - new TransactionBufferSnapshotSegment(); - transactionBufferSnapshotSegment.setAborts(lastAbortedTxnIDs); - transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); - transactionBufferSnapshotSegment.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); - transactionBufferSnapshotSegment.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); - indexes.setSnapshot(transactionBufferSnapshotSegment); + TransactionBufferSnapshotIndexesMetadata transactionBufferSnapshotIndexesMetadata = + new TransactionBufferSnapshotIndexesMetadata(); + transactionBufferSnapshotIndexesMetadata.setAborts(unsealedAbortedTxnIdSegment); + transactionBufferSnapshotIndexesMetadata.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); + transactionBufferSnapshotIndexesMetadata.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); + indexes.setSnapshot(transactionBufferSnapshotIndexesMetadata); indexes.setIndexList(new ArrayList<>()); indexes.setTopicName(this.topic.getName()); return indexesWriter.writeAsync(topic.getName(), indexes); }) .thenRun(() -> { - theLatestSnapshotIndexes.setSnapshot(indexes.getSnapshot()); + //TODO: check again + persistentSnapshotIndexes.setSnapshot(indexes.getSnapshot()); indexes.setIndexList(new ArrayList<>()); indexes.setTopicName(this.topic.getName()); this.lastSnapshotTimestamps = System.currentTimeMillis(); @@ -350,7 +383,7 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe TransactionBufferSnapshotIndexes transactionBufferSnapshotIndexes = message.getValue(); if (transactionBufferSnapshotIndexes != null) { hasIndex = true; - this.theLatestSnapshotIndexes = transactionBufferSnapshotIndexes; + this.persistentSnapshotIndexes = transactionBufferSnapshotIndexes; startReadCursorPosition = PositionImpl.get( transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionLedgerId(), transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); @@ -362,17 +395,17 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe callBack.noNeedToRecover(); return CompletableFuture.completedFuture(null); } else { - theLatestSnapshotIndexes.getIndexList() + persistentSnapshotIndexes.getIndexList() .forEach(transactionBufferSnapshotIndex -> indexes.put(new PositionImpl( transactionBufferSnapshotIndex.persistentPositionLedgerID, transactionBufferSnapshotIndex.persistentPositionEntryID), transactionBufferSnapshotIndex)); - this.lastAbortedTxnIDs = (ArrayList) theLatestSnapshotIndexes + this.unsealedAbortedTxnIdSegment = (ArrayList) persistentSnapshotIndexes .getSnapshot().getAborts(); - this.maxReadPosition = new PositionImpl(theLatestSnapshotIndexes + this.maxReadPosition = new PositionImpl(persistentSnapshotIndexes .getSnapshot().getMaxReadPositionLedgerId(), - theLatestSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); + persistentSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); if (indexes.size() != 0) { sequenceID.set(indexes.lastEntry().getValue().sequenceID + 1); } @@ -384,7 +417,7 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe .OpenReadOnlyManagedLedgerCallback() { @Override public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { - theLatestSnapshotIndexes.getIndexList().forEach(index -> { + persistentSnapshotIndexes.getIndexList().forEach(index -> { CompletableFuture completableFuture1 = new CompletableFuture<>(); completableFutures.add(completableFuture1); readOnlyManagedLedger.asyncReadEntry( @@ -431,8 +464,9 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob //Wait the processor recover completely and the allow TB to recover the messages // after the startReadCursorPosition. FutureUtil.waitForAll(completableFutures).get(); - if (invalidIndex.get() != 0 && segmentUpdateSemaphore.tryAcquire()) { - updateSnapshotIndex(); + if (invalidIndex.get() != 0 && STATE_UPDATER.compareAndSet(this, + OperationState.None, OperationState.UpdatingIndex)) { + updateSnapshotIndex(this.maxReadPosition, this.unsealedAbortedTxnIdSegment); } return CompletableFuture.completedFuture(startReadCursorPosition); } catch (Exception ex) { @@ -450,32 +484,32 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob @Override public CompletableFuture clearSnapshot() { ArrayList> completableFutures = new ArrayList<>(); - try { - segmentUpdateSemaphore.acquire(); - while (!abortTxnSegment.isEmpty()) { - if (log.isDebugEnabled()) { - log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", - topic.getName(), abortTxnSegment.firstKey()); - } - PositionImpl positionNeedToDelete = abortTxnSegment.firstKey(); - completableFutures.add(clearSnapshotSegment(positionNeedToDelete)); + //TODO: Wait all operation completely and then clear the snapshot + STATE_UPDATER.set(this, OperationState.Closing); + while (!abortTxnSegments.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", + topic.getName(), abortTxnSegments.firstKey()); } - return FutureUtil.waitForAll(completableFutures) - .thenCompose((ignore) -> snapshotIndexWriterFuture - .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) - .thenRun(() -> log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", - topic.getName())) - .exceptionally(e -> { - log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", - topic.getName(), e); - return null; - }); - } catch (InterruptedException e) { - return FutureUtil.failedFuture(e); + PositionImpl positionNeedToDelete = abortTxnSegments.firstKey(); + completableFutures.add(deleteSnapshotSegment(positionNeedToDelete)); } + return FutureUtil.waitForAll(completableFutures) + .thenCompose((ignore) -> snapshotIndexWriterFuture + .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) + .thenRun(() -> { + log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", + topic.getName()); + STATE_UPDATER.compareAndSet(this, OperationState.Closing, OperationState.Closed); + }) + .exceptionally(e -> { + log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", + topic.getName(), e); + return null; + }); } - private CompletableFuture clearSnapshotSegment(PositionImpl positionNeedToDelete) { + private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedToDelete) { long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); return snapshotSegmentsWriterFuture.thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) @@ -485,14 +519,14 @@ private CompletableFuture clearSnapshotSegment(PositionImpl positionNeedTo + "whose sequenceId is [{}] and maxReadPosition is [{}]", this.topic.getName(), this.sequenceID, this.maxReadPosition); } - abortTxnSegment.remove(positionNeedToDelete); - indexes.remove(positionNeedToDelete); + abortTxnSegments.remove(positionNeedToDelete); //The process will check whether the snapshot segment is null, and update index when recovered. - updateSnapshotIndex(); + indexes.remove(positionNeedToDelete); + updateSnapshotIndex(this.maxReadPosition, this.unsealedAbortedTxnIdSegment); }).exceptionally(e -> { - log.error("[{}] Failed to delete the snapshot segment, " + log.warn("[{}] Failed to delete the snapshot segment, " + "whose sequenceId is [{}] and maxReadPosition is [{}]", - this.topic.getName(), this.sequenceID, this.maxReadPosition); + this.topic.getName(), this.sequenceID, this.maxReadPosition, e); return null; }); } @@ -515,7 +549,7 @@ private void handleSnapshotSegmentEntry(Entry entry) { Commands.parseMessageMetadata(headersAndPayload); TransactionBufferSnapshotSegment snapshotSegment = Schema.AVRO(TransactionBufferSnapshotSegment.class) .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer()); - abortTxnSegment.put(new PositionImpl(snapshotSegment.getMaxReadPositionLedgerId(), + abortTxnSegments.put(new PositionImpl(snapshotSegment.getMaxReadPositionLedgerId(), snapshotSegment.getMaxReadPositionEntryId()), (ArrayList) snapshotSegment.getAborts()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 3fcfdf02b8552..f36a936c25ad3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -460,9 +460,9 @@ public CompletableFuture closeAsync() { } @Override - public boolean isTxnAborted(TxnID txnID) { + public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { return snapshotAbortedTxnProcessor.checkAbortedTransaction( - new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits()), null); + new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits()), readPosition); } @Override @@ -493,7 +493,7 @@ public PositionImpl getMaxReadPosition() { @Override public TransactionInBufferStats getTransactionInBufferStats(TxnID txnID) { TransactionInBufferStats transactionInBufferStats = new TransactionInBufferStats(); - transactionInBufferStats.aborted = isTxnAborted(txnID); + transactionInBufferStats.aborted = isTxnAborted(txnID, null); if (ongoingTxns.containsKey(txnID)) { transactionInBufferStats.startPosition = ongoingTxns.get(txnID).toString(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java index d700195416c1f..dcda9f836a5c0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferDisable.java @@ -79,7 +79,7 @@ public CompletableFuture closeAsync() { } @Override - public boolean isTxnAborted(TxnID txnID) { + public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { return false; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexes.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexes.java index 28b2b05a4969e..2c417e32a78fa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexes.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexes.java @@ -35,6 +35,6 @@ public class TransactionBufferSnapshotIndexes { private List indexList; - private TransactionBufferSnapshotSegment snapshot; + private TransactionBufferSnapshotIndexesMetadata snapshot; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java new file mode 100644 index 0000000000000..e5194e2ab5b6e --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.buffer.metadata.v2; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class TransactionBufferSnapshotIndexesMetadata { + private long maxReadPositionLedgerId; + private long maxReadPositionEntryId; + private List aborts; +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java index cba15b0631006..5c8dd8aaac827 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractBaseDispatcherTest.java @@ -130,7 +130,7 @@ public void testFilterEntriesForConsumerOfTxnBufferAbort() { PersistentTopic mockTopic = mock(PersistentTopic.class); when(this.subscriptionMock.getTopic()).thenReturn(mockTopic); - when(mockTopic.isTxnAborted(any(TxnID.class))).thenReturn(true); + when(mockTopic.isTxnAborted(any(TxnID.class), any())).thenReturn(true); List entries = new ArrayList<>(); entries.add(EntryImpl.create(1, 1, createTnxMessage("message1", 1))); From 393bd0cf66f93e1af31d238d150a99cd05272e92 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Thu, 27 Oct 2022 22:54:58 +0800 Subject: [PATCH 11/32] Add persistent Worker --- ...napshotSegmentAbortedTxnProcessorImpl.java | 494 +++++++++++------- .../buffer/impl/TopicTransactionBuffer.java | 7 +- 2 files changed, 295 insertions(+), 206 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index e829ebd29b344..584d8ccd30ba2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -9,7 +9,7 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, + * Unless required by applicable law or agreed to in writing,2 * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the @@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; @@ -58,18 +59,6 @@ @Slf4j public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { - public enum OperationState { - None, - UpdatingIndex, - TakingSegment, - DeletingSegment, - Closing, - Closed - } - - private volatile OperationState operationState = OperationState.None; - private final AtomicLong sequenceID = new AtomicLong(0); - //Store the latest aborted transaction IDs and the latest max read position. private PositionImpl maxReadPosition; private ArrayList unsealedAbortedTxnIdSegment = new ArrayList<>(); @@ -78,17 +67,6 @@ public enum OperationState { private final ConcurrentSkipListMap> abortTxnSegments = new ConcurrentSkipListMap<>(); - //The queue of snapshot segment, the writer of snapshot segment will write segment in the order of the queue. - //Only one segment can be written at the same time. - //Remove the segment from the queue only when persistent successfully. - private final ConcurrentSkipListMap> snapshotSegmentQueue - = new ConcurrentSkipListMap<>(); - - private static final AtomicReferenceFieldUpdater STATE_UPDATER = - AtomicReferenceFieldUpdater.newUpdater(SnapshotSegmentAbortedTxnProcessorImpl.class, - SnapshotSegmentAbortedTxnProcessorImpl.OperationState.class, "operationState"); - private final ConcurrentSkipListMap indexes = new ConcurrentSkipListMap<>(); //The latest persistent snapshot index. This is used to combine new segment indexes with the latest metadata and @@ -116,9 +94,11 @@ public enum OperationState { snapshotSegmentsWriterFuture; private final CompletableFuture> snapshotIndexWriterFuture; + private final PersistentWorker persistentWorker; public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; + this.persistentWorker = new PersistentWorker(topic); this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); this.takeSnapshotIntervalNumber = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMaxTransactionCount(); @@ -143,41 +123,12 @@ public void appendAbortedTxn(TxnIDData abortedTxnId, PositionImpl position) { if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { changeMaxReadPositionAndAddAbortTimes.set(0); abortTxnSegments.put(position, unsealedAbortedTxnIdSegment); - //Guarantee the order of the segments. - snapshotSegmentQueue.put(position, unsealedAbortedTxnIdSegment); - takeSnapshotSegment(); + persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> + persistentWorker.takeSnapshotSegmentAsync(unsealedAbortedTxnIdSegment, position)); unsealedAbortedTxnIdSegment = new ArrayList<>(); } } - private void takeSnapshotSegment() { - //Only one segment can be written at the same time. - if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.TakingSegment)) { - ArrayList abortedTxns = snapshotSegmentQueue.firstEntry().getValue(); - PositionImpl maxReadPosition = snapshotSegmentQueue.firstKey(); - - takeSnapshotSegmentAsync(abortedTxns, maxReadPosition).thenRun(() -> { - if (log.isDebugEnabled()) { - log.debug("Successes to take snapshot segment [{}] at maxReadPosition [{}] " - + "for the topic [{}], and the size of the segment is [{}]", - sequenceID, maxReadPosition, topic.getName(), abortedTxns.size()); - } - snapshotSegmentQueue.remove(maxReadPosition); - sequenceID.getAndIncrement(); - }).exceptionally(e -> { - //Just log the error, and the processor will try to take snapshot again when the transactionBuffer - //append aborted txn nex time. - log.error("Failed to take snapshot segment [{}] at maxReadPosition [{}] " - + "for the topic [{}], and the size of the segment is [{}]", - sequenceID, maxReadPosition, topic.getName(), abortedTxns.size(), e); - //Try again - timer.newTimeout((ignore) -> takeSnapshotSegment(), takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); - return null; - }); - } - } - - @Override public void updateMaxReadPosition(Position position) { if (position != this.maxReadPosition) { @@ -185,12 +136,12 @@ public void updateMaxReadPosition(Position position) { updateSnapshotIndexMetadataByChangeTimes(); } } + @Override public void updateMaxReadPositionNotIncreaseChangeTimes(Position maxReadPosition) { this.maxReadPosition = (PositionImpl) maxReadPosition; } - @Override public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { if (readPosition == null) { @@ -210,14 +161,14 @@ public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { public void trimExpiredTxnIDDataOrSnapshotSegments() { //Checking whether there are some segment expired. while (!abortTxnSegments.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) - .ledgerExists(abortTxnSegments.firstKey().getLedgerId()) - && STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.DeletingSegment)) { + .ledgerExists(abortTxnSegments.firstKey().getLedgerId())) { if (log.isDebugEnabled()) { log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", topic.getName(), abortTxnSegments.firstKey()); } PositionImpl positionNeedToDelete = abortTxnSegments.firstKey(); - deleteSnapshotSegment(positionNeedToDelete); + persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment, + () -> persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); } } @@ -227,27 +178,17 @@ private String buildKey(long sequenceId) { private void updateSnapshotIndexMetadataByChangeTimes() { if (this.changeMaxReadPositionAndAddAbortTimes.incrementAndGet() == takeSnapshotIntervalNumber) { - if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.UpdatingIndex)) { - changeMaxReadPositionAndAddAbortTimes.set(0); - if (snapshotSegmentQueue.isEmpty()) { - updateIndexMetadataForTheLastSnapshot(); - } else { - takeSnapshotSegment(); - } - } + changeMaxReadPositionAndAddAbortTimes.set(0); + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, + persistentWorker::updateIndexMetadataForTheLastSnapshot); } } private void takeSnapshotByTimeout() { if (changeMaxReadPositionAndAddAbortTimes.get() > 0) { changeMaxReadPositionAndAddAbortTimes.set(0); - if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.UpdatingIndex)) { - if (snapshotSegmentQueue.isEmpty()) { - updateIndexMetadataForTheLastSnapshot(); - } else { - takeSnapshotSegment(); - } - } + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, + persistentWorker::updateIndexMetadataForTheLastSnapshot); } timer.newTimeout(SnapshotSegmentAbortedTxnProcessorImpl.this, takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); @@ -258,87 +199,11 @@ public void run(Timeout timeout) { takeSnapshotByTimeout(); } - private CompletableFuture takeSnapshotSegmentAsync(List segment, PositionImpl maxReadPosition) { - TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = new TransactionBufferSnapshotSegment(); - transactionBufferSnapshotSegment.setAborts(segment); - transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); - transactionBufferSnapshotSegment.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); - transactionBufferSnapshotSegment.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); - - return snapshotSegmentsWriterFuture.thenCompose(segmentWriter -> { - transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get()); - return segmentWriter.writeAsync(buildKey(this.sequenceID.get()), transactionBufferSnapshotSegment); - }).thenCompose((messageId) -> { - //Build index for this segment - TransactionBufferSnapshotIndex index = new TransactionBufferSnapshotIndex(); - index.setSequenceID(transactionBufferSnapshotSegment.getSequenceId()); - index.setMaxReadPositionLedgerID(maxReadPosition.getLedgerId()); - index.setMaxReadPositionEntryID(maxReadPosition.getEntryId()); - index.setPersistentPositionLedgerID(((MessageIdImpl) messageId).getLedgerId()); - index.setPersistentPositionEntryID(((MessageIdImpl) messageId).getEntryId()); - - indexes.put(maxReadPosition, index); - //update snapshot segment index. - return updateSnapshotIndex(maxReadPosition, new ArrayList<>()); - }); - } - - //Update the indexes and metadata in the transactionBufferSnapshotIndexe. - //Concurrency control is performed by snapshotIndexWriterFuture. - private CompletableFuture updateSnapshotIndex(PositionImpl maxReadPosition, - ArrayList unsealedAbortedTxnIdSegment) { - TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); - return snapshotIndexWriterFuture - .thenCompose((indexesWriter) -> { - snapshotIndexes.setIndexList(indexes.values().stream().toList()); - //update the metadata in the idnexes. - snapshotIndexes.setSnapshot(new TransactionBufferSnapshotIndexesMetadata( - maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), unsealedAbortedTxnIdSegment)); - return indexesWriter.writeAsync(topic.getName(), snapshotIndexes); - }) - .thenRun(() -> { - persistentSnapshotIndexes.setIndexList(snapshotIndexes.getIndexList()); - this.lastSnapshotTimestamps = System.currentTimeMillis(); - STATE_UPDATER.set(this, OperationState.None); - }) - .exceptionally(e -> { - log.error("[{}] Failed to update snapshot segment index", snapshotIndexes.getTopicName(), e); - STATE_UPDATER.set(this, OperationState.None); - return null; - }); - } - - //Only update the metadata in the transactionBufferSnapshotIndexes. - //Concurrency control is performed by snapshotIndexWriterFuture. - private void updateIndexMetadataForTheLastSnapshot() { - TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); - snapshotIndexWriterFuture - .thenCompose((indexesWriter) -> { - //Store the latest metadata - TransactionBufferSnapshotIndexesMetadata transactionBufferSnapshotSegment = - new TransactionBufferSnapshotIndexesMetadata(); - transactionBufferSnapshotSegment.setAborts(unsealedAbortedTxnIdSegment); - indexes.setSnapshot(transactionBufferSnapshotSegment); - //Only update the metadata in indexes and keep the index in indexes unchanged. - indexes.setIndexList(persistentSnapshotIndexes.getIndexList()); - return indexesWriter.writeAsync(topic.getName(), indexes); - }) - .thenRun(() -> { - persistentSnapshotIndexes.setSnapshot(indexes.getSnapshot()); - STATE_UPDATER.set(this, OperationState.None); - this.lastSnapshotTimestamps = System.currentTimeMillis(); - }) - .exceptionally(e -> { - STATE_UPDATER.set(this, OperationState.None); - log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); - return null; - }); - - } @Override public CompletableFuture takesFirstSnapshot() { - if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.UpdatingIndex)) { + CompletableFuture completableFuture = new CompletableFuture<>(); + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> { TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); return snapshotIndexWriterFuture .thenCompose((indexesWriter) -> { @@ -358,13 +223,15 @@ public CompletableFuture takesFirstSnapshot() { indexes.setIndexList(new ArrayList<>()); indexes.setTopicName(this.topic.getName()); this.lastSnapshotTimestamps = System.currentTimeMillis(); + completableFuture.complete(null); }) .exceptionally(e -> { log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); + completableFuture.completeExceptionally(e); return null; }); - } - return CompletableFuture.completedFuture(null); + }); + return completableFuture; } @@ -407,7 +274,7 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe .getSnapshot().getMaxReadPositionLedgerId(), persistentSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); if (indexes.size() != 0) { - sequenceID.set(indexes.lastEntry().getValue().sequenceID + 1); + persistentWorker.sequenceID.set(indexes.lastEntry().getValue().sequenceID + 1); } } //Read snapshot segment to recover aborts. @@ -464,9 +331,9 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob //Wait the processor recover completely and the allow TB to recover the messages // after the startReadCursorPosition. FutureUtil.waitForAll(completableFutures).get(); - if (invalidIndex.get() != 0 && STATE_UPDATER.compareAndSet(this, - OperationState.None, OperationState.UpdatingIndex)) { - updateSnapshotIndex(this.maxReadPosition, this.unsealedAbortedTxnIdSegment); + if (invalidIndex.get() != 0 ) { + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () + -> persistentWorker.updateSnapshotIndex(null)); } return CompletableFuture.completedFuture(startReadCursorPosition); } catch (Exception ex) { @@ -483,52 +350,35 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob @Override public CompletableFuture clearSnapshot() { - ArrayList> completableFutures = new ArrayList<>(); - //TODO: Wait all operation completely and then clear the snapshot - STATE_UPDATER.set(this, OperationState.Closing); - while (!abortTxnSegments.isEmpty()) { - if (log.isDebugEnabled()) { - log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", - topic.getName(), abortTxnSegments.firstKey()); + CompletableFuture completableFuture = new CompletableFuture<>(); + persistentWorker.appendTask(PersistentWorker.OperationType.Close, () -> { + ArrayList> completableFutures = new ArrayList<>(); + //Delete all segment + while (!abortTxnSegments.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", + topic.getName(), abortTxnSegments.firstKey()); + } + PositionImpl positionNeedToDelete = abortTxnSegments.firstKey(); + completableFutures.add(persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); } - PositionImpl positionNeedToDelete = abortTxnSegments.firstKey(); - completableFutures.add(deleteSnapshotSegment(positionNeedToDelete)); - } - return FutureUtil.waitForAll(completableFutures) - .thenCompose((ignore) -> snapshotIndexWriterFuture - .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) - .thenRun(() -> { - log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", - topic.getName()); - STATE_UPDATER.compareAndSet(this, OperationState.Closing, OperationState.Closed); - }) - .exceptionally(e -> { - log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", - topic.getName(), e); - return null; - }); - } - - private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedToDelete) { - - long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); - return snapshotSegmentsWriterFuture.thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) - .thenRun(() -> { - if (log.isDebugEnabled()) { - log.debug("[{}] Successes to delete the snapshot segment, " - + "whose sequenceId is [{}] and maxReadPosition is [{}]", - this.topic.getName(), this.sequenceID, this.maxReadPosition); - } - abortTxnSegments.remove(positionNeedToDelete); - //The process will check whether the snapshot segment is null, and update index when recovered. - indexes.remove(positionNeedToDelete); - updateSnapshotIndex(this.maxReadPosition, this.unsealedAbortedTxnIdSegment); - }).exceptionally(e -> { - log.warn("[{}] Failed to delete the snapshot segment, " - + "whose sequenceId is [{}] and maxReadPosition is [{}]", - this.topic.getName(), this.sequenceID, this.maxReadPosition, e); - return null; - }); + //Delete index + return FutureUtil.waitForAll(completableFutures) + .thenCompose((ignore) -> snapshotIndexWriterFuture + .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) + .thenRun(() -> { + log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", + topic.getName()); + completableFuture.thenCompose(null); + }) + .exceptionally(e -> { + log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", + topic.getName(), e); + completableFuture.completeExceptionally(e); + return null; + }); + }); + return completableFuture; } @@ -560,4 +410,242 @@ private void closeReader(SystemTopicClient.Reader reader) { return null; }); } + + private class PersistentWorker { + protected final AtomicLong sequenceID = new AtomicLong(0); + + private final PersistentTopic topic; + + private enum OperationState { + None, + UpdatingIndex, + WritingSegment, + DeletingSegment, + Closing, + Closed + } + private static final AtomicReferenceFieldUpdater + STATE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(PersistentWorker.class, + PersistentWorker.OperationState.class, "operationState"); + + public enum OperationType { + UpdateIndex, + WriteSegment, + DeleteSegment, + Close + } + + private volatile OperationState operationState = OperationState.None; + + ConcurrentSkipListMap>> taskQueue = + new ConcurrentSkipListMap<>(); + private CompletableFuture lastOperationFuture; + private final Timer timer; + + public PersistentWorker(PersistentTopic topic) { + this.topic = topic; + this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); + + } + + public void appendTask(OperationType operationType, Supplier> task) { + switch (operationType) { + case UpdateIndex -> { + if (!taskQueue.isEmpty()) { + return; + } else if(STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.UpdatingIndex)) { + lastOperationFuture = task.get(); + lastOperationFuture.whenComplete((ignore, throwable) -> { + if (throwable != null && log.isDebugEnabled()) { + log.debug("[{}] Failed to update index snapshot", topic.getName(), throwable); + } + + STATE_UPDATER.compareAndSet(this, OperationState.UpdatingIndex, OperationState.None); + }); + } + } + case WriteSegment, DeleteSegment -> { + taskQueue.put(operationType, task); + executeTask(); + } + case Close -> { + STATE_UPDATER.set(this, OperationState.Closing); + taskQueue.clear(); + lastOperationFuture.thenRun(() -> { + lastOperationFuture = task.get(); + lastOperationFuture.thenRun(() -> + STATE_UPDATER.compareAndSet(this, OperationState.Closing, OperationState.Closed)); + }); + } + } + } + + private void executeTask() { + OperationType operationType = taskQueue.firstKey(); + switch (operationType) { + case WriteSegment -> { + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.WritingSegment)) { + if (taskQueue.firstKey() == OperationType.WriteSegment) { + lastOperationFuture = taskQueue.firstEntry().getValue().get(); + lastOperationFuture.whenComplete((ignore, throwable) -> { + if (throwable != null) { + if (log.isDebugEnabled()) { + log.debug("[{}] Failed to write snapshot segment", topic.getName(), throwable); + } + timer.newTimeout(timeout -> executeTask(), + takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); + } else { + taskQueue.remove(taskQueue.firstKey()); + } + STATE_UPDATER.compareAndSet(this, OperationState.WritingSegment, OperationState.None); + }); + } + } + } + case DeleteSegment -> { + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.DeletingSegment)) { + if (taskQueue.firstKey() == OperationType.DeleteSegment) { + lastOperationFuture = taskQueue.firstEntry().getValue().get(); + lastOperationFuture.whenComplete((ignore, throwable) -> { + if (throwable != null) { + if (log.isDebugEnabled()) { + log.debug("[{}] Failed to delete snapshot segment", topic.getName(), throwable); + } + timer.newTimeout(timeout -> executeTask(), + takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); + } else { + taskQueue.remove(taskQueue.firstKey()); + } + + STATE_UPDATER.compareAndSet(this, OperationState.DeletingSegment, OperationState.None); + }); + } + } + } + } + } + + protected CompletableFuture takeSnapshotSegmentAsync(ArrayList sealedAbortedTxnIdSegment, + PositionImpl maxReadPosition) { + return writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, maxReadPosition).thenRun(() -> { + if (log.isDebugEnabled()) { + log.debug("Successes to take snapshot segment [{}] at maxReadPosition [{}] " + + "for the topic [{}], and the size of the segment is [{}]", + this.sequenceID, maxReadPosition, topic.getName(), sealedAbortedTxnIdSegment.size()); + } + this.sequenceID.getAndIncrement(); + }).exceptionally(e -> { + //Just log the error, and the processor will try to take snapshot again when the transactionBuffer + //append aborted txn nex time. + log.error("Failed to take snapshot segment [{}] at maxReadPosition [{}] " + + "for the topic [{}], and the size of the segment is [{}]", + this.sequenceID, maxReadPosition, topic.getName(), sealedAbortedTxnIdSegment.size(), e); + return null; + }); + } + + private CompletableFuture writeSnapshotSegmentAsync(List segment, PositionImpl maxReadPosition) { + TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = new TransactionBufferSnapshotSegment(); + transactionBufferSnapshotSegment.setAborts(segment); + transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); + transactionBufferSnapshotSegment.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); + transactionBufferSnapshotSegment.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); + + return snapshotSegmentsWriterFuture.thenCompose(segmentWriter -> { + transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get()); + return segmentWriter.writeAsync(buildKey(this.sequenceID.get()), transactionBufferSnapshotSegment); + }).thenCompose((messageId) -> { + //Build index for this segment + TransactionBufferSnapshotIndex index = new TransactionBufferSnapshotIndex(); + index.setSequenceID(transactionBufferSnapshotSegment.getSequenceId()); + index.setMaxReadPositionLedgerID(maxReadPosition.getLedgerId()); + index.setMaxReadPositionEntryID(maxReadPosition.getEntryId()); + index.setPersistentPositionLedgerID(((MessageIdImpl) messageId).getLedgerId()); + index.setPersistentPositionEntryID(((MessageIdImpl) messageId).getEntryId()); + + indexes.put(maxReadPosition, index); + //update snapshot segment index. + return updateSnapshotIndex(maxReadPosition); + }); + } + + private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedToDelete) { + long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); + return snapshotSegmentsWriterFuture.thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) + .thenRun(() -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Successes to delete the snapshot segment, " + + "whose sequenceId is [{}] and maxReadPosition is [{}]", + this.topic.getName(), this.sequenceID, positionNeedToDelete); + } + abortTxnSegments.remove(positionNeedToDelete); + //The process will check whether the snapshot segment is null, and update index when recovered. + indexes.remove(positionNeedToDelete); + updateSnapshotIndex(null); + }).exceptionally(e -> { + log.warn("[{}] Failed to delete the snapshot segment, " + + "whose sequenceId is [{}] and maxReadPosition is [{}]", + this.topic.getName(), this.sequenceID, positionNeedToDelete, e); + return null; + }); + } + + //Update the indexes in the transactionBufferSnapshotIndexe. + //Concurrency control is performed by snapshotIndexWriterFuture. + private CompletableFuture updateSnapshotIndex(PositionImpl maxReadPosition) { + TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); + return snapshotIndexWriterFuture + .thenCompose((indexesWriter) -> { + snapshotIndexes.setIndexList(indexes.values().stream().toList()); + //update the metadata in the indexes. + if (maxReadPosition != null) { + snapshotIndexes.setSnapshot(new TransactionBufferSnapshotIndexesMetadata( + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), new ArrayList<>())); + } else { + //metadata keep no change + snapshotIndexes.setSnapshot(persistentSnapshotIndexes.getSnapshot()); + } + return indexesWriter.writeAsync(topic.getName(), snapshotIndexes); + }) + .thenRun(() -> { + persistentSnapshotIndexes.setIndexList(snapshotIndexes.getIndexList()); + if (maxReadPosition != null) { + persistentSnapshotIndexes.setSnapshot(new TransactionBufferSnapshotIndexesMetadata( + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), new ArrayList<>())); + } + lastSnapshotTimestamps = System.currentTimeMillis(); + }) + .exceptionally(e -> { + log.error("[{}] Failed to update snapshot segment index", snapshotIndexes.getTopicName(), e); + return null; + }); + } + + //Only update the metadata in the transactionBufferSnapshotIndexes. + //Concurrency control is performed by snapshotIndexWriterFuture. + private CompletableFuture updateIndexMetadataForTheLastSnapshot() { + TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); + return snapshotIndexWriterFuture + .thenCompose((indexesWriter) -> { + //Store the latest metadata + TransactionBufferSnapshotIndexesMetadata transactionBufferSnapshotSegment = + new TransactionBufferSnapshotIndexesMetadata(); + transactionBufferSnapshotSegment.setAborts(unsealedAbortedTxnIdSegment); + indexes.setSnapshot(transactionBufferSnapshotSegment); + //Only update the metadata in indexes and keep the index in indexes unchanged. + indexes.setIndexList(persistentSnapshotIndexes.getIndexList()); + return indexesWriter.writeAsync(topic.getName(), indexes); + }) + .thenRun(() -> { + persistentSnapshotIndexes.setSnapshot(indexes.getSnapshot()); + lastSnapshotTimestamps = System.currentTimeMillis(); + }) + .exceptionally(e -> { + log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); + return null; + }); + } + + } + } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index f36a936c25ad3..b71e6bf3384fa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -361,9 +361,9 @@ public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { + PositionImpl maxReadPosition = updateMaxReadPosition(txnID); snapshotAbortedTxnProcessor.appendAbortedTxn(new TxnIDData(txnID.getMostSigBits(), - txnID.getLeastSigBits()), (PositionImpl) position); - updateMaxReadPosition(txnID); + txnID.getLeastSigBits()), maxReadPosition); snapshotAbortedTxnProcessor.trimExpiredTxnIDDataOrSnapshotSegments(); } txnAbortedCounter.increment(); @@ -429,7 +429,7 @@ private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { } } - void updateMaxReadPosition(TxnID txnID) { + PositionImpl updateMaxReadPosition(TxnID txnID) { ongoingTxns.remove(txnID); PositionImpl maxReadPosition; if (!ongoingTxns.isEmpty()) { @@ -440,6 +440,7 @@ void updateMaxReadPosition(TxnID txnID) { maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); } snapshotAbortedTxnProcessor.updateMaxReadPosition(maxReadPosition); + return maxReadPosition; } @Override From 98ad078937238d6a80ad0e1d8b9c64e7d2e09964 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Thu, 27 Oct 2022 23:40:27 +0800 Subject: [PATCH 12/32] make updateSnapshotIndex have a fixed index snapshot --- ...napshotSegmentAbortedTxnProcessorImpl.java | 37 ++++++++----------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 584d8ccd30ba2..f5b14cb115c76 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -218,10 +218,9 @@ public CompletableFuture takesFirstSnapshot() { return indexesWriter.writeAsync(topic.getName(), indexes); }) .thenRun(() -> { - //TODO: check again persistentSnapshotIndexes.setSnapshot(indexes.getSnapshot()); - indexes.setIndexList(new ArrayList<>()); - indexes.setTopicName(this.topic.getName()); + persistentSnapshotIndexes.setIndexList(indexes.getIndexList()); + persistentSnapshotIndexes.setTopicName(this.topic.getName()); this.lastSnapshotTimestamps = System.currentTimeMillis(); completableFuture.complete(null); }) @@ -333,7 +332,8 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob FutureUtil.waitForAll(completableFutures).get(); if (invalidIndex.get() != 0 ) { persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () - -> persistentWorker.updateSnapshotIndex(null)); + -> persistentWorker.updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), + indexes.values().stream().toList())); } return CompletableFuture.completedFuture(startReadCursorPosition); } catch (Exception ex) { @@ -565,7 +565,9 @@ private CompletableFuture writeSnapshotSegmentAsync(List segmen indexes.put(maxReadPosition, index); //update snapshot segment index. - return updateSnapshotIndex(maxReadPosition); + return updateSnapshotIndex(new TransactionBufferSnapshotIndexesMetadata( + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), new ArrayList<>()), + indexes.values().stream().toList()); }); } @@ -581,7 +583,9 @@ private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedT abortTxnSegments.remove(positionNeedToDelete); //The process will check whether the snapshot segment is null, and update index when recovered. indexes.remove(positionNeedToDelete); - updateSnapshotIndex(null); + //Keep index snapshot and update index + updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), + indexes.values().stream().toList()); }).exceptionally(e -> { log.warn("[{}] Failed to delete the snapshot segment, " + "whose sequenceId is [{}] and maxReadPosition is [{}]", @@ -592,27 +596,18 @@ private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedT //Update the indexes in the transactionBufferSnapshotIndexe. //Concurrency control is performed by snapshotIndexWriterFuture. - private CompletableFuture updateSnapshotIndex(PositionImpl maxReadPosition) { + private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotIndexesMetadata snapshotSegment, + List indexList) { TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); return snapshotIndexWriterFuture .thenCompose((indexesWriter) -> { - snapshotIndexes.setIndexList(indexes.values().stream().toList()); - //update the metadata in the indexes. - if (maxReadPosition != null) { - snapshotIndexes.setSnapshot(new TransactionBufferSnapshotIndexesMetadata( - maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), new ArrayList<>())); - } else { - //metadata keep no change - snapshotIndexes.setSnapshot(persistentSnapshotIndexes.getSnapshot()); - } + snapshotIndexes.setIndexList(indexList); + snapshotIndexes.setSnapshot(snapshotSegment); return indexesWriter.writeAsync(topic.getName(), snapshotIndexes); }) .thenRun(() -> { - persistentSnapshotIndexes.setIndexList(snapshotIndexes.getIndexList()); - if (maxReadPosition != null) { - persistentSnapshotIndexes.setSnapshot(new TransactionBufferSnapshotIndexesMetadata( - maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), new ArrayList<>())); - } + persistentSnapshotIndexes.setIndexList(indexList); + persistentSnapshotIndexes.setSnapshot(snapshotSegment); lastSnapshotTimestamps = System.currentTimeMillis(); }) .exceptionally(e -> { From 9904eb82b79ed828e5aa7423013a1d412ed717b1 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Fri, 28 Oct 2022 12:03:24 +0800 Subject: [PATCH 13/32] add test and fix bug for check aborted transactiion id --- ...napshotSegmentAbortedTxnProcessorImpl.java | 23 +++-- .../TopicTransactionBufferRecoverTest.java | 84 +++++++++++++++++-- 2 files changed, 92 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index f5b14cb115c76..b5ff6db9a75e3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -117,14 +117,14 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { } @Override - public void appendAbortedTxn(TxnIDData abortedTxnId, PositionImpl position) { + public void appendAbortedTxn(TxnIDData abortedTxnId, PositionImpl maxReadPosition) { unsealedAbortedTxnIdSegment.add(abortedTxnId); //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { changeMaxReadPositionAndAddAbortTimes.set(0); - abortTxnSegments.put(position, unsealedAbortedTxnIdSegment); + abortTxnSegments.put(maxReadPosition, unsealedAbortedTxnIdSegment); persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> - persistentWorker.takeSnapshotSegmentAsync(unsealedAbortedTxnIdSegment, position)); + persistentWorker.takeSnapshotSegmentAsync(unsealedAbortedTxnIdSegment, maxReadPosition)); unsealedAbortedTxnIdSegment = new ArrayList<>(); } } @@ -147,13 +147,18 @@ public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { if (readPosition == null) { return abortTxnSegments.values().stream() .anyMatch(list -> list.contains(txnID)) || unsealedAbortedTxnIdSegment.contains(txnID); - } - Map.Entry> ceilingEntry = abortTxnSegments - .ceilingEntry((PositionImpl) readPosition); - if (ceilingEntry == null) { - return unsealedAbortedTxnIdSegment.contains(txnID); } else { - return ceilingEntry.getValue().contains(txnID); + PositionImpl maxReadPosition = abortTxnSegments.ceilingKey((PositionImpl) readPosition); + if (maxReadPosition != null) { + return abortTxnSegments.keySet().stream() + .filter((position) -> position.compareTo(maxReadPosition) <= 0) + .anyMatch((position -> abortTxnSegments.get(position).contains(txnID))); + } else { + return abortTxnSegments.keySet().stream() + .filter((position) -> position.compareTo((PositionImpl) readPosition) <= 0) + .anyMatch((position -> abortTxnSegments.get(position).contains(txnID))) + || unsealedAbortedTxnIdSegment.contains(txnID); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 82667f533eda4..1e317d53877e3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -30,17 +30,13 @@ import static org.testng.Assert.assertTrue; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; -import java.io.IOException; import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.NavigableMap; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; @@ -64,7 +60,6 @@ import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.impl.SingleSnapshotAbortedTxnProcessorImpl; -import org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex; @@ -80,6 +75,7 @@ import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderBuilder; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -406,7 +402,7 @@ private void testTopicTransactionBufferDeleteAbort(Boolean enableSnapshotSegment AbortedTxnProcessor abortedTxnProcessor = (AbortedTxnProcessor) field.get(topicTransactionBuffer); if (enableSnapshotSegment) { - //TODO: + //TODO exist = true; } else { Field abortsField = SingleSnapshotAbortedTxnProcessorImpl.class.getDeclaredField("aborts"); @@ -746,4 +742,80 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob assertEquals(snapshot.getAborts().get(0), new TxnIDData(1, 1)); } + @Test + public void testSnapshotSegment() throws Exception { + String topic = NAMESPACE1 + "/testSnapshotSegment"; + String subName = "testSnapshotSegment"; + + LinkedMap ongoingTxns = new LinkedMap<>(); + LinkedList abortedTxns = new LinkedList<>(); + + this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSnapshotSegmentSize(10); + this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSnapshotMaxTransactionCount(3); + + Producer producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(false) + .create(); + + Consumer consumer = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Exclusive) + .subscribe(); + + for (int i = 0; i < 10; i++) { + int maxReadMessage = 19; + int abortedTxnSize = 0; + for (int j = 0; j < 20; j++) { + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MINUTES).build().get(); + //half common message and half transaction message. + //the transaction message have a half which are aborted. + if (RandomUtils.nextInt() % 2 == 0) { + MessageId messageId = producer.newMessage(transaction).value(i * 10 + j).send(); + if (RandomUtils.nextInt() % 2 == 0) { + transaction.abort().get(); + abortedTxns.add(messageId); + abortedTxnSize++; + } else { + ongoingTxns.put(transaction, messageId); + if (maxReadMessage == 19) { + //The except number of the messages that can be read + maxReadMessage = j - abortedTxnSize; + } + } + } else { + MessageId messageId = producer.newMessage().value(i * 10 + j).send(); + transaction.commit().get(); + } + } + for (int k = 0; k < maxReadMessage; k++) { + Message message = consumer.receive(2, TimeUnit.SECONDS); + assertNotNull(message); + assertFalse(abortedTxns.contains(message.getMessageId())); + } + Message message = consumer.receive(2, TimeUnit.SECONDS); + assertNull(message); + + for (Transaction ongoingTxn: ongoingTxns.keySet()) { + ongoingTxn.commit().get(); + } + ongoingTxns.clear(); + for (int k = maxReadMessage; k < 20 - abortedTxnSize; k++) { + message = consumer.receive(2, TimeUnit.SECONDS); + assertNotNull(message); + assertFalse(abortedTxns.contains(message.getMessageId())); + } + } + + admin.topics().unload(topic); + + for (int i = 0; i < 200 - abortedTxns.size(); i++) { + Message message = consumer.receive(2, TimeUnit.SECONDS); + assertNotNull(message); + assertFalse(abortedTxns.contains(message.getMessageId())); + } + } } From 8c53f1a8eb29ce033f78f7de2061e7c0a0eaf435 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Fri, 28 Oct 2022 12:22:30 +0800 Subject: [PATCH 14/32] fix some comments --- .../buffer/AbortedTxnProcessor.java | 7 +++--- ...SingleSnapshotAbortedTxnProcessorImpl.java | 12 +++++----- ...napshotSegmentAbortedTxnProcessorImpl.java | 22 +++++++++---------- .../buffer/impl/TopicTransactionBuffer.java | 14 ++++++------ 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index 8bf62af172ed2..bb822ea7da3aa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -22,7 +22,6 @@ import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferRecoverCallBack; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; @@ -54,7 +53,7 @@ public interface AbortedTxnProcessor extends TimerTask { * In the old implementation we clear the invalid aborted txn ID one by one. * In the new implementation, we adopt snapshot segments. And then we clear invalid segment by its max read position. */ - void trimExpiredTxnIDDataOrSnapshotSegments(); + void trimExpiredAbortedTxns(); /** * Check whether the transaction ID is an aborted transaction ID. @@ -69,13 +68,13 @@ public interface AbortedTxnProcessor extends TimerTask { * @return a pair consists of a Boolean if the transaction buffer needs to recover and a Position (startReadCursorPosition) determiner where to start to recover in the original topic. */ - CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack); + CompletableFuture recoverFromSnapshot(); /** * Clear the snapshot/snapshot segment and index for this topic. * @return a completableFuture. */ - CompletableFuture clearSnapshot(); + CompletableFuture clearAndCloseAsync(); /** * Take the frist snapshot if the topic has no snapshot before. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 9cbb98b6f931e..8a82af847a90d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -38,6 +38,7 @@ import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.util.FutureUtil; @Slf4j public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcessor { @@ -96,7 +97,7 @@ public void updateMaxReadPositionNotIncreaseChangeTimes(Position maxReadPosition } @Override - public void trimExpiredTxnIDDataOrSnapshotSegments() { + public void trimExpiredAbortedTxns() { while (!aborts.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) .ledgerExists(aborts.get(aborts.firstKey()).getLedgerId())) { if (log.isDebugEnabled()) { @@ -115,7 +116,7 @@ public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { @Override - public CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack) { + public CompletableFuture recoverFromSnapshot() { return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotService() .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { @@ -137,16 +138,14 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe } closeReader(reader); if (!hasSnapshot) { - callBack.noNeedToRecover(); return CompletableFuture.completedFuture(null); } return CompletableFuture.completedFuture(startReadCursorPosition); } catch (Exception ex) { log.error("[{}] Transaction buffer recover fail when read " + "transactionBufferSnapshot!", topic.getName(), ex); - callBack.recoverExceptionally(ex); closeReader(reader); - return null; + return FutureUtil.failedFuture(ex); } }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() @@ -154,7 +153,8 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe } @Override - public CompletableFuture clearSnapshot() { + public CompletableFuture clearAndCloseAsync() { + timer.stop(); return this.takeSnapshotWriter.thenCompose(writer -> { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); snapshot.setTopicName(topic.getName()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index b5ff6db9a75e3..0f6258ee2e618 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -163,7 +163,7 @@ public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { } @Override - public void trimExpiredTxnIDDataOrSnapshotSegments() { + public void trimExpiredAbortedTxns() { //Checking whether there are some segment expired. while (!abortTxnSegments.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) .ledgerExists(abortTxnSegments.firstKey().getLedgerId())) { @@ -240,7 +240,7 @@ public CompletableFuture takesFirstSnapshot() { @Override - public CompletableFuture recoverFromSnapshot(TopicTransactionBufferRecoverCallBack callBack) { + public CompletableFuture recoverFromSnapshot() { return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotIndexService() .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { @@ -263,7 +263,6 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe } closeReader(reader); if (!hasIndex) { - callBack.noNeedToRecover(); return CompletableFuture.completedFuture(null); } else { persistentSnapshotIndexes.getIndexList() @@ -289,8 +288,9 @@ public CompletableFuture recoverFromSnapshot(TopicTransactionBuffe @Override public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { persistentSnapshotIndexes.getIndexList().forEach(index -> { - CompletableFuture completableFuture1 = new CompletableFuture<>(); - completableFutures.add(completableFuture1); + //TODO: read on demand + CompletableFuture handleSegmentFuture = new CompletableFuture<>(); + completableFutures.add(handleSegmentFuture); readOnlyManagedLedger.asyncReadEntry( new PositionImpl(index.getPersistentPositionLedgerID(), index.getPersistentPositionEntryID()), @@ -302,17 +302,17 @@ public void readEntryComplete(Entry entry, Object ctx) { indexes.remove(new PositionImpl( index.getMaxReadPositionLedgerID(), index.getMaxReadPositionEntryID())); - completableFuture1.complete(null); + handleSegmentFuture.complete(null); invalidIndex.getAndIncrement(); return; } handleSnapshotSegmentEntry(entry); - completableFuture1.complete(null); + handleSegmentFuture.complete(null); } @Override public void readEntryFailed(ManagedLedgerException exception, Object ctx) { - completableFuture1.completeExceptionally(exception); + handleSegmentFuture.completeExceptionally(exception); } }, null); }); @@ -344,9 +344,8 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob } catch (Exception ex) { log.error("[{}] Transaction buffer recover fail when read " + "transactionBufferSnapshot!", topic.getName(), ex); - callBack.recoverExceptionally(ex); closeReader(reader); - return null; + return FutureUtil.failedFuture(ex); } }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() @@ -354,7 +353,8 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob } @Override - public CompletableFuture clearSnapshot() { + public CompletableFuture clearAndCloseAsync() { + timer.stop(); CompletableFuture completableFuture = new CompletableFuture<>(); persistentWorker.appendTask(PersistentWorker.OperationType.Close, () -> { ArrayList> completableFutures = new ArrayList<>(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index b71e6bf3384fa..52ee6f93ec0b8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -121,7 +121,7 @@ private void recover() { @Override public void recoverComplete() { synchronized (TopicTransactionBuffer.this) { - // sync maxReadPosition change to LAC when TopicTransaction buffer have not recover + // sync maxReadPosition change to LAC when TopicTransaction buffer have not recovered // completely the normal message have been sent to broker and state is // not Ready can't sync maxReadPosition when no ongoing transactions if (ongoingTxns.isEmpty()) { @@ -202,8 +202,7 @@ public void recoverExceptionally(Throwable e) { recoverTime.setRecoverEndTime(System.currentTimeMillis()); topic.close(true); } - }, this.topic, - this, takeSnapshotWriter, snapshotAbortedTxnProcessor)); + }, this.topic, this, takeSnapshotWriter, snapshotAbortedTxnProcessor)); } @Override @@ -318,7 +317,7 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { updateMaxReadPosition(txnID); handleLowWaterMark(txnID, lowWaterMark); - snapshotAbortedTxnProcessor.trimExpiredTxnIDDataOrSnapshotSegments(); + snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); } txnCommittedCounter.increment(); completableFuture.complete(null); @@ -364,7 +363,7 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { PositionImpl maxReadPosition = updateMaxReadPosition(txnID); snapshotAbortedTxnProcessor.appendAbortedTxn(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits()), maxReadPosition); - snapshotAbortedTxnProcessor.trimExpiredTxnIDDataOrSnapshotSegments(); + snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); } txnAbortedCounter.increment(); completableFuture.complete(null); @@ -450,7 +449,7 @@ public CompletableFuture purgeTxns(List dataLedgers) { @Override public CompletableFuture clearSnapshot() { - return snapshotAbortedTxnProcessor.clearSnapshot(); + return snapshotAbortedTxnProcessor.clearAndCloseAsync(); } @@ -561,9 +560,10 @@ public void run() { this, topic.getName()); return; } - abortedTxnProcessor.recoverFromSnapshot(callBack).thenAcceptAsync(startReadCursorPosition -> { + abortedTxnProcessor.recoverFromSnapshot().thenAcceptAsync(startReadCursorPosition -> { //Transaction is not enable for this topic, so just make maxReadPosition as LAC. if (startReadCursorPosition == null) { + callBack.noNeedToRecover(); return; } else { this.startReadCursorPosition = startReadCursorPosition; From b951329b1b1f5c8d73662e590612abca2283dee6 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Fri, 28 Oct 2022 14:32:48 +0800 Subject: [PATCH 15/32] optimization --- .../buffer/AbortedTxnProcessor.java | 2 + ...SingleSnapshotAbortedTxnProcessorImpl.java | 5 + ...napshotSegmentAbortedTxnProcessorImpl.java | 142 ++++++++++-------- .../buffer/impl/TopicTransactionBuffer.java | 124 +++++++-------- 4 files changed, 134 insertions(+), 139 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index bb822ea7da3aa..61199c0ce82df 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -94,4 +94,6 @@ public interface AbortedTxnProcessor extends TimerTask { */ long getLastSnapshotTimestamps(); + CompletableFuture closeAsync(); + } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 8a82af847a90d..a3692159dc106 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -177,6 +177,11 @@ public long getLastSnapshotTimestamps() { return this.lastSnapshotTimestamps; } + @Override + public CompletableFuture closeAsync() { + return takeSnapshotWriter.thenCompose(SystemTopicClient.Writer::closeAsync); + } + private void closeReader(SystemTopicClient.Reader reader) { reader.closeAsync().exceptionally(e -> { log.error("[{}]Transaction buffer reader close error!", topic.getName(), e); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 0f6258ee2e618..e976a39f11ef4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -245,8 +245,8 @@ public CompletableFuture recoverFromSnapshot() { .getTxnBufferSnapshotIndexService() .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { PositionImpl startReadCursorPosition = null; + boolean hasIndex = false; try { - boolean hasIndex = false; //Read Index to recover the sequenceID, indexes, lastAbortedTxns and maxReadPosition. while (reader.hasMoreEvents()) { Message message = reader.readNext(); @@ -261,62 +261,69 @@ public CompletableFuture recoverFromSnapshot() { } } } + } catch (Exception ex) { + log.error("[{}] Transaction buffer recover fail when read " + + "transactionBufferSnapshot!", topic.getName(), ex); closeReader(reader); - if (!hasIndex) { - return CompletableFuture.completedFuture(null); - } else { - persistentSnapshotIndexes.getIndexList() - .forEach(transactionBufferSnapshotIndex -> - indexes.put(new PositionImpl( - transactionBufferSnapshotIndex.persistentPositionLedgerID, - transactionBufferSnapshotIndex.persistentPositionEntryID), - transactionBufferSnapshotIndex)); - this.unsealedAbortedTxnIdSegment = (ArrayList) persistentSnapshotIndexes - .getSnapshot().getAborts(); - this.maxReadPosition = new PositionImpl(persistentSnapshotIndexes - .getSnapshot().getMaxReadPositionLedgerId(), - persistentSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); - if (indexes.size() != 0) { - persistentWorker.sequenceID.set(indexes.lastEntry().getValue().sequenceID + 1); - } + return FutureUtil.failedFuture(ex); + } + closeReader(reader); + PositionImpl finalStartReadCursorPosition = startReadCursorPosition; + if (!hasIndex) { + return CompletableFuture.completedFuture(null); + } else { + persistentSnapshotIndexes.getIndexList() + .forEach(transactionBufferSnapshotIndex -> + indexes.put(new PositionImpl( + transactionBufferSnapshotIndex.persistentPositionLedgerID, + transactionBufferSnapshotIndex.persistentPositionEntryID), + transactionBufferSnapshotIndex)); + this.unsealedAbortedTxnIdSegment = (ArrayList) persistentSnapshotIndexes + .getSnapshot().getAborts(); + this.maxReadPosition = new PositionImpl(persistentSnapshotIndexes + .getSnapshot().getMaxReadPositionLedgerId(), + persistentSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); + if (indexes.size() != 0) { + persistentWorker.sequenceID.set(indexes.lastEntry().getValue().sequenceID + 1); } - //Read snapshot segment to recover aborts. - ArrayList> completableFutures = new ArrayList<>(); - AtomicLong invalidIndex = new AtomicLong(0); - AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks - .OpenReadOnlyManagedLedgerCallback() { - @Override - public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { - persistentSnapshotIndexes.getIndexList().forEach(index -> { - //TODO: read on demand - CompletableFuture handleSegmentFuture = new CompletableFuture<>(); - completableFutures.add(handleSegmentFuture); - readOnlyManagedLedger.asyncReadEntry( - new PositionImpl(index.getPersistentPositionLedgerID(), - index.getPersistentPositionEntryID()), - new AsyncCallbacks.ReadEntryCallback() { - @Override - public void readEntryComplete(Entry entry, Object ctx) { - //Remove invalid index - if (entry == null) { - indexes.remove(new PositionImpl( - index.getMaxReadPositionLedgerID(), - index.getMaxReadPositionEntryID())); - handleSegmentFuture.complete(null); - invalidIndex.getAndIncrement(); - return; - } - handleSnapshotSegmentEntry(entry); + } + //Read snapshot segment to recover aborts. + ArrayList> completableFutures = new ArrayList<>(); + AtomicLong invalidIndex = new AtomicLong(0); + AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks + .OpenReadOnlyManagedLedgerCallback() { + @Override + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { + persistentSnapshotIndexes.getIndexList().forEach(index -> { + //TODO: read on demand + CompletableFuture handleSegmentFuture = new CompletableFuture<>(); + completableFutures.add(handleSegmentFuture); + readOnlyManagedLedger.asyncReadEntry( + new PositionImpl(index.getPersistentPositionLedgerID(), + index.getPersistentPositionEntryID()), + new AsyncCallbacks.ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + //Remove invalid index + if (entry == null) { + indexes.remove(new PositionImpl( + index.getMaxReadPositionLedgerID(), + index.getMaxReadPositionEntryID())); handleSegmentFuture.complete(null); + invalidIndex.getAndIncrement(); + return; } - - @Override - public void readEntryFailed(ManagedLedgerException exception, Object ctx) { - handleSegmentFuture.completeExceptionally(exception); - } - }, null); - }); - } + handleSnapshotSegmentEntry(entry); + handleSegmentFuture.complete(null); + } + + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + handleSegmentFuture.completeExceptionally(exception); + } + }, null); + }); + } @Override public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx) { @@ -334,19 +341,14 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob null); //Wait the processor recover completely and the allow TB to recover the messages // after the startReadCursorPosition. - FutureUtil.waitForAll(completableFutures).get(); - if (invalidIndex.get() != 0 ) { - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () - -> persistentWorker.updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), - indexes.values().stream().toList())); - } - return CompletableFuture.completedFuture(startReadCursorPosition); - } catch (Exception ex) { - log.error("[{}] Transaction buffer recover fail when read " - + "transactionBufferSnapshot!", topic.getName(), ex); - closeReader(reader); - return FutureUtil.failedFuture(ex); - } + return FutureUtil.waitForAll(completableFutures).thenCompose((ignore) -> { + if (invalidIndex.get() != 0 ) { + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () + -> persistentWorker.updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), + indexes.values().stream().toList())); + } + return CompletableFuture.completedFuture(finalStartReadCursorPosition); + }); }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() .getExecutor(this)); @@ -397,6 +399,14 @@ public long getLastSnapshotTimestamps() { return this.lastSnapshotTimestamps; } + @Override + public CompletableFuture closeAsync() { + ArrayList> completableFutures = new ArrayList<>(); + completableFutures.add(this.snapshotIndexWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync)); + completableFutures.add(this.snapshotSegmentsWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync)); + return FutureUtil.waitForAll(completableFutures); + } + private void handleSnapshotSegmentEntry(Entry entry) { //decode snapshot from entry ByteBuf headersAndPayload = entry.getDataBuffer(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 52ee6f93ec0b8..c751f8967b704 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -73,8 +73,6 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen */ private final LinkedMap ongoingTxns = new LinkedMap<>(); - private final CompletableFuture> takeSnapshotWriter; - private final LongAdder txnCommittedCounter = new LongAdder(); private final LongAdder txnAbortedCounter = new LongAdder(); @@ -99,9 +97,6 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen public TopicTransactionBuffer(PersistentTopic topic) { super(State.None); this.topic = topic; - this.takeSnapshotWriter = this.topic.getBrokerService().getPulsar() - .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName())); this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); @@ -121,12 +116,10 @@ private void recover() { @Override public void recoverComplete() { synchronized (TopicTransactionBuffer.this) { - // sync maxReadPosition change to LAC when TopicTransaction buffer have not recovered - // completely the normal message have been sent to broker and state is - // not Ready can't sync maxReadPosition when no ongoing transactions if (ongoingTxns.isEmpty()) { snapshotAbortedTxnProcessor - .updateMaxReadPosition(topic.getManagedLedger().getLastConfirmedEntry()); + .updateMaxReadPositionNotIncreaseChangeTimes( + topic.getManagedLedger().getLastConfirmedEntry()); } if (!changeToReadyState()) { log.error("[{}]Transaction buffer recover fail, current state: {}", @@ -147,9 +140,6 @@ public void recoverComplete() { @Override public void noNeedToRecover() { synchronized (TopicTransactionBuffer.this) { - // sync maxReadPosition change to LAC when TopicTransaction buffer have not recover - // completely the normal message have been sent to broker and state is - // not NoSnapshot can't sync maxReadPosition snapshotAbortedTxnProcessor .updateMaxReadPositionNotIncreaseChangeTimes(topic.getManagedLedger() .getLastConfirmedEntry()); @@ -202,7 +192,7 @@ public void recoverExceptionally(Throwable e) { recoverTime.setRecoverEndTime(System.currentTimeMillis()); topic.close(true); } - }, this.topic, this, takeSnapshotWriter, snapshotAbortedTxnProcessor)); + }, this.topic, this, snapshotAbortedTxnProcessor)); } @Override @@ -456,7 +446,7 @@ public CompletableFuture clearSnapshot() { @Override public CompletableFuture closeAsync() { changeToCloseState(); - return this.takeSnapshotWriter.thenCompose(SystemTopicClient.Writer::closeAsync); + return this.snapshotAbortedTxnProcessor.closeAsync(); } @Override @@ -535,88 +525,76 @@ public static class TopicTransactionBufferRecover implements Runnable { private final TopicTransactionBuffer topicTransactionBuffer; - private final CompletableFuture> takeSnapshotWriter; - private final AbortedTxnProcessor abortedTxnProcessor; private TopicTransactionBufferRecover(TopicTransactionBufferRecoverCallBack callBack, PersistentTopic topic, - TopicTransactionBuffer transactionBuffer, CompletableFuture< - SystemTopicClient.Writer> takeSnapshotWriter, + TopicTransactionBuffer transactionBuffer, AbortedTxnProcessor abortedTxnProcessor) { this.topic = topic; this.callBack = callBack; this.entryQueue = new SpscArrayQueue<>(2000); this.topicTransactionBuffer = transactionBuffer; - this.takeSnapshotWriter = takeSnapshotWriter; this.abortedTxnProcessor = abortedTxnProcessor; } @SneakyThrows @Override public void run() { - this.takeSnapshotWriter.thenRunAsync(() -> { - if (!this.topicTransactionBuffer.changeToInitializingState()) { - log.warn("TransactionBuffer {} of topic {} can not change state to Initializing", - this, topic.getName()); + if (!this.topicTransactionBuffer.changeToInitializingState()) { + log.warn("TransactionBuffer {} of topic {} can not change state to Initializing", + this, topic.getName()); + return; + } + abortedTxnProcessor.recoverFromSnapshot().thenAcceptAsync(startReadCursorPosition -> { + //Transaction is not enable for this topic, so just make maxReadPosition as LAC. + if (startReadCursorPosition == null) { + callBack.noNeedToRecover(); return; + } else { + this.startReadCursorPosition = startReadCursorPosition; } - abortedTxnProcessor.recoverFromSnapshot().thenAcceptAsync(startReadCursorPosition -> { - //Transaction is not enable for this topic, so just make maxReadPosition as LAC. - if (startReadCursorPosition == null) { - callBack.noNeedToRecover(); - return; - } else { - this.startReadCursorPosition = startReadCursorPosition; - } - ManagedCursor managedCursor; - try { - managedCursor = topic.getManagedLedger() - .newNonDurableCursor(this.startReadCursorPosition, SUBSCRIPTION_NAME); - } catch (ManagedLedgerException e) { - callBack.recoverExceptionally(e); - log.error("[{}]Transaction buffer recover fail when open cursor!", topic.getName(), e); - return; - } - PositionImpl lastConfirmedEntry = - (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); - PositionImpl currentLoadPosition = (PositionImpl) this.startReadCursorPosition; - FillEntryQueueCallback fillEntryQueueCallback = new FillEntryQueueCallback(entryQueue, - managedCursor, TopicTransactionBufferRecover.this); - if (lastConfirmedEntry.getEntryId() != -1) { - while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0 - && fillEntryQueueCallback.fillQueue()) { - Entry entry = entryQueue.poll(); - if (entry != null) { - try { - currentLoadPosition = PositionImpl.get(entry.getLedgerId(), - entry.getEntryId()); - callBack.handleTxnEntry(entry); - } finally { - entry.release(); - } - } else { - try { - Thread.sleep(1); - } catch (InterruptedException e) { - //no-op - } + ManagedCursor managedCursor; + try { + managedCursor = topic.getManagedLedger() + .newNonDurableCursor(this.startReadCursorPosition, SUBSCRIPTION_NAME); + } catch (ManagedLedgerException e) { + callBack.recoverExceptionally(e); + log.error("[{}]Transaction buffer recover fail when open cursor!", topic.getName(), e); + return; + } + PositionImpl lastConfirmedEntry = + (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); + PositionImpl currentLoadPosition = (PositionImpl) this.startReadCursorPosition; + FillEntryQueueCallback fillEntryQueueCallback = new FillEntryQueueCallback(entryQueue, + managedCursor, TopicTransactionBufferRecover.this); + if (lastConfirmedEntry.getEntryId() != -1) { + while (lastConfirmedEntry.compareTo(currentLoadPosition) > 0 + && fillEntryQueueCallback.fillQueue()) { + Entry entry = entryQueue.poll(); + if (entry != null) { + try { + currentLoadPosition = PositionImpl.get(entry.getLedgerId(), + entry.getEntryId()); + callBack.handleTxnEntry(entry); + } finally { + entry.release(); + } + } else { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + //no-op } } } + } - closeCursor(SUBSCRIPTION_NAME); - callBack.recoverComplete(); - }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() - .getExecutor(this)).exceptionally(e -> { - callBack.recoverExceptionally(e.getCause()); - log.error("[{}]Transaction buffer new snapshot reader fail!", topic.getName(), e); - return null; - }); + closeCursor(SUBSCRIPTION_NAME); + callBack.recoverComplete(); }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() .getExecutor(this)).exceptionally(e -> { callBack.recoverExceptionally(e.getCause()); - log.error("[{}]Transaction buffer create snapshot writer fail!", - topic.getName(), e); + log.error("[{}]Transaction buffer failed to recover snapshot!", topic.getName(), e); return null; }); } From 7b9fdadd1de7aeba206faad1a788f72abd2fd4dc Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Fri, 28 Oct 2022 15:01:16 +0800 Subject: [PATCH 16/32] change to TxnID --- .../buffer/AbortedTxnProcessor.java | 6 +- ...SingleSnapshotAbortedTxnProcessorImpl.java | 10 +- ...napshotSegmentAbortedTxnProcessorImpl.java | 103 +++++++++++------- .../buffer/impl/TopicTransactionBuffer.java | 13 +-- 4 files changed, 75 insertions(+), 57 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index 61199c0ce82df..85d9a8cca2dac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -22,7 +22,7 @@ import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; +import org.apache.pulsar.client.api.transaction.TxnID; public interface AbortedTxnProcessor extends TimerTask { @@ -32,7 +32,7 @@ public interface AbortedTxnProcessor extends TimerTask { * the transaction buffer will add the aborted transaction ID to AbortedTxnProcessor. * @param txnID aborted transaction ID. */ - void appendAbortedTxn(TxnIDData txnID, PositionImpl position); + void appendAbortedTxn(TxnID txnID, PositionImpl position); /** * After the transaction buffer writes a transaction aborted mark to the topic, @@ -61,7 +61,7 @@ public interface AbortedTxnProcessor extends TimerTask { * @param readPosition the read position of the transaction message, can be used to find the segment. * @return a boolean, whether the transaction ID is an aborted transaction ID. */ - boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition); + boolean checkAbortedTransaction(TxnID txnID, Position readPosition); /** * Recover transaction buffer by transaction buffer snapshot. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index a3692159dc106..ad1bca9ade078 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -35,8 +35,8 @@ import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.metadata.AbortTxnMetadata; import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; -import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.FutureUtil; @@ -52,7 +52,7 @@ public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcesso * Aborts, map for jude message is aborted, linked for remove abort txn in memory when this * position have been deleted. */ - private final LinkedMap aborts = new LinkedMap<>(); + private final LinkedMap aborts = new LinkedMap<>(); private volatile long lastSnapshotTimestamps; private final int takeSnapshotIntervalNumber; @@ -79,7 +79,7 @@ public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { } @Override - public void appendAbortedTxn(TxnIDData abortedTxnId, PositionImpl position) { + public void appendAbortedTxn(TxnID abortedTxnId, PositionImpl position) { aborts.put(abortedTxnId, position); } @@ -110,7 +110,7 @@ public void trimExpiredAbortedTxns() { } @Override - public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { + public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { return aborts.containsKey(txnID); } @@ -208,7 +208,7 @@ private void handleSnapshot(TransactionBufferSnapshot snapshot) { snapshot.getMaxReadPositionEntryId()); if (snapshot.getAborts() != null) { snapshot.getAborts().forEach(abortTxnMetadata -> - aborts.put(new TxnIDData(abortTxnMetadata.getTxnIdMostBits(), + aborts.put(new TxnID(abortTxnMetadata.getTxnIdMostBits(), abortTxnMetadata.getTxnIdLeastBits()), PositionImpl.get(abortTxnMetadata.getLedgerId(), abortTxnMetadata.getEntryId()))); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index e976a39f11ef4..252e0ea1d3233 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -24,7 +24,6 @@ import io.netty.util.Timer; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.TimeUnit; @@ -49,6 +48,7 @@ import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.naming.TopicDomain; @@ -61,10 +61,10 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess //Store the latest aborted transaction IDs and the latest max read position. private PositionImpl maxReadPosition; - private ArrayList unsealedAbortedTxnIdSegment = new ArrayList<>(); + private ArrayList unsealedAbortedTxnIdSegment = new ArrayList<>(); //Store the fixed aborted transaction segment - private final ConcurrentSkipListMap> abortTxnSegments + private final ConcurrentSkipListMap> abortTxnSegments = new ConcurrentSkipListMap<>(); private final ConcurrentSkipListMap indexes @@ -117,7 +117,7 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { } @Override - public void appendAbortedTxn(TxnIDData abortedTxnId, PositionImpl maxReadPosition) { + public void appendAbortedTxn(TxnID abortedTxnId, PositionImpl maxReadPosition) { unsealedAbortedTxnIdSegment.add(abortedTxnId); //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { @@ -143,7 +143,7 @@ public void updateMaxReadPositionNotIncreaseChangeTimes(Position maxReadPosition } @Override - public boolean checkAbortedTransaction(TxnIDData txnID, Position readPosition) { + public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { if (readPosition == null) { return abortTxnSegments.values().stream() .anyMatch(list -> list.contains(txnID)) || unsealedAbortedTxnIdSegment.contains(txnID); @@ -208,37 +208,16 @@ public void run(Timeout timeout) { @Override public CompletableFuture takesFirstSnapshot() { CompletableFuture completableFuture = new CompletableFuture<>(); - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> { - TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); - return snapshotIndexWriterFuture - .thenCompose((indexesWriter) -> { - TransactionBufferSnapshotIndexesMetadata transactionBufferSnapshotIndexesMetadata = - new TransactionBufferSnapshotIndexesMetadata(); - transactionBufferSnapshotIndexesMetadata.setAborts(unsealedAbortedTxnIdSegment); - transactionBufferSnapshotIndexesMetadata.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); - transactionBufferSnapshotIndexesMetadata.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); - indexes.setSnapshot(transactionBufferSnapshotIndexesMetadata); - indexes.setIndexList(new ArrayList<>()); - indexes.setTopicName(this.topic.getName()); - return indexesWriter.writeAsync(topic.getName(), indexes); - }) - .thenRun(() -> { - persistentSnapshotIndexes.setSnapshot(indexes.getSnapshot()); - persistentSnapshotIndexes.setIndexList(indexes.getIndexList()); - persistentSnapshotIndexes.setTopicName(this.topic.getName()); - this.lastSnapshotTimestamps = System.currentTimeMillis(); - completableFuture.complete(null); - }) - .exceptionally(e -> { - log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); - completableFuture.completeExceptionally(e); - return null; - }); - }); + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, + () -> persistentWorker.writeFirstSnapshot() + .thenRun(() -> completableFuture.complete(null)) + .exceptionally(e -> { + completableFuture.completeExceptionally(e); + return null; + })); return completableFuture; } - @Override public CompletableFuture recoverFromSnapshot() { return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() @@ -278,8 +257,8 @@ public CompletableFuture recoverFromSnapshot() { transactionBufferSnapshotIndex.persistentPositionLedgerID, transactionBufferSnapshotIndex.persistentPositionEntryID), transactionBufferSnapshotIndex)); - this.unsealedAbortedTxnIdSegment = (ArrayList) persistentSnapshotIndexes - .getSnapshot().getAborts(); + this.unsealedAbortedTxnIdSegment = deserializationFotSnapshotSegment(persistentSnapshotIndexes + .getSnapshot().getAborts()); this.maxReadPosition = new PositionImpl(persistentSnapshotIndexes .getSnapshot().getMaxReadPositionLedgerId(), persistentSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); @@ -415,7 +394,8 @@ private void handleSnapshotSegmentEntry(Entry entry) { TransactionBufferSnapshotSegment snapshotSegment = Schema.AVRO(TransactionBufferSnapshotSegment.class) .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer()); abortTxnSegments.put(new PositionImpl(snapshotSegment.getMaxReadPositionLedgerId(), - snapshotSegment.getMaxReadPositionEntryId()), (ArrayList) snapshotSegment.getAborts()); + snapshotSegment.getMaxReadPositionEntryId()), deserializationFotSnapshotSegment( + snapshotSegment.getAborts())); } @@ -540,7 +520,7 @@ private void executeTask() { } } - protected CompletableFuture takeSnapshotSegmentAsync(ArrayList sealedAbortedTxnIdSegment, + protected CompletableFuture takeSnapshotSegmentAsync(ArrayList sealedAbortedTxnIdSegment, PositionImpl maxReadPosition) { return writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, maxReadPosition).thenRun(() -> { if (log.isDebugEnabled()) { @@ -559,9 +539,9 @@ protected CompletableFuture takeSnapshotSegmentAsync(ArrayList }); } - private CompletableFuture writeSnapshotSegmentAsync(List segment, PositionImpl maxReadPosition) { + private CompletableFuture writeSnapshotSegmentAsync(List segment, PositionImpl maxReadPosition) { TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = new TransactionBufferSnapshotSegment(); - transactionBufferSnapshotSegment.setAborts(segment); + transactionBufferSnapshotSegment.setAborts(serializationForSegment(segment)); transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); transactionBufferSnapshotSegment.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); transactionBufferSnapshotSegment.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); @@ -640,7 +620,8 @@ private CompletableFuture updateIndexMetadataForTheLastSnapshot() { //Store the latest metadata TransactionBufferSnapshotIndexesMetadata transactionBufferSnapshotSegment = new TransactionBufferSnapshotIndexesMetadata(); - transactionBufferSnapshotSegment.setAborts(unsealedAbortedTxnIdSegment); + transactionBufferSnapshotSegment + .setAborts(serializationForSegment(unsealedAbortedTxnIdSegment)); indexes.setSnapshot(transactionBufferSnapshotSegment); //Only update the metadata in indexes and keep the index in indexes unchanged. indexes.setIndexList(persistentSnapshotIndexes.getIndexList()); @@ -656,6 +637,48 @@ private CompletableFuture updateIndexMetadataForTheLastSnapshot() { }); } + protected CompletableFuture writeFirstSnapshot() { + TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); + return snapshotIndexWriterFuture + .thenCompose((indexesWriter) -> { + TransactionBufferSnapshotIndexesMetadata transactionBufferSnapshotIndexesMetadata = + new TransactionBufferSnapshotIndexesMetadata(); + transactionBufferSnapshotIndexesMetadata + .setAborts(serializationForSegment(unsealedAbortedTxnIdSegment)); + transactionBufferSnapshotIndexesMetadata.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); + transactionBufferSnapshotIndexesMetadata.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); + indexes.setSnapshot(transactionBufferSnapshotIndexesMetadata); + indexes.setIndexList(new ArrayList<>()); + indexes.setTopicName(this.topic.getName()); + return indexesWriter.writeAsync(topic.getName(), indexes); + }) + .thenRun(() -> { + persistentSnapshotIndexes.setSnapshot(indexes.getSnapshot()); + persistentSnapshotIndexes.setIndexList(indexes.getIndexList()); + persistentSnapshotIndexes.setTopicName(this.topic.getName()); + lastSnapshotTimestamps = System.currentTimeMillis(); + }) + .exceptionally(e -> { + log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); + return null; + }); + } + } + + ArrayList deserializationFotSnapshotSegment(List snapshotSegment) { + ArrayList arrayList = new ArrayList<>(); + snapshotSegment.forEach(txnIDData -> { + arrayList.add(new TxnID(txnIDData.getMostSigBits(), txnIDData.getLeastSigBits())); + }); + return arrayList; + } + + ArrayList serializationForSegment(List segment) { + ArrayList arrayList = new ArrayList<>(); + segment.forEach(txnID -> { + arrayList.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits())); + }); + return arrayList; } } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index c751f8967b704..196aaef6b6c70 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -46,11 +46,9 @@ import org.apache.pulsar.broker.transaction.buffer.TransactionBufferReader; import org.apache.pulsar.broker.transaction.buffer.TransactionMeta; import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; -import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.proto.MessageMetadata; -import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TransactionBufferStats; import org.apache.pulsar.common.policies.data.TransactionInBufferStats; import org.apache.pulsar.common.protocol.Commands; @@ -163,8 +161,7 @@ public void handleTxnEntry(Entry entry) { PositionImpl position = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); if (Markers.isTxnMarker(msgMetadata)) { if (Markers.isTxnAbortMarker(msgMetadata)) { - snapshotAbortedTxnProcessor.appendAbortedTxn( - new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits()), position); + snapshotAbortedTxnProcessor.appendAbortedTxn(txnID, position); } updateMaxReadPosition(txnID); } else { @@ -276,7 +273,7 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { private void handleTransactionMessage(TxnID txnId, Position position) { if (!ongoingTxns.containsKey(txnId) && !this.snapshotAbortedTxnProcessor.checkAbortedTransaction( - new TxnIDData(txnId.getMostSigBits(), txnId.getLeastSigBits()), position)) { + txnId, position)) { ongoingTxns.put(txnId, (PositionImpl) position); PositionImpl firstPosition = ongoingTxns.get(ongoingTxns.firstKey()); //max read position is less than first ongoing transaction message position, so entryId -1 @@ -351,8 +348,7 @@ public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { PositionImpl maxReadPosition = updateMaxReadPosition(txnID); - snapshotAbortedTxnProcessor.appendAbortedTxn(new TxnIDData(txnID.getMostSigBits(), - txnID.getLeastSigBits()), maxReadPosition); + snapshotAbortedTxnProcessor.appendAbortedTxn(txnID, maxReadPosition); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); } txnAbortedCounter.increment(); @@ -451,8 +447,7 @@ public CompletableFuture closeAsync() { @Override public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { - return snapshotAbortedTxnProcessor.checkAbortedTransaction( - new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits()), readPosition); + return snapshotAbortedTxnProcessor.checkAbortedTransaction(txnID, readPosition); } @Override From da0566844b16828698754d6e8aeae55ffd8f6181 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Fri, 28 Oct 2022 15:21:17 +0800 Subject: [PATCH 17/32] Move the task implementation to PersistentWorker --- ...napshotSegmentAbortedTxnProcessorImpl.java | 128 ++++++++---------- 1 file changed, 59 insertions(+), 69 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 252e0ea1d3233..8b0e733030dc9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -88,12 +88,6 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess private final int takeSnapshotIntervalTime; private final int transactionBufferMaxAbortedTxnsOfSnapshotSegment; - - //Persistent snapshot segment and index at the single thread. - private final CompletableFuture> - snapshotSegmentsWriterFuture; - private final CompletableFuture> - snapshotIndexWriterFuture; private final PersistentWorker persistentWorker; public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { @@ -106,13 +100,6 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotSegmentSize(); - snapshotSegmentsWriterFuture = this.topic.getBrokerService().getPulsar() - .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName())); - snapshotIndexWriterFuture = this.topic.getBrokerService().getPulsar() - .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName())); - this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); } @@ -208,8 +195,16 @@ public void run(Timeout timeout) { @Override public CompletableFuture takesFirstSnapshot() { CompletableFuture completableFuture = new CompletableFuture<>(); + //build first snapshot index + TransactionBufferSnapshotIndexesMetadata transactionBufferSnapshotIndexesMetadata = + new TransactionBufferSnapshotIndexesMetadata(); + transactionBufferSnapshotIndexesMetadata + .setAborts(serializationForSegment(unsealedAbortedTxnIdSegment)); + transactionBufferSnapshotIndexesMetadata.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); + transactionBufferSnapshotIndexesMetadata.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, - () -> persistentWorker.writeFirstSnapshot() + () -> persistentWorker.updateSnapshotIndex(transactionBufferSnapshotIndexesMetadata, new ArrayList<>()) .thenRun(() -> completableFuture.complete(null)) .exceptionally(e -> { completableFuture.completeExceptionally(e); @@ -337,37 +332,17 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob public CompletableFuture clearAndCloseAsync() { timer.stop(); CompletableFuture completableFuture = new CompletableFuture<>(); - persistentWorker.appendTask(PersistentWorker.OperationType.Close, () -> { - ArrayList> completableFutures = new ArrayList<>(); - //Delete all segment - while (!abortTxnSegments.isEmpty()) { - if (log.isDebugEnabled()) { - log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", - topic.getName(), abortTxnSegments.firstKey()); - } - PositionImpl positionNeedToDelete = abortTxnSegments.firstKey(); - completableFutures.add(persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); - } - //Delete index - return FutureUtil.waitForAll(completableFutures) - .thenCompose((ignore) -> snapshotIndexWriterFuture - .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) - .thenRun(() -> { - log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", - topic.getName()); - completableFuture.thenCompose(null); - }) - .exceptionally(e -> { - log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", - topic.getName(), e); - completableFuture.completeExceptionally(e); - return null; - }); - }); + persistentWorker.appendTask(PersistentWorker.OperationType.Close, + () -> persistentWorker.clearSnapshotSegmentAndIndexes() + .thenRun(() -> { + completableFuture.thenCompose(null); + }).exceptionally(e -> { + completableFuture.completeExceptionally(e); + return null; + })); return completableFuture; } - @Override public PositionImpl getMaxReadPosition() { return this.maxReadPosition; @@ -380,10 +355,7 @@ public long getLastSnapshotTimestamps() { @Override public CompletableFuture closeAsync() { - ArrayList> completableFutures = new ArrayList<>(); - completableFutures.add(this.snapshotIndexWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync)); - completableFutures.add(this.snapshotSegmentsWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync)); - return FutureUtil.waitForAll(completableFutures); + return persistentWorker.closeAsync(); } private void handleSnapshotSegmentEntry(Entry entry) { @@ -411,6 +383,12 @@ private class PersistentWorker { private final PersistentTopic topic; + //Persistent snapshot segment and index at the single thread. + private final CompletableFuture> + snapshotSegmentsWriterFuture; + private final CompletableFuture> + snapshotIndexWriterFuture; + private enum OperationState { None, UpdatingIndex, @@ -440,6 +418,12 @@ public enum OperationType { public PersistentWorker(PersistentTopic topic) { this.topic = topic; this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); + this.snapshotSegmentsWriterFuture = this.topic.getBrokerService().getPulsar() + .getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName())); + this.snapshotIndexWriterFuture = this.topic.getBrokerService().getPulsar() + .getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName())); } @@ -589,8 +573,7 @@ private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedT }); } - //Update the indexes in the transactionBufferSnapshotIndexe. - //Concurrency control is performed by snapshotIndexWriterFuture. + //Update the indexes with the giving index snapshot and indexlist in the transactionBufferSnapshotIndexe. private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotIndexesMetadata snapshotSegment, List indexList) { TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); @@ -612,7 +595,6 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd } //Only update the metadata in the transactionBufferSnapshotIndexes. - //Concurrency control is performed by snapshotIndexWriterFuture. private CompletableFuture updateIndexMetadataForTheLastSnapshot() { TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); return snapshotIndexWriterFuture @@ -637,32 +619,40 @@ private CompletableFuture updateIndexMetadataForTheLastSnapshot() { }); } - protected CompletableFuture writeFirstSnapshot() { - TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); - return snapshotIndexWriterFuture - .thenCompose((indexesWriter) -> { - TransactionBufferSnapshotIndexesMetadata transactionBufferSnapshotIndexesMetadata = - new TransactionBufferSnapshotIndexesMetadata(); - transactionBufferSnapshotIndexesMetadata - .setAborts(serializationForSegment(unsealedAbortedTxnIdSegment)); - transactionBufferSnapshotIndexesMetadata.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); - transactionBufferSnapshotIndexesMetadata.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); - indexes.setSnapshot(transactionBufferSnapshotIndexesMetadata); - indexes.setIndexList(new ArrayList<>()); - indexes.setTopicName(this.topic.getName()); - return indexesWriter.writeAsync(topic.getName(), indexes); - }) + protected CompletableFuture clearSnapshotSegmentAndIndexes() { + ArrayList> completableFutures = new ArrayList<>(); + //Delete all segment + while (!abortTxnSegments.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", + topic.getName(), abortTxnSegments.firstKey()); + } + PositionImpl positionNeedToDelete = abortTxnSegments.firstKey(); + completableFutures.add(persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); + } + //Delete index + return FutureUtil.waitForAll(completableFutures) + .thenCompose((ignore) -> snapshotIndexWriterFuture + .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) .thenRun(() -> { - persistentSnapshotIndexes.setSnapshot(indexes.getSnapshot()); - persistentSnapshotIndexes.setIndexList(indexes.getIndexList()); - persistentSnapshotIndexes.setTopicName(this.topic.getName()); - lastSnapshotTimestamps = System.currentTimeMillis(); + log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", + topic.getName()); + }) .exceptionally(e -> { - log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); + log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", + topic.getName(), e); + return null; }); } + + + CompletableFuture closeAsync() { + return CompletableFuture.allOf( + this.snapshotIndexWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync), + this.snapshotSegmentsWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync)); + } } ArrayList deserializationFotSnapshotSegment(List snapshotSegment) { From 721c096aa769ac25b0695cbb4f05742897cb57de Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Fri, 28 Oct 2022 15:35:38 +0800 Subject: [PATCH 18/32] clear logic --- ...napshotSegmentAbortedTxnProcessorImpl.java | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 8b0e733030dc9..3b962efedb4c5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -195,16 +195,8 @@ public void run(Timeout timeout) { @Override public CompletableFuture takesFirstSnapshot() { CompletableFuture completableFuture = new CompletableFuture<>(); - //build first snapshot index - TransactionBufferSnapshotIndexesMetadata transactionBufferSnapshotIndexesMetadata = - new TransactionBufferSnapshotIndexesMetadata(); - transactionBufferSnapshotIndexesMetadata - .setAborts(serializationForSegment(unsealedAbortedTxnIdSegment)); - transactionBufferSnapshotIndexesMetadata.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); - transactionBufferSnapshotIndexesMetadata.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, - () -> persistentWorker.updateSnapshotIndex(transactionBufferSnapshotIndexesMetadata, new ArrayList<>()) + () -> persistentWorker.updateIndexMetadataForTheLastSnapshot() .thenRun(() -> completableFuture.complete(null)) .exceptionally(e -> { completableFuture.completeExceptionally(e); @@ -600,17 +592,23 @@ private CompletableFuture updateIndexMetadataForTheLastSnapshot() { return snapshotIndexWriterFuture .thenCompose((indexesWriter) -> { //Store the latest metadata - TransactionBufferSnapshotIndexesMetadata transactionBufferSnapshotSegment = + TransactionBufferSnapshotIndexesMetadata transactionBufferSnapshotIndexesMetadata = new TransactionBufferSnapshotIndexesMetadata(); - transactionBufferSnapshotSegment + transactionBufferSnapshotIndexesMetadata .setAborts(serializationForSegment(unsealedAbortedTxnIdSegment)); - indexes.setSnapshot(transactionBufferSnapshotSegment); + transactionBufferSnapshotIndexesMetadata.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); + transactionBufferSnapshotIndexesMetadata.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); + indexes.setSnapshot(transactionBufferSnapshotIndexesMetadata); //Only update the metadata in indexes and keep the index in indexes unchanged. - indexes.setIndexList(persistentSnapshotIndexes.getIndexList()); + if (indexes.getIndexList() == null) { + indexes.setIndexList(new ArrayList<>()); + } else { + indexes.setIndexList(persistentSnapshotIndexes.getIndexList()); + } return indexesWriter.writeAsync(topic.getName(), indexes); }) .thenRun(() -> { - persistentSnapshotIndexes.setSnapshot(indexes.getSnapshot()); + persistentSnapshotIndexes = indexes; lastSnapshotTimestamps = System.currentTimeMillis(); }) .exceptionally(e -> { From ae9af72460bf36995b3d3a1e87192ec0d53b1e09 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Fri, 28 Oct 2022 22:09:09 +0800 Subject: [PATCH 19/32] fix some comments --- ...napshotSegmentAbortedTxnProcessorImpl.java | 23 +++++++++++-------- .../buffer/impl/TopicTransactionBuffer.java | 10 ++++---- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 3b962efedb4c5..5c8da1c435911 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -38,6 +39,8 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; @@ -402,8 +405,8 @@ public enum OperationType { private volatile OperationState operationState = OperationState.None; - ConcurrentSkipListMap>> taskQueue = - new ConcurrentSkipListMap<>(); + ConcurrentLinkedDeque>>> taskQueue = + new ConcurrentLinkedDeque<>(); private CompletableFuture lastOperationFuture; private final Timer timer; @@ -436,7 +439,7 @@ public void appendTask(OperationType operationType, Supplier { - taskQueue.put(operationType, task); + taskQueue.add(new MutablePair<>(operationType, task)); executeTask(); } case Close -> { @@ -452,12 +455,12 @@ public void appendTask(OperationType operationType, Supplier { if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.WritingSegment)) { - if (taskQueue.firstKey() == OperationType.WriteSegment) { - lastOperationFuture = taskQueue.firstEntry().getValue().get(); + if (taskQueue.getFirst().getKey() == OperationType.WriteSegment) { + lastOperationFuture = taskQueue.getFirst().getValue().get(); lastOperationFuture.whenComplete((ignore, throwable) -> { if (throwable != null) { if (log.isDebugEnabled()) { @@ -466,7 +469,7 @@ private void executeTask() { timer.newTimeout(timeout -> executeTask(), takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); } else { - taskQueue.remove(taskQueue.firstKey()); + taskQueue.removeFirst(); } STATE_UPDATER.compareAndSet(this, OperationState.WritingSegment, OperationState.None); }); @@ -475,8 +478,8 @@ private void executeTask() { } case DeleteSegment -> { if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.DeletingSegment)) { - if (taskQueue.firstKey() == OperationType.DeleteSegment) { - lastOperationFuture = taskQueue.firstEntry().getValue().get(); + if (taskQueue.getFirst().getKey() == OperationType.DeleteSegment) { + lastOperationFuture = taskQueue.getFirst().getValue().get(); lastOperationFuture.whenComplete((ignore, throwable) -> { if (throwable != null) { if (log.isDebugEnabled()) { @@ -485,7 +488,7 @@ private void executeTask() { timer.newTimeout(timeout -> executeTask(), takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); } else { - taskQueue.remove(taskQueue.firstKey()); + taskQueue.removeFirst(); } STATE_UPDATER.compareAndSet(this, OperationState.DeletingSegment, OperationState.None); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 196aaef6b6c70..1420026fd2e3b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -163,7 +163,7 @@ public void handleTxnEntry(Entry entry) { if (Markers.isTxnAbortMarker(msgMetadata)) { snapshotAbortedTxnProcessor.appendAbortedTxn(txnID, position); } - updateMaxReadPosition(txnID); + snapshotAbortedTxnProcessor.updateMaxReadPosition(getMaxReadPosition(txnID)); } else { handleTransactionMessage(txnID, position); } @@ -302,7 +302,7 @@ public CompletableFuture commitTxn(TxnID txnID, long lowWaterMark) { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { - updateMaxReadPosition(txnID); + snapshotAbortedTxnProcessor.updateMaxReadPosition(getMaxReadPosition(txnID)); handleLowWaterMark(txnID, lowWaterMark); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); } @@ -347,8 +347,9 @@ public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { - PositionImpl maxReadPosition = updateMaxReadPosition(txnID); + PositionImpl maxReadPosition = getMaxReadPosition(txnID); snapshotAbortedTxnProcessor.appendAbortedTxn(txnID, maxReadPosition); + snapshotAbortedTxnProcessor.updateMaxReadPosition(maxReadPosition); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); } txnAbortedCounter.increment(); @@ -414,7 +415,7 @@ private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { } } - PositionImpl updateMaxReadPosition(TxnID txnID) { + PositionImpl getMaxReadPosition(TxnID txnID) { ongoingTxns.remove(txnID); PositionImpl maxReadPosition; if (!ongoingTxns.isEmpty()) { @@ -424,7 +425,6 @@ PositionImpl updateMaxReadPosition(TxnID txnID) { } else { maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); } - snapshotAbortedTxnProcessor.updateMaxReadPosition(maxReadPosition); return maxReadPosition; } From 0e4a73eeddb1aba972222c974e6c45cb9cd409b6 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Fri, 28 Oct 2022 22:15:51 +0800 Subject: [PATCH 20/32] change synchronized `(SingleSnapshotAbortedTxnProcessorImpl.this)` to `synchronized (topic)` --- .../impl/SingleSnapshotAbortedTxnProcessorImpl.java | 2 +- .../buffer/impl/TopicTransactionBuffer.java | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index ad1bca9ade078..9e0e942e821e4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -219,7 +219,7 @@ private CompletableFuture takeSnapshot() { changeMaxReadPositionAndAddAbortTimes.set(0); return takeSnapshotWriter.thenCompose(writer -> { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); - synchronized (SingleSnapshotAbortedTxnProcessorImpl.this) { + synchronized (topic) { snapshot.setTopicName(topic.getName()); snapshot.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); snapshot.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 1420026fd2e3b..a88cd28e236c4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -113,7 +113,7 @@ private void recover() { .execute(new TopicTransactionBufferRecover(new TopicTransactionBufferRecoverCallBack() { @Override public void recoverComplete() { - synchronized (TopicTransactionBuffer.this) { + synchronized (topic) { if (ongoingTxns.isEmpty()) { snapshotAbortedTxnProcessor .updateMaxReadPositionNotIncreaseChangeTimes( @@ -137,7 +137,7 @@ public void recoverComplete() { @Override public void noNeedToRecover() { - synchronized (TopicTransactionBuffer.this) { + synchronized (topic) { snapshotAbortedTxnProcessor .updateMaxReadPositionNotIncreaseChangeTimes(topic.getManagedLedger() .getLastConfirmedEntry()); @@ -256,7 +256,7 @@ public CompletableFuture appendBufferToTxn(TxnID txnId, long sequenceI topic.getManagedLedger().asyncAddEntry(buffer, new AsyncCallbacks.AddEntryCallback() { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { - synchronized (TopicTransactionBuffer.this) { + synchronized (topic) { handleTransactionMessage(txnId, position); } completableFuture.complete(position); @@ -346,7 +346,7 @@ public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { topic.getManagedLedger().asyncAddEntry(abortMarker, new AsyncCallbacks.AddEntryCallback() { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { - synchronized (TopicTransactionBuffer.this) { + synchronized (topic) { PositionImpl maxReadPosition = getMaxReadPosition(txnID); snapshotAbortedTxnProcessor.appendAbortedTxn(txnID, maxReadPosition); snapshotAbortedTxnProcessor.updateMaxReadPosition(maxReadPosition); @@ -454,7 +454,7 @@ public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { public void syncMaxReadPositionForNormalPublish(PositionImpl position) { // when ongoing transaction is empty, proved that lastAddConfirm is can read max position, because callback // thread is the same tread, in this time the lastAddConfirm don't content transaction message. - synchronized (TopicTransactionBuffer.this) { + synchronized (topic) { if (checkIfNoSnapshot()) { //TODO:The changes time here should not be changed. snapshotAbortedTxnProcessor.updateMaxReadPositionNotIncreaseChangeTimes(position); From 45da1e68751869e39a6f4d9c3a83fc34882c8c6f Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Sat, 29 Oct 2022 22:36:07 +0800 Subject: [PATCH 21/32] move maxReadPosition back to TB --- .../buffer/AbortedTxnProcessor.java | 23 +--- ...SingleSnapshotAbortedTxnProcessorImpl.java | 45 +------- ...napshotSegmentAbortedTxnProcessorImpl.java | 101 +++--------------- .../buffer/impl/TopicTransactionBuffer.java | 77 +++++++++---- .../TopicTransactionBufferRecoverTest.java | 2 +- .../broker/transaction/TransactionTest.java | 5 +- 6 files changed, 77 insertions(+), 176 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index 85d9a8cca2dac..74b05952b7f43 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -25,7 +25,7 @@ import org.apache.pulsar.client.api.transaction.TxnID; -public interface AbortedTxnProcessor extends TimerTask { +public interface AbortedTxnProcessor { /** * After the transaction buffer writes a transaction aborted mark to the topic, @@ -34,19 +34,6 @@ public interface AbortedTxnProcessor extends TimerTask { */ void appendAbortedTxn(TxnID txnID, PositionImpl position); - /** - * After the transaction buffer writes a transaction aborted mark to the topic, - * the transaction buffer will update max read position in AbortedTxnProcessor - * @param maxReadPosition the max read position after the transaction is aborted. - */ - void updateMaxReadPosition(Position maxReadPosition); - - /** - * This method is used to updated max read position for the topic which nerver used transaction send message. - * @param maxReadPosition the max read position after the transaction is aborted. - */ - void updateMaxReadPositionNotIncreaseChangeTimes(Position maxReadPosition); - /** * Pulsar has a configuration for ledger retention time. * If the transaction aborted mark position has been deleted, the transaction is valid and can be clear. @@ -80,13 +67,7 @@ public interface AbortedTxnProcessor extends TimerTask { * Take the frist snapshot if the topic has no snapshot before. * @return a completableFuture. */ - CompletableFuture takesFirstSnapshot(); - - /** - * Get the max read position. - * @return the maxReadPosition. - */ - PositionImpl getMaxReadPosition(); + CompletableFuture takeAbortedTxnSnapshot(PositionImpl maxReadPosition); /** * Get the lastSnapshotTimestamps. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 9e0e942e821e4..3e082dec40759 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -18,12 +18,10 @@ */ package org.apache.pulsar.broker.transaction.buffer.impl; -import io.netty.util.Timeout; import io.netty.util.Timer; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; @@ -83,19 +81,6 @@ public void appendAbortedTxn(TxnID abortedTxnId, PositionImpl position) { aborts.put(abortedTxnId, position); } - @Override - public void updateMaxReadPosition(Position maxReadPosition) { - if (this.maxReadPosition != maxReadPosition) { - this.maxReadPosition = (PositionImpl) maxReadPosition; - takeSnapshotByChangeTimes(); - } - } - - @Override - public void updateMaxReadPositionNotIncreaseChangeTimes(Position maxReadPosition) { - this.maxReadPosition = (PositionImpl) maxReadPosition; - } - @Override public void trimExpiredAbortedTxns() { while (!aborts.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) @@ -163,13 +148,8 @@ public CompletableFuture clearAndCloseAsync() { } @Override - public CompletableFuture takesFirstSnapshot() { - return takeSnapshot(); - } - - @Override - public PositionImpl getMaxReadPosition() { - return maxReadPosition; + public CompletableFuture takeAbortedTxnSnapshot(PositionImpl maxReadPosition) { + return takeAbortedTxnSnapshot(maxReadPosition, aborts); } @Override @@ -189,20 +169,6 @@ private void closeReader(SystemTopicClient.Reader rea }); } - private void takeSnapshotByChangeTimes() { - if (changeMaxReadPositionAndAddAbortTimes.incrementAndGet() >= takeSnapshotIntervalNumber) { - takeSnapshot(); - } - } - - private void takeSnapshotByTimeout() { - if (changeMaxReadPositionAndAddAbortTimes.get() > 0) { - takeSnapshot(); - } - this.timer.newTimeout(SingleSnapshotAbortedTxnProcessorImpl.this, - takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); - } - private void handleSnapshot(TransactionBufferSnapshot snapshot) { maxReadPosition = PositionImpl.get(snapshot.getMaxReadPositionLedgerId(), snapshot.getMaxReadPositionEntryId()); @@ -215,7 +181,7 @@ private void handleSnapshot(TransactionBufferSnapshot snapshot) { } } - private CompletableFuture takeSnapshot() { + private CompletableFuture takeAbortedTxnSnapshot(PositionImpl maxReadPosition, LinkedMap aborts) { changeMaxReadPositionAndAddAbortTimes.set(0); return takeSnapshotWriter.thenCompose(writer -> { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); @@ -247,9 +213,4 @@ private CompletableFuture takeSnapshot() { }); } - @Override - public void run(Timeout timeout) { - takeSnapshotByTimeout(); - } - } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 5c8da1c435911..ad33798e53913 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -62,8 +62,6 @@ @Slf4j public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { - //Store the latest aborted transaction IDs and the latest max read position. - private PositionImpl maxReadPosition; private ArrayList unsealedAbortedTxnIdSegment = new ArrayList<>(); //Store the fixed aborted transaction segment @@ -80,14 +78,8 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess private final PersistentTopic topic; - //When add abort or change max read position, the count will +1. Take snapshot will set 0 into it. - private final AtomicLong changeMaxReadPositionAndAddAbortTimes = new AtomicLong(); - private volatile long lastSnapshotTimestamps; - //Configurations - private final int takeSnapshotIntervalNumber; - private final int takeSnapshotIntervalTime; private final int transactionBufferMaxAbortedTxnsOfSnapshotSegment; @@ -96,9 +88,6 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; this.persistentWorker = new PersistentWorker(topic); - this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); - this.takeSnapshotIntervalNumber = topic.getBrokerService().getPulsar() - .getConfiguration().getTransactionBufferSnapshotMaxTransactionCount(); this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = topic.getBrokerService().getPulsar() @@ -111,7 +100,6 @@ public void appendAbortedTxn(TxnID abortedTxnId, PositionImpl maxReadPosition) { unsealedAbortedTxnIdSegment.add(abortedTxnId); //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { - changeMaxReadPositionAndAddAbortTimes.set(0); abortTxnSegments.put(maxReadPosition, unsealedAbortedTxnIdSegment); persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> persistentWorker.takeSnapshotSegmentAsync(unsealedAbortedTxnIdSegment, maxReadPosition)); @@ -119,19 +107,6 @@ public void appendAbortedTxn(TxnID abortedTxnId, PositionImpl maxReadPosition) { } } - @Override - public void updateMaxReadPosition(Position position) { - if (position != this.maxReadPosition) { - this.maxReadPosition = (PositionImpl) position; - updateSnapshotIndexMetadataByChangeTimes(); - } - } - - @Override - public void updateMaxReadPositionNotIncreaseChangeTimes(Position maxReadPosition) { - this.maxReadPosition = (PositionImpl) maxReadPosition; - } - @Override public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { if (readPosition == null) { @@ -168,38 +143,19 @@ public void trimExpiredAbortedTxns() { } private String buildKey(long sequenceId) { - return "multiple-" + sequenceId + this.topic.getName(); - } - - private void updateSnapshotIndexMetadataByChangeTimes() { - if (this.changeMaxReadPositionAndAddAbortTimes.incrementAndGet() == takeSnapshotIntervalNumber) { - changeMaxReadPositionAndAddAbortTimes.set(0); - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, - persistentWorker::updateIndexMetadataForTheLastSnapshot); - } - } - - private void takeSnapshotByTimeout() { - if (changeMaxReadPositionAndAddAbortTimes.get() > 0) { - changeMaxReadPositionAndAddAbortTimes.set(0); - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, - persistentWorker::updateIndexMetadataForTheLastSnapshot); - } - timer.newTimeout(SnapshotSegmentAbortedTxnProcessorImpl.this, - takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); + return "multiple-" + sequenceId + "-" + this.topic.getName(); } @Override - public void run(Timeout timeout) { - takeSnapshotByTimeout(); + public CompletableFuture takeAbortedTxnSnapshot(PositionImpl maxReadPosition) { + return takeAbortedTxnSnapshot(maxReadPosition, unsealedAbortedTxnIdSegment); } - - @Override - public CompletableFuture takesFirstSnapshot() { + private CompletableFuture takeAbortedTxnSnapshot(PositionImpl maxReadPosition, ArrayList aborts) { CompletableFuture completableFuture = new CompletableFuture<>(); persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, - () -> persistentWorker.updateIndexMetadataForTheLastSnapshot() + () -> persistentWorker + .updateIndexMetadataForTheLastSnapshot(maxReadPosition, aborts) .thenRun(() -> completableFuture.complete(null)) .exceptionally(e -> { completableFuture.completeExceptionally(e); @@ -249,9 +205,6 @@ public CompletableFuture recoverFromSnapshot() { transactionBufferSnapshotIndex)); this.unsealedAbortedTxnIdSegment = deserializationFotSnapshotSegment(persistentSnapshotIndexes .getSnapshot().getAborts()); - this.maxReadPosition = new PositionImpl(persistentSnapshotIndexes - .getSnapshot().getMaxReadPositionLedgerId(), - persistentSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); if (indexes.size() != 0) { persistentWorker.sequenceID.set(indexes.lastEntry().getValue().sequenceID + 1); } @@ -338,11 +291,6 @@ public CompletableFuture clearAndCloseAsync() { return completableFuture; } - @Override - public PositionImpl getMaxReadPosition() { - return this.maxReadPosition; - } - @Override public long getLastSnapshotTimestamps() { return this.lastSnapshotTimestamps; @@ -579,8 +527,7 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd return indexesWriter.writeAsync(topic.getName(), snapshotIndexes); }) .thenRun(() -> { - persistentSnapshotIndexes.setIndexList(indexList); - persistentSnapshotIndexes.setSnapshot(snapshotSegment); + persistentSnapshotIndexes = snapshotIndexes; lastSnapshotTimestamps = System.currentTimeMillis(); }) .exceptionally(e -> { @@ -590,34 +537,12 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd } //Only update the metadata in the transactionBufferSnapshotIndexes. - private CompletableFuture updateIndexMetadataForTheLastSnapshot() { - TransactionBufferSnapshotIndexes indexes = new TransactionBufferSnapshotIndexes(); - return snapshotIndexWriterFuture - .thenCompose((indexesWriter) -> { - //Store the latest metadata - TransactionBufferSnapshotIndexesMetadata transactionBufferSnapshotIndexesMetadata = - new TransactionBufferSnapshotIndexesMetadata(); - transactionBufferSnapshotIndexesMetadata - .setAborts(serializationForSegment(unsealedAbortedTxnIdSegment)); - transactionBufferSnapshotIndexesMetadata.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); - transactionBufferSnapshotIndexesMetadata.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); - indexes.setSnapshot(transactionBufferSnapshotIndexesMetadata); - //Only update the metadata in indexes and keep the index in indexes unchanged. - if (indexes.getIndexList() == null) { - indexes.setIndexList(new ArrayList<>()); - } else { - indexes.setIndexList(persistentSnapshotIndexes.getIndexList()); - } - return indexesWriter.writeAsync(topic.getName(), indexes); - }) - .thenRun(() -> { - persistentSnapshotIndexes = indexes; - lastSnapshotTimestamps = System.currentTimeMillis(); - }) - .exceptionally(e -> { - log.error("[{}] Failed to update snapshot segment index", indexes.getTopicName(), e); - return null; - }); + private CompletableFuture updateIndexMetadataForTheLastSnapshot(PositionImpl maxReadPosition, + ArrayList abortedTxns) { + TransactionBufferSnapshotIndexesMetadata metadata = new TransactionBufferSnapshotIndexesMetadata( + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), serializationForSegment(abortedTxns)); + + return updateSnapshotIndex(metadata, persistentSnapshotIndexes.getIndexList()); } protected CompletableFuture clearSnapshotSegmentAndIndexes() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index a88cd28e236c4..9a14bb9c0bc5b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -20,7 +20,9 @@ import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; +import io.netty.util.Timeout; import io.netty.util.Timer; +import io.netty.util.TimerTask; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -62,7 +64,7 @@ * Transaction buffer based on normal persistent topic. */ @Slf4j -public class TopicTransactionBuffer extends TopicTransactionBufferState implements TransactionBuffer { +public class TopicTransactionBuffer extends TopicTransactionBufferState implements TransactionBuffer, TimerTask { private final PersistentTopic topic; @@ -71,6 +73,14 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen */ private final LinkedMap ongoingTxns = new LinkedMap<>(); + //Store the latest aborted transaction IDs and the latest max read position. + private PositionImpl maxReadPosition; + + //When add abort or change max read position, the count will +1. Take snapshot will set 0 into it. + private final AtomicLong changeMaxReadPositionAndAddAbortTimes = new AtomicLong(); + + private final int takeSnapshotIntervalNumber; + private final LongAdder txnCommittedCounter = new LongAdder(); private final LongAdder txnAbortedCounter = new LongAdder(); @@ -98,7 +108,8 @@ public TopicTransactionBuffer(PersistentTopic topic) { this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); - + this.takeSnapshotIntervalNumber = topic.getBrokerService().getPulsar() + .getConfiguration().getTransactionBufferSnapshotMaxTransactionCount(); if (topic.getBrokerService().getPulsar().getConfiguration().isTransactionBufferSegmentedSnapshotEnabled()) { snapshotAbortedTxnProcessor = new SnapshotSegmentAbortedTxnProcessorImpl(topic); } else { @@ -115,9 +126,7 @@ private void recover() { public void recoverComplete() { synchronized (topic) { if (ongoingTxns.isEmpty()) { - snapshotAbortedTxnProcessor - .updateMaxReadPositionNotIncreaseChangeTimes( - topic.getManagedLedger().getLastConfirmedEntry()); + updateMaxReadPosition(topic.getManagedLedger().getLastConfirmedEntry(), false); } if (!changeToReadyState()) { log.error("[{}]Transaction buffer recover fail, current state: {}", @@ -127,7 +136,7 @@ public void recoverComplete() { "Transaction buffer recover failed to change the status to Ready," + "current state is: " + getState())); } else { - timer.newTimeout(snapshotAbortedTxnProcessor, + timer.newTimeout(TopicTransactionBuffer.this, takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); transactionBufferFuture.complete(null); recoverTime.setRecoverEndTime(System.currentTimeMillis()); @@ -138,9 +147,7 @@ public void recoverComplete() { @Override public void noNeedToRecover() { synchronized (topic) { - snapshotAbortedTxnProcessor - .updateMaxReadPositionNotIncreaseChangeTimes(topic.getManagedLedger() - .getLastConfirmedEntry()); + updateMaxReadPosition(topic.getManagedLedger().getLastConfirmedEntry(), false); if (!changeToNoSnapshotState()) { log.error("[{}]Transaction buffer recover fail", topic.getName()); @@ -163,7 +170,7 @@ public void handleTxnEntry(Entry entry) { if (Markers.isTxnAbortMarker(msgMetadata)) { snapshotAbortedTxnProcessor.appendAbortedTxn(txnID, position); } - snapshotAbortedTxnProcessor.updateMaxReadPosition(getMaxReadPosition(txnID)); + updateMaxReadPosition(getMaxReadPosition(txnID), true); } else { handleTransactionMessage(txnID, position); } @@ -205,10 +212,10 @@ public CompletableFuture checkIfTBRecoverCompletely(boolean isTxnEnabled) CompletableFuture completableFuture = new CompletableFuture<>(); transactionBufferFuture.thenRun(() -> { if (checkIfNoSnapshot()) { - snapshotAbortedTxnProcessor.takesFirstSnapshot().thenRun(() -> { + snapshotAbortedTxnProcessor.takeAbortedTxnSnapshot(maxReadPosition).thenRun(() -> { if (changeToReadyStateFromNoSnapshot()) { - timer.newTimeout(snapshotAbortedTxnProcessor, - takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); + timer.newTimeout(TopicTransactionBuffer.this, takeSnapshotIntervalTime, + TimeUnit.MILLISECONDS); } completableFuture.complete(null); }).exceptionally(exception -> { @@ -228,6 +235,32 @@ public CompletableFuture checkIfTBRecoverCompletely(boolean isTxnEnabled) } } + private void updateSnapshotIndexMetadataByChangeTimes() { + if (this.changeMaxReadPositionAndAddAbortTimes.incrementAndGet() == takeSnapshotIntervalNumber) { + changeMaxReadPositionAndAddAbortTimes.set(0); + this.snapshotAbortedTxnProcessor.takeAbortedTxnSnapshot(maxReadPosition); + } + } + + private void takeSnapshotByTimeout() { + if (changeMaxReadPositionAndAddAbortTimes.get() > 0) { + changeMaxReadPositionAndAddAbortTimes.set(0); + this.snapshotAbortedTxnProcessor.takeAbortedTxnSnapshot(maxReadPosition); + } + timer.newTimeout(TopicTransactionBuffer.this, + takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); + } + + public void updateMaxReadPosition(Position position, boolean isUpdateChangeTimes) { + if (position != this.maxReadPosition && isUpdateChangeTimes) { + updateSnapshotIndexMetadataByChangeTimes(); + } + this.maxReadPosition = (PositionImpl) position; + } + @Override + public void run(Timeout timeout) { + takeSnapshotByTimeout(); + } @Override public long getOngoingTxnCount() { return this.ongoingTxns.size(); @@ -277,8 +310,8 @@ private void handleTransactionMessage(TxnID txnId, Position position) { ongoingTxns.put(txnId, (PositionImpl) position); PositionImpl firstPosition = ongoingTxns.get(ongoingTxns.firstKey()); //max read position is less than first ongoing transaction message position, so entryId -1 - snapshotAbortedTxnProcessor.updateMaxReadPosition(PositionImpl.get(firstPosition.getLedgerId(), - firstPosition.getEntryId() - 1)); + updateMaxReadPosition(PositionImpl.get(firstPosition.getLedgerId(), firstPosition.getEntryId() - 1), + true); } } @@ -302,7 +335,7 @@ public CompletableFuture commitTxn(TxnID txnID, long lowWaterMark) { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { - snapshotAbortedTxnProcessor.updateMaxReadPosition(getMaxReadPosition(txnID)); + updateMaxReadPosition(getMaxReadPosition(txnID), true); handleLowWaterMark(txnID, lowWaterMark); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); } @@ -349,7 +382,7 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (topic) { PositionImpl maxReadPosition = getMaxReadPosition(txnID); snapshotAbortedTxnProcessor.appendAbortedTxn(txnID, maxReadPosition); - snapshotAbortedTxnProcessor.updateMaxReadPosition(maxReadPosition); + updateMaxReadPosition(maxReadPosition, true); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); } txnAbortedCounter.increment(); @@ -456,11 +489,10 @@ public void syncMaxReadPositionForNormalPublish(PositionImpl position) { // thread is the same tread, in this time the lastAddConfirm don't content transaction message. synchronized (topic) { if (checkIfNoSnapshot()) { - //TODO:The changes time here should not be changed. - snapshotAbortedTxnProcessor.updateMaxReadPositionNotIncreaseChangeTimes(position); + updateMaxReadPosition(position, false); } else if (checkIfReady()) { if (ongoingTxns.isEmpty()) { - snapshotAbortedTxnProcessor.updateMaxReadPosition(position); + updateMaxReadPosition(position, true); } } } @@ -469,7 +501,7 @@ public void syncMaxReadPositionForNormalPublish(PositionImpl position) { @Override public PositionImpl getMaxReadPosition() { if (checkIfReady() || checkIfNoSnapshot()) { - return this.snapshotAbortedTxnProcessor.getMaxReadPosition(); + return this.maxReadPosition; } else { return PositionImpl.EARLIEST; } @@ -490,7 +522,7 @@ public TransactionBufferStats getStats(boolean lowWaterMarks) { TransactionBufferStats transactionBufferStats = new TransactionBufferStats(); transactionBufferStats.lastSnapshotTimestamps = this.snapshotAbortedTxnProcessor.getLastSnapshotTimestamps(); transactionBufferStats.state = this.getState().name(); - transactionBufferStats.maxReadPosition = this.snapshotAbortedTxnProcessor.getMaxReadPosition().toString(); + transactionBufferStats.maxReadPosition = this.maxReadPosition.toString(); if (lowWaterMarks) { transactionBufferStats.lowWaterMarks = this.lowWaterMarks; } @@ -547,6 +579,7 @@ public void run() { return; } else { this.startReadCursorPosition = startReadCursorPosition; + topicTransactionBuffer.maxReadPosition = startReadCursorPosition; } ManagedCursor managedCursor; try { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 1e317d53877e3..eb659b147bfe5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -449,7 +449,7 @@ public void clearTransactionBufferSnapshotTest(Boolean enableSnapshotSegment) th abortedTxnProcessorField.setAccessible(true); AbortedTxnProcessor abortedTxnProcessor = (AbortedTxnProcessor) abortedTxnProcessorField.get(topicTransactionBuffer); - abortedTxnProcessor.takesFirstSnapshot(); + abortedTxnProcessor.takeAbortedTxnSnapshot(topicTransactionBuffer.getMaxReadPosition()); TopicName transactionBufferTopicName = NamespaceEventsSystemTopicFactory.getSystemTopicName( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 924eae54e16ae..29165516ff7c4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -669,7 +669,7 @@ public void testMaxReadPositionForNormalPublish() throws Exception { field.set(topicTransactionBuffer, TopicTransactionBufferState.State.Initializing); MessageIdImpl messageId5 = (MessageIdImpl) normalProducer.newMessage().value("normal message").send(); - PositionImpl position5 = abortedTxnProcessor.getMaxReadPosition(); + PositionImpl position5 = topicTransactionBuffer.getMaxReadPosition(); Assert.assertEquals(position5.getLedgerId(), messageId4.getLedgerId()); Assert.assertEquals(position5.getEntryId(), messageId4.getEntryId()); } @@ -949,7 +949,8 @@ public void testNoEntryCanBeReadWhenRecovery() throws Exception { Field processorField = TopicTransactionBuffer.class.getDeclaredField("snapshotAbortedTxnProcessor"); processorField.setAccessible(true); AbortedTxnProcessor abortedTxnProcessor = (AbortedTxnProcessor) processorField.get(topicTransactionBuffer); - CompletableFuture completableFuture = abortedTxnProcessor.takesFirstSnapshot(); + CompletableFuture completableFuture = abortedTxnProcessor.takeAbortedTxnSnapshot( + topicTransactionBuffer.getMaxReadPosition()); completableFuture.get(); doReturn(PositionImpl.LATEST).when(managedLedger).getLastConfirmedEntry(); From d3c99c160ad140338d2275f8d70ed465309e0e5d Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Sat, 29 Oct 2022 22:57:05 +0800 Subject: [PATCH 22/32] fix some comments --- .../impl/SingleSnapshotAbortedTxnProcessorImpl.java | 9 +-------- .../transaction/buffer/impl/TopicTransactionBuffer.java | 4 ++-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 3e082dec40759..19828c790ea73 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -73,7 +73,6 @@ public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { .getConfiguration().getTransactionBufferSnapshotMaxTransactionCount(); this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); - this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); } @Override @@ -86,7 +85,6 @@ public void trimExpiredAbortedTxns() { while (!aborts.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) .ledgerExists(aborts.get(aborts.firstKey()).getLedgerId())) { if (log.isDebugEnabled()) { - aborts.firstKey(); log.debug("[{}] Topic transaction buffer clear aborted transaction, TxnId : {}, Position : {}", topic.getName(), aborts.firstKey(), aborts.get(aborts.firstKey())); } @@ -107,13 +105,11 @@ public CompletableFuture recoverFromSnapshot() { .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { PositionImpl startReadCursorPosition = null; try { - boolean hasSnapshot = false; while (reader.hasMoreEvents()) { Message message = reader.readNext(); if (topic.getName().equals(message.getKey())) { TransactionBufferSnapshot transactionBufferSnapshot = message.getValue(); if (transactionBufferSnapshot != null) { - hasSnapshot = true; handleSnapshot(transactionBufferSnapshot); startReadCursorPosition = PositionImpl.get( transactionBufferSnapshot.getMaxReadPositionLedgerId(), @@ -122,9 +118,6 @@ public CompletableFuture recoverFromSnapshot() { } } closeReader(reader); - if (!hasSnapshot) { - return CompletableFuture.completedFuture(null); - } return CompletableFuture.completedFuture(startReadCursorPosition); } catch (Exception ex) { log.error("[{}] Transaction buffer recover fail when read " @@ -144,7 +137,7 @@ public CompletableFuture clearAndCloseAsync() { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); snapshot.setTopicName(topic.getName()); return writer.deleteAsync(snapshot.getTopicName(), snapshot); - }).thenCompose(__ -> CompletableFuture.completedFuture(null)); + }).thenRun(this::closeAsync); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 9a14bb9c0bc5b..a51d781c8cad5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -242,7 +242,7 @@ private void updateSnapshotIndexMetadataByChangeTimes() { } } - private void takeSnapshotByTimeout() { + private void scheduledTriggerSnapshot() { if (changeMaxReadPositionAndAddAbortTimes.get() > 0) { changeMaxReadPositionAndAddAbortTimes.set(0); this.snapshotAbortedTxnProcessor.takeAbortedTxnSnapshot(maxReadPosition); @@ -259,7 +259,7 @@ public void updateMaxReadPosition(Position position, boolean isUpdateChangeTimes } @Override public void run(Timeout timeout) { - takeSnapshotByTimeout(); + scheduledTriggerSnapshot(); } @Override public long getOngoingTxnCount() { From c2425b5141cb9fa704408f2ced984c17d3f8959c Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Sun, 30 Oct 2022 10:26:29 +0800 Subject: [PATCH 23/32] fix some comments --- ...SingleSnapshotAbortedTxnProcessorImpl.java | 5 --- ...napshotSegmentAbortedTxnProcessorImpl.java | 43 +++++++++++-------- .../buffer/impl/TopicTransactionBuffer.java | 1 + 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 19828c790ea73..29f978096fe94 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -43,9 +43,6 @@ public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcesso private final PersistentTopic topic; private final CompletableFuture> takeSnapshotWriter; private volatile PositionImpl maxReadPosition; - - private final Timer timer; - /** * Aborts, map for jude message is aborted, linked for remove abort txn in memory when this * position have been deleted. @@ -68,7 +65,6 @@ public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { this.takeSnapshotWriter = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName())); - this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); this.takeSnapshotIntervalNumber = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMaxTransactionCount(); this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() @@ -132,7 +128,6 @@ public CompletableFuture recoverFromSnapshot() { @Override public CompletableFuture clearAndCloseAsync() { - timer.stop(); return this.takeSnapshotWriter.thenCompose(writer -> { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); snapshot.setTopicName(topic.getName()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index ad33798e53913..3d45448e34e0e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -20,7 +20,6 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; -import io.netty.util.Timeout; import io.netty.util.Timer; import java.util.ArrayList; import java.util.List; @@ -74,8 +73,6 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess // indexes. private TransactionBufferSnapshotIndexes persistentSnapshotIndexes = new TransactionBufferSnapshotIndexes(); - private final Timer timer; - private final PersistentTopic topic; private volatile long lastSnapshotTimestamps; @@ -92,7 +89,6 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotSegmentSize(); - this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); } @Override @@ -189,10 +185,10 @@ public CompletableFuture recoverFromSnapshot() { } catch (Exception ex) { log.error("[{}] Transaction buffer recover fail when read " + "transactionBufferSnapshot!", topic.getName(), ex); - closeReader(reader); return FutureUtil.failedFuture(ex); + } finally { + closeReader(reader); } - closeReader(reader); PositionImpl finalStartReadCursorPosition = startReadCursorPosition; if (!hasIndex) { return CompletableFuture.completedFuture(null); @@ -211,6 +207,7 @@ public CompletableFuture recoverFromSnapshot() { } //Read snapshot segment to recover aborts. ArrayList> completableFutures = new ArrayList<>(); + CompletableFuture openManagedLedgerFuture = new CompletableFuture<>(); AtomicLong invalidIndex = new AtomicLong(0); AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks .OpenReadOnlyManagedLedgerCallback() { @@ -245,11 +242,13 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { } }, null); }); + openManagedLedgerFuture.complete(null); } @Override public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx) { - // + log.error("[{}] Failed to open readOnly managed ledger", topic, exception); + openManagedLedgerFuture.completeExceptionally(exception); } }; @@ -263,22 +262,28 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob null); //Wait the processor recover completely and the allow TB to recover the messages // after the startReadCursorPosition. - return FutureUtil.waitForAll(completableFutures).thenCompose((ignore) -> { - if (invalidIndex.get() != 0 ) { - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () - -> persistentWorker.updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), - indexes.values().stream().toList())); - } - return CompletableFuture.completedFuture(finalStartReadCursorPosition); + + return openManagedLedgerFuture.thenCompose((ignore) -> { + return FutureUtil.waitForAll(completableFutures).thenCompose((i) -> { + if (invalidIndex.get() != 0 ) { + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () + -> persistentWorker.updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), + indexes.values().stream().toList())); + } + return CompletableFuture.completedFuture(finalStartReadCursorPosition); + }); + }).exceptionally(ex -> { + log.error("[{}] Failed to recover snapshot segment", this.topic.getName(), ex); + return null; }); + }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() .getExecutor(this)); } @Override public CompletableFuture clearAndCloseAsync() { - timer.stop(); CompletableFuture completableFuture = new CompletableFuture<>(); persistentWorker.appendTask(PersistentWorker.OperationType.Close, () -> persistentWorker.clearSnapshotSegmentAndIndexes() @@ -447,7 +452,7 @@ private void executeTask() { } } - protected CompletableFuture takeSnapshotSegmentAsync(ArrayList sealedAbortedTxnIdSegment, + private CompletableFuture takeSnapshotSegmentAsync(ArrayList sealedAbortedTxnIdSegment, PositionImpl maxReadPosition) { return writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, maxReadPosition).thenRun(() -> { if (log.isDebugEnabled()) { @@ -545,7 +550,7 @@ private CompletableFuture updateIndexMetadataForTheLastSnapshot(PositionIm return updateSnapshotIndex(metadata, persistentSnapshotIndexes.getIndexList()); } - protected CompletableFuture clearSnapshotSegmentAndIndexes() { + private CompletableFuture clearSnapshotSegmentAndIndexes() { ArrayList> completableFutures = new ArrayList<>(); //Delete all segment while (!abortTxnSegments.isEmpty()) { @@ -581,7 +586,7 @@ CompletableFuture closeAsync() { } } - ArrayList deserializationFotSnapshotSegment(List snapshotSegment) { + private ArrayList deserializationFotSnapshotSegment(List snapshotSegment) { ArrayList arrayList = new ArrayList<>(); snapshotSegment.forEach(txnIDData -> { arrayList.add(new TxnID(txnIDData.getMostSigBits(), txnIDData.getLeastSigBits())); @@ -589,7 +594,7 @@ ArrayList deserializationFotSnapshotSegment(List snapshotSegme return arrayList; } - ArrayList serializationForSegment(List segment) { + private ArrayList serializationForSegment(List segment) { ArrayList arrayList = new ArrayList<>(); segment.forEach(txnID -> { arrayList.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits())); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index a51d781c8cad5..0d0bc2f96d9aa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -474,6 +474,7 @@ public CompletableFuture clearSnapshot() { @Override public CompletableFuture closeAsync() { + this.timer.stop(); changeToCloseState(); return this.snapshotAbortedTxnProcessor.closeAsync(); } From 5191c3dcd26ebbb1546955a0fa1141abc006b151 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Sun, 30 Oct 2022 21:14:25 +0800 Subject: [PATCH 24/32] fix some comments --- .../buffer/AbortedTxnProcessor.java | 19 ++--- ...SingleSnapshotAbortedTxnProcessorImpl.java | 73 ++++++++----------- ...napshotSegmentAbortedTxnProcessorImpl.java | 45 ++++++------ .../buffer/impl/TopicTransactionBuffer.java | 6 +- ...nsactionBufferSnapshotIndexesMetadata.java | 4 +- .../v2/TransactionBufferSnapshotSegment.java | 4 +- .../TopicTransactionBufferRecoverTest.java | 9 ++- 7 files changed, 73 insertions(+), 87 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index 74b05952b7f43..cb05b7e2d7782 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.transaction.buffer; -import io.netty.util.TimerTask; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; @@ -28,17 +27,15 @@ public interface AbortedTxnProcessor { /** - * After the transaction buffer writes a transaction aborted mark to the topic, - * the transaction buffer will add the aborted transaction ID to AbortedTxnProcessor. + * After the transaction buffer writes a transaction aborted marker to the topic, + * the transaction buffer will put the aborted txnID and the aborted marker position to AbortedTxnProcessor. * @param txnID aborted transaction ID. + * @param position the position of the abort txnID */ - void appendAbortedTxn(TxnID txnID, PositionImpl position); + void putAbortedTxnAndPosition(TxnID txnID, PositionImpl position); /** - * Pulsar has a configuration for ledger retention time. - * If the transaction aborted mark position has been deleted, the transaction is valid and can be clear. - * In the old implementation we clear the invalid aborted txn ID one by one. - * In the new implementation, we adopt snapshot segments. And then we clear invalid segment by its max read position. + * Clean up invalid aborted transactions. */ void trimExpiredAbortedTxns(); @@ -52,16 +49,16 @@ public interface AbortedTxnProcessor { /** * Recover transaction buffer by transaction buffer snapshot. - * @return a pair consists of a Boolean if the transaction buffer needs to recover and a Position (startReadCursorPosition) determiner where to start to recover in the original topic. + * @return a Position (startReadCursorPosition) determiner where to start to recover in the original topic. */ CompletableFuture recoverFromSnapshot(); /** - * Clear the snapshot/snapshot segment and index for this topic. + * Delete the transaction buffer aborted transaction snapshot. * @return a completableFuture. */ - CompletableFuture clearAndCloseAsync(); + CompletableFuture deleteAbortedTxnSnapshot(); /** * Take the frist snapshot if the topic has no snapshot before. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 29f978096fe94..b56af1a9ff8e7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -42,7 +42,6 @@ public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcessor { private final PersistentTopic topic; private final CompletableFuture> takeSnapshotWriter; - private volatile PositionImpl maxReadPosition; /** * Aborts, map for jude message is aborted, linked for remove abort txn in memory when this * position have been deleted. @@ -54,14 +53,8 @@ public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcesso private final int takeSnapshotIntervalTime; - - // when add abort or change max read position, the count will +1. Take snapshot will set 0 into it. - private final AtomicLong changeMaxReadPositionAndAddAbortTimes = new AtomicLong(); - - public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; - this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); this.takeSnapshotWriter = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName())); @@ -72,10 +65,11 @@ public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { } @Override - public void appendAbortedTxn(TxnID abortedTxnId, PositionImpl position) { + public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl position) { aborts.put(abortedTxnId, position); } + //In this implementation we clear the invalid aborted txn ID one by one. @Override public void trimExpiredAbortedTxns() { while (!aborts.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) @@ -127,7 +121,7 @@ public CompletableFuture recoverFromSnapshot() { } @Override - public CompletableFuture clearAndCloseAsync() { + public CompletableFuture deleteAbortedTxnSnapshot() { return this.takeSnapshotWriter.thenCompose(writer -> { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); snapshot.setTopicName(topic.getName()); @@ -137,7 +131,32 @@ public CompletableFuture clearAndCloseAsync() { @Override public CompletableFuture takeAbortedTxnSnapshot(PositionImpl maxReadPosition) { - return takeAbortedTxnSnapshot(maxReadPosition, aborts); + return takeSnapshotWriter.thenCompose(writer -> { + TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); + snapshot.setTopicName(topic.getName()); + snapshot.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); + snapshot.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); + List list = new ArrayList<>(); + aborts.forEach((k, v) -> { + AbortTxnMetadata abortTxnMetadata = new AbortTxnMetadata(); + abortTxnMetadata.setTxnIdMostBits(k.getMostSigBits()); + abortTxnMetadata.setTxnIdLeastBits(k.getLeastSigBits()); + abortTxnMetadata.setLedgerId(v.getLedgerId()); + abortTxnMetadata.setEntryId(v.getEntryId()); + list.add(abortTxnMetadata); + }); + snapshot.setAborts(list); + return writer.writeAsync(snapshot.getTopicName(), snapshot).thenAccept(messageId -> { + this.lastSnapshotTimestamps = System.currentTimeMillis(); + if (log.isDebugEnabled()) { + log.debug("[{}]Transaction buffer take snapshot success! " + + "messageId : {}", topic.getName(), messageId); + } + }).exceptionally(e -> { + log.warn("[{}]Transaction buffer take snapshot fail! ", topic.getName(), e.getCause()); + return null; + }); + }); } @Override @@ -158,8 +177,6 @@ private void closeReader(SystemTopicClient.Reader rea } private void handleSnapshot(TransactionBufferSnapshot snapshot) { - maxReadPosition = PositionImpl.get(snapshot.getMaxReadPositionLedgerId(), - snapshot.getMaxReadPositionEntryId()); if (snapshot.getAborts() != null) { snapshot.getAborts().forEach(abortTxnMetadata -> aborts.put(new TxnID(abortTxnMetadata.getTxnIdMostBits(), @@ -169,36 +186,4 @@ private void handleSnapshot(TransactionBufferSnapshot snapshot) { } } - private CompletableFuture takeAbortedTxnSnapshot(PositionImpl maxReadPosition, LinkedMap aborts) { - changeMaxReadPositionAndAddAbortTimes.set(0); - return takeSnapshotWriter.thenCompose(writer -> { - TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); - synchronized (topic) { - snapshot.setTopicName(topic.getName()); - snapshot.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); - snapshot.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); - List list = new ArrayList<>(); - aborts.forEach((k, v) -> { - AbortTxnMetadata abortTxnMetadata = new AbortTxnMetadata(); - abortTxnMetadata.setTxnIdMostBits(k.getMostSigBits()); - abortTxnMetadata.setTxnIdLeastBits(k.getLeastSigBits()); - abortTxnMetadata.setLedgerId(v.getLedgerId()); - abortTxnMetadata.setEntryId(v.getEntryId()); - list.add(abortTxnMetadata); - }); - snapshot.setAborts(list); - } - return writer.writeAsync(snapshot.getTopicName(), snapshot).thenAccept(messageId -> { - this.lastSnapshotTimestamps = System.currentTimeMillis(); - if (log.isDebugEnabled()) { - log.debug("[{}]Transaction buffer take snapshot success! " - + "messageId : {}", topic.getName(), messageId); - } - }).exceptionally(e -> { - log.warn("[{}]Transaction buffer take snapshot fail! ", topic.getName(), e); - return null; - }); - }); - } - } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 3d45448e34e0e..fa7738ded8756 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -22,7 +22,9 @@ import io.netty.buffer.Unpooled; import io.netty.util.Timer; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentSkipListMap; @@ -57,14 +59,15 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; @Slf4j public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { - private ArrayList unsealedAbortedTxnIdSegment = new ArrayList<>(); + private ConcurrentOpenHashSet unsealedAbortedTxnIdSegment = new ConcurrentOpenHashSet<>(); //Store the fixed aborted transaction segment - private final ConcurrentSkipListMap> abortTxnSegments + private final ConcurrentSkipListMap> abortTxnSegments = new ConcurrentSkipListMap<>(); private final ConcurrentSkipListMap indexes @@ -92,14 +95,14 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { } @Override - public void appendAbortedTxn(TxnID abortedTxnId, PositionImpl maxReadPosition) { + public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl maxReadPosition) { unsealedAbortedTxnIdSegment.add(abortedTxnId); //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { abortTxnSegments.put(maxReadPosition, unsealedAbortedTxnIdSegment); persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> persistentWorker.takeSnapshotSegmentAsync(unsealedAbortedTxnIdSegment, maxReadPosition)); - unsealedAbortedTxnIdSegment = new ArrayList<>(); + unsealedAbortedTxnIdSegment = new ConcurrentOpenHashSet<>(); } } @@ -123,6 +126,7 @@ public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { } } + //In this implementation, we adopt snapshot segments. And then we clear invalid segment by its max read position. @Override public void trimExpiredAbortedTxns() { //Checking whether there are some segment expired. @@ -144,10 +148,7 @@ private String buildKey(long sequenceId) { @Override public CompletableFuture takeAbortedTxnSnapshot(PositionImpl maxReadPosition) { - return takeAbortedTxnSnapshot(maxReadPosition, unsealedAbortedTxnIdSegment); - } - - private CompletableFuture takeAbortedTxnSnapshot(PositionImpl maxReadPosition, ArrayList aborts) { + ConcurrentOpenHashSet aborts = unsealedAbortedTxnIdSegment; CompletableFuture completableFuture = new CompletableFuture<>(); persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> persistentWorker @@ -214,7 +215,6 @@ public CompletableFuture recoverFromSnapshot() { @Override public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { persistentSnapshotIndexes.getIndexList().forEach(index -> { - //TODO: read on demand CompletableFuture handleSegmentFuture = new CompletableFuture<>(); completableFutures.add(handleSegmentFuture); readOnlyManagedLedger.asyncReadEntry( @@ -283,7 +283,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob } @Override - public CompletableFuture clearAndCloseAsync() { + public CompletableFuture deleteAbortedTxnSnapshot() { CompletableFuture completableFuture = new CompletableFuture<>(); persistentWorker.appendTask(PersistentWorker.OperationType.Close, () -> persistentWorker.clearSnapshotSegmentAndIndexes() @@ -452,7 +452,7 @@ private void executeTask() { } } - private CompletableFuture takeSnapshotSegmentAsync(ArrayList sealedAbortedTxnIdSegment, + private CompletableFuture takeSnapshotSegmentAsync(ConcurrentOpenHashSet sealedAbortedTxnIdSegment, PositionImpl maxReadPosition) { return writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, maxReadPosition).thenRun(() -> { if (log.isDebugEnabled()) { @@ -471,7 +471,8 @@ private CompletableFuture takeSnapshotSegmentAsync(ArrayList sealed }); } - private CompletableFuture writeSnapshotSegmentAsync(List segment, PositionImpl maxReadPosition) { + private CompletableFuture writeSnapshotSegmentAsync(ConcurrentOpenHashSet segment, + PositionImpl maxReadPosition) { TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = new TransactionBufferSnapshotSegment(); transactionBufferSnapshotSegment.setAborts(serializationForSegment(segment)); transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); @@ -493,7 +494,7 @@ private CompletableFuture writeSnapshotSegmentAsync(List segment, P indexes.put(maxReadPosition, index); //update snapshot segment index. return updateSnapshotIndex(new TransactionBufferSnapshotIndexesMetadata( - maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), new ArrayList<>()), + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), new HashSet<>()), indexes.values().stream().toList()); }); } @@ -543,7 +544,7 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd //Only update the metadata in the transactionBufferSnapshotIndexes. private CompletableFuture updateIndexMetadataForTheLastSnapshot(PositionImpl maxReadPosition, - ArrayList abortedTxns) { + ConcurrentOpenHashSet abortedTxns) { TransactionBufferSnapshotIndexesMetadata metadata = new TransactionBufferSnapshotIndexesMetadata( maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), serializationForSegment(abortedTxns)); @@ -586,20 +587,20 @@ CompletableFuture closeAsync() { } } - private ArrayList deserializationFotSnapshotSegment(List snapshotSegment) { - ArrayList arrayList = new ArrayList<>(); + private ConcurrentOpenHashSet deserializationFotSnapshotSegment(Set snapshotSegment) { + ConcurrentOpenHashSet set = new ConcurrentOpenHashSet<>(); snapshotSegment.forEach(txnIDData -> { - arrayList.add(new TxnID(txnIDData.getMostSigBits(), txnIDData.getLeastSigBits())); + set.add(new TxnID(txnIDData.getMostSigBits(), txnIDData.getLeastSigBits())); }); - return arrayList; + return set; } - private ArrayList serializationForSegment(List segment) { - ArrayList arrayList = new ArrayList<>(); + private Set serializationForSegment(ConcurrentOpenHashSet segment) { + Set set = new HashSet<>(); segment.forEach(txnID -> { - arrayList.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits())); + set.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits())); }); - return arrayList; + return set; } } \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 0d0bc2f96d9aa..579dede423322 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -168,7 +168,7 @@ public void handleTxnEntry(Entry entry) { PositionImpl position = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); if (Markers.isTxnMarker(msgMetadata)) { if (Markers.isTxnAbortMarker(msgMetadata)) { - snapshotAbortedTxnProcessor.appendAbortedTxn(txnID, position); + snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, position); } updateMaxReadPosition(getMaxReadPosition(txnID), true); } else { @@ -381,7 +381,7 @@ public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (topic) { PositionImpl maxReadPosition = getMaxReadPosition(txnID); - snapshotAbortedTxnProcessor.appendAbortedTxn(txnID, maxReadPosition); + snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, maxReadPosition); updateMaxReadPosition(maxReadPosition, true); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); } @@ -468,7 +468,7 @@ public CompletableFuture purgeTxns(List dataLedgers) { @Override public CompletableFuture clearSnapshot() { - return snapshotAbortedTxnProcessor.clearAndCloseAsync(); + return snapshotAbortedTxnProcessor.deleteAbortedTxnSnapshot(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java index e5194e2ab5b6e..59fbea0e35769 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.broker.transaction.buffer.metadata.v2; -import java.util.List; +import java.util.Set; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -29,5 +29,5 @@ public class TransactionBufferSnapshotIndexesMetadata { private long maxReadPositionLedgerId; private long maxReadPositionEntryId; - private List aborts; + private Set aborts; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java index 478ec53ba293a..9c0bbd22abdbf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.broker.transaction.buffer.metadata.v2; -import java.util.List; +import java.util.Set; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -31,5 +31,5 @@ public class TransactionBufferSnapshotSegment { private long sequenceId; private long maxReadPositionLedgerId; private long maxReadPositionEntryId; - private List aborts; + private Set aborts; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index eb659b147bfe5..876e30a7cee1b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -32,11 +32,13 @@ import io.netty.buffer.Unpooled; import java.lang.reflect.Field; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.NavigableMap; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; @@ -688,8 +690,9 @@ public void testTransactionBufferSegmentSystemTopic() throws Exception { snapshot.setSequenceId(1L); snapshot.setMaxReadPositionLedgerId(2L); snapshot.setMaxReadPositionEntryId(3L); - snapshot.setAborts(Collections.singletonList( - new TxnIDData(1, 1))); + HashSet txnIDSet = new HashSet<>(); + txnIDSet.add(new TxnIDData(1, 1)); + snapshot.setAborts(txnIDSet ); segmentWriter.write(buildKey(snapshot), snapshot); snapshot.setSequenceId(2L); @@ -739,7 +742,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob assertEquals(snapshot.getSequenceId(), 2L); assertEquals(snapshot.getMaxReadPositionLedgerId(), 2L); assertEquals(snapshot.getMaxReadPositionEntryId(), 3L); - assertEquals(snapshot.getAborts().get(0), new TxnIDData(1, 1)); + assertEquals(snapshot.getAborts().toArray()[0], new TxnIDData(1, 1)); } @Test From ba9c3746da8cf866fdfd316d5e655391ce95d49c Mon Sep 17 00:00:00 2001 From: congbobo184 Date: Mon, 31 Oct 2022 19:03:58 +0800 Subject: [PATCH 25/32] topicTransactionBuffer changed --- .../buffer/impl/TopicTransactionBuffer.java | 121 +++++++++--------- 1 file changed, 60 insertions(+), 61 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 579dede423322..a00e09db1308e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -23,6 +23,7 @@ import io.netty.util.Timeout; import io.netty.util.Timer; import io.netty.util.TimerTask; + import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -68,25 +69,24 @@ public class TopicTransactionBuffer extends TopicTransactionBufferState implemen private final PersistentTopic topic; + private volatile PositionImpl maxReadPosition; + /** * Ongoing transaction, map for remove txn stable position, linked for find max read position. */ private final LinkedMap ongoingTxns = new LinkedMap<>(); - //Store the latest aborted transaction IDs and the latest max read position. - private PositionImpl maxReadPosition; - - //When add abort or change max read position, the count will +1. Take snapshot will set 0 into it. + // when add abort or change max read position, the count will +1. Take snapshot will set 0 into it. private final AtomicLong changeMaxReadPositionAndAddAbortTimes = new AtomicLong(); - private final int takeSnapshotIntervalNumber; - private final LongAdder txnCommittedCounter = new LongAdder(); private final LongAdder txnAbortedCounter = new LongAdder(); private final Timer timer; + private final int takeSnapshotIntervalNumber; + private final int takeSnapshotIntervalTime; private final CompletableFuture transactionBufferFuture = new CompletableFuture<>(); @@ -106,10 +106,11 @@ public TopicTransactionBuffer(PersistentTopic topic) { super(State.None); this.topic = topic; this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); - this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() - .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); this.takeSnapshotIntervalNumber = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMaxTransactionCount(); + this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() + .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); + this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); if (topic.getBrokerService().getPulsar().getConfiguration().isTransactionBufferSegmentedSnapshotEnabled()) { snapshotAbortedTxnProcessor = new SnapshotSegmentAbortedTxnProcessorImpl(topic); } else { @@ -124,9 +125,9 @@ private void recover() { .execute(new TopicTransactionBufferRecover(new TopicTransactionBufferRecoverCallBack() { @Override public void recoverComplete() { - synchronized (topic) { + synchronized (TopicTransactionBuffer.this) { if (ongoingTxns.isEmpty()) { - updateMaxReadPosition(topic.getManagedLedger().getLastConfirmedEntry(), false); + maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); } if (!changeToReadyState()) { log.error("[{}]Transaction buffer recover fail, current state: {}", @@ -146,9 +147,8 @@ public void recoverComplete() { @Override public void noNeedToRecover() { - synchronized (topic) { - updateMaxReadPosition(topic.getManagedLedger().getLastConfirmedEntry(), false); - + synchronized (TopicTransactionBuffer.this) { + maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); if (!changeToNoSnapshotState()) { log.error("[{}]Transaction buffer recover fail", topic.getName()); } else { @@ -170,7 +170,7 @@ public void handleTxnEntry(Entry entry) { if (Markers.isTxnAbortMarker(msgMetadata)) { snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, position); } - updateMaxReadPosition(getMaxReadPosition(txnID), true); + updateMaxReadPosition(txnID); } else { handleTransactionMessage(txnID, position); } @@ -214,8 +214,8 @@ public CompletableFuture checkIfTBRecoverCompletely(boolean isTxnEnabled) if (checkIfNoSnapshot()) { snapshotAbortedTxnProcessor.takeAbortedTxnSnapshot(maxReadPosition).thenRun(() -> { if (changeToReadyStateFromNoSnapshot()) { - timer.newTimeout(TopicTransactionBuffer.this, takeSnapshotIntervalTime, - TimeUnit.MILLISECONDS); + timer.newTimeout(TopicTransactionBuffer.this, + takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); } completableFuture.complete(null); }).exceptionally(exception -> { @@ -235,32 +235,6 @@ public CompletableFuture checkIfTBRecoverCompletely(boolean isTxnEnabled) } } - private void updateSnapshotIndexMetadataByChangeTimes() { - if (this.changeMaxReadPositionAndAddAbortTimes.incrementAndGet() == takeSnapshotIntervalNumber) { - changeMaxReadPositionAndAddAbortTimes.set(0); - this.snapshotAbortedTxnProcessor.takeAbortedTxnSnapshot(maxReadPosition); - } - } - - private void scheduledTriggerSnapshot() { - if (changeMaxReadPositionAndAddAbortTimes.get() > 0) { - changeMaxReadPositionAndAddAbortTimes.set(0); - this.snapshotAbortedTxnProcessor.takeAbortedTxnSnapshot(maxReadPosition); - } - timer.newTimeout(TopicTransactionBuffer.this, - takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); - } - - public void updateMaxReadPosition(Position position, boolean isUpdateChangeTimes) { - if (position != this.maxReadPosition && isUpdateChangeTimes) { - updateSnapshotIndexMetadataByChangeTimes(); - } - this.maxReadPosition = (PositionImpl) position; - } - @Override - public void run(Timeout timeout) { - scheduledTriggerSnapshot(); - } @Override public long getOngoingTxnCount() { return this.ongoingTxns.size(); @@ -289,7 +263,7 @@ public CompletableFuture appendBufferToTxn(TxnID txnId, long sequenceI topic.getManagedLedger().asyncAddEntry(buffer, new AsyncCallbacks.AddEntryCallback() { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { - synchronized (topic) { + synchronized (TopicTransactionBuffer.this) { handleTransactionMessage(txnId, position); } completableFuture.complete(position); @@ -305,13 +279,12 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { } private void handleTransactionMessage(TxnID txnId, Position position) { - if (!ongoingTxns.containsKey(txnId) && !this.snapshotAbortedTxnProcessor.checkAbortedTransaction( - txnId, position)) { + if (!ongoingTxns.containsKey(txnId) && !this.snapshotAbortedTxnProcessor + .checkAbortedTransaction(txnId, position)) { ongoingTxns.put(txnId, (PositionImpl) position); PositionImpl firstPosition = ongoingTxns.get(ongoingTxns.firstKey()); //max read position is less than first ongoing transaction message position, so entryId -1 - updateMaxReadPosition(PositionImpl.get(firstPosition.getLedgerId(), firstPosition.getEntryId() - 1), - true); + maxReadPosition = PositionImpl.get(firstPosition.getLedgerId(), firstPosition.getEntryId() - 1); } } @@ -335,9 +308,10 @@ public CompletableFuture commitTxn(TxnID txnID, long lowWaterMark) { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { - updateMaxReadPosition(getMaxReadPosition(txnID), true); + updateMaxReadPosition(txnID); handleLowWaterMark(txnID, lowWaterMark); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); + takeSnapshotByChangeTimes(); } txnCommittedCounter.increment(); completableFuture.complete(null); @@ -379,11 +353,11 @@ public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { topic.getManagedLedger().asyncAddEntry(abortMarker, new AsyncCallbacks.AddEntryCallback() { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { - synchronized (topic) { - PositionImpl maxReadPosition = getMaxReadPosition(txnID); + synchronized (TopicTransactionBuffer.this) { + updateMaxReadPosition(txnID); snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, maxReadPosition); - updateMaxReadPosition(maxReadPosition, true); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); + takeSnapshotByChangeTimes(); } txnAbortedCounter.increment(); completableFuture.complete(null); @@ -448,9 +422,25 @@ private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { } } - PositionImpl getMaxReadPosition(TxnID txnID) { + private void takeSnapshotByChangeTimes() { + if (changeMaxReadPositionAndAddAbortTimes.get() >= takeSnapshotIntervalNumber) { + this.changeMaxReadPositionAndAddAbortTimes.set(0); + this.snapshotAbortedTxnProcessor.takeAbortedTxnSnapshot(this.maxReadPosition); + } + } + + private void takeSnapshotByTimeout() { + if (changeMaxReadPositionAndAddAbortTimes.get() > 0) { + this.changeMaxReadPositionAndAddAbortTimes.set(0); + this.snapshotAbortedTxnProcessor.takeAbortedTxnSnapshot(this.maxReadPosition); + } + this.timer.newTimeout(TopicTransactionBuffer.this, + takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); + } + + void updateMaxReadPosition(TxnID txnID) { + PositionImpl preMaxReadPosition = this.maxReadPosition; ongoingTxns.remove(txnID); - PositionImpl maxReadPosition; if (!ongoingTxns.isEmpty()) { PositionImpl position = ongoingTxns.get(ongoingTxns.firstKey()); //max read position is less than first ongoing transaction message position, so entryId -1 @@ -458,7 +448,9 @@ PositionImpl getMaxReadPosition(TxnID txnID) { } else { maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); } - return maxReadPosition; + if (preMaxReadPosition.compareTo(this.maxReadPosition) != 0) { + this.changeMaxReadPositionAndAddAbortTimes.getAndIncrement(); + } } @Override @@ -469,12 +461,10 @@ public CompletableFuture purgeTxns(List dataLedgers) { @Override public CompletableFuture clearSnapshot() { return snapshotAbortedTxnProcessor.deleteAbortedTxnSnapshot(); - } @Override public CompletableFuture closeAsync() { - this.timer.stop(); changeToCloseState(); return this.snapshotAbortedTxnProcessor.closeAsync(); } @@ -488,12 +478,13 @@ public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { public void syncMaxReadPositionForNormalPublish(PositionImpl position) { // when ongoing transaction is empty, proved that lastAddConfirm is can read max position, because callback // thread is the same tread, in this time the lastAddConfirm don't content transaction message. - synchronized (topic) { + synchronized (TopicTransactionBuffer.this) { if (checkIfNoSnapshot()) { - updateMaxReadPosition(position, false); + this.maxReadPosition = position; } else if (checkIfReady()) { if (ongoingTxns.isEmpty()) { - updateMaxReadPosition(position, true); + maxReadPosition = position; + changeMaxReadPositionAndAddAbortTimes.incrementAndGet(); } } } @@ -534,6 +525,15 @@ public TransactionBufferStats getStats(boolean lowWaterMarks) { return transactionBufferStats; } + @Override + public void run(Timeout timeout) { + if (checkIfReady()) { + synchronized (TopicTransactionBuffer.this) { + takeSnapshotByTimeout(); + } + } + } + // we store the maxReadPosition from snapshot then open the non-durable cursor by this topic's manageLedger. // the non-durable cursor will read to lastConfirmedEntry. @VisibleForTesting @@ -574,13 +574,12 @@ public void run() { return; } abortedTxnProcessor.recoverFromSnapshot().thenAcceptAsync(startReadCursorPosition -> { - //Transaction is not enable for this topic, so just make maxReadPosition as LAC. + //Transaction is not use for this topic, so just make maxReadPosition as LAC. if (startReadCursorPosition == null) { callBack.noNeedToRecover(); return; } else { this.startReadCursorPosition = startReadCursorPosition; - topicTransactionBuffer.maxReadPosition = startReadCursorPosition; } ManagedCursor managedCursor; try { From 765795c333fde5e1668fee0130979526d74ac6c5 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Mon, 31 Oct 2022 19:52:02 +0800 Subject: [PATCH 26/32] fix --- .../impl/SingleSnapshotAbortedTxnProcessorImpl.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index b56af1a9ff8e7..b23fab6592987 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -49,19 +49,12 @@ public class SingleSnapshotAbortedTxnProcessorImpl implements AbortedTxnProcesso private final LinkedMap aborts = new LinkedMap<>(); private volatile long lastSnapshotTimestamps; - private final int takeSnapshotIntervalNumber; - - private final int takeSnapshotIntervalTime; public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; this.takeSnapshotWriter = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName())); - this.takeSnapshotIntervalNumber = topic.getBrokerService().getPulsar() - .getConfiguration().getTransactionBufferSnapshotMaxTransactionCount(); - this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() - .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); } @Override @@ -126,7 +119,9 @@ public CompletableFuture deleteAbortedTxnSnapshot() { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); snapshot.setTopicName(topic.getName()); return writer.deleteAsync(snapshot.getTopicName(), snapshot); - }).thenRun(this::closeAsync); + }).thenRun(() -> { + log.info("[{}] Successes to delete the aborted transaction snapshot", this.topic); + }); } @Override From 2ad68112018786346c9ef8a34a5ab77abe20c218 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Mon, 31 Oct 2022 20:10:35 +0800 Subject: [PATCH 27/32] delete SnapshotSegmentAbortedTxnProcessorImpl.java --- ...napshotSegmentAbortedTxnProcessorImpl.java | 606 ------------------ .../buffer/impl/TopicTransactionBuffer.java | 6 +- .../TopicTransactionBufferRecoverTest.java | 79 +-- .../broker/transaction/TransactionTest.java | 13 +- 4 files changed, 8 insertions(+), 696 deletions(-) delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java deleted file mode 100644 index fa7738ded8756..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ /dev/null @@ -1,606 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing,2 - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.transaction.buffer.impl; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.util.Timer; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import java.util.function.Supplier; -import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.AsyncCallbacks; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.PositionImpl; -import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; -import org.apache.commons.lang3.tuple.MutablePair; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.broker.service.persistent.PersistentTopic; -import org.apache.pulsar.broker.systopic.SystemTopicClient; -import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; -import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex; -import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; -import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexesMetadata; -import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; -import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; -import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.transaction.TxnID; -import org.apache.pulsar.client.impl.MessageIdImpl; -import org.apache.pulsar.common.events.EventType; -import org.apache.pulsar.common.naming.TopicDomain; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; - -@Slf4j -public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { - - private ConcurrentOpenHashSet unsealedAbortedTxnIdSegment = new ConcurrentOpenHashSet<>(); - - //Store the fixed aborted transaction segment - private final ConcurrentSkipListMap> abortTxnSegments - = new ConcurrentSkipListMap<>(); - - private final ConcurrentSkipListMap indexes - = new ConcurrentSkipListMap<>(); - //The latest persistent snapshot index. This is used to combine new segment indexes with the latest metadata and - // indexes. - private TransactionBufferSnapshotIndexes persistentSnapshotIndexes = new TransactionBufferSnapshotIndexes(); - - private final PersistentTopic topic; - - private volatile long lastSnapshotTimestamps; - - private final int takeSnapshotIntervalTime; - - private final int transactionBufferMaxAbortedTxnsOfSnapshotSegment; - private final PersistentWorker persistentWorker; - - public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { - this.topic = topic; - this.persistentWorker = new PersistentWorker(topic); - this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() - .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); - this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = topic.getBrokerService().getPulsar() - .getConfiguration().getTransactionBufferSnapshotSegmentSize(); - } - - @Override - public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl maxReadPosition) { - unsealedAbortedTxnIdSegment.add(abortedTxnId); - //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. - if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { - abortTxnSegments.put(maxReadPosition, unsealedAbortedTxnIdSegment); - persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> - persistentWorker.takeSnapshotSegmentAsync(unsealedAbortedTxnIdSegment, maxReadPosition)); - unsealedAbortedTxnIdSegment = new ConcurrentOpenHashSet<>(); - } - } - - @Override - public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { - if (readPosition == null) { - return abortTxnSegments.values().stream() - .anyMatch(list -> list.contains(txnID)) || unsealedAbortedTxnIdSegment.contains(txnID); - } else { - PositionImpl maxReadPosition = abortTxnSegments.ceilingKey((PositionImpl) readPosition); - if (maxReadPosition != null) { - return abortTxnSegments.keySet().stream() - .filter((position) -> position.compareTo(maxReadPosition) <= 0) - .anyMatch((position -> abortTxnSegments.get(position).contains(txnID))); - } else { - return abortTxnSegments.keySet().stream() - .filter((position) -> position.compareTo((PositionImpl) readPosition) <= 0) - .anyMatch((position -> abortTxnSegments.get(position).contains(txnID))) - || unsealedAbortedTxnIdSegment.contains(txnID); - } - } - } - - //In this implementation, we adopt snapshot segments. And then we clear invalid segment by its max read position. - @Override - public void trimExpiredAbortedTxns() { - //Checking whether there are some segment expired. - while (!abortTxnSegments.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) - .ledgerExists(abortTxnSegments.firstKey().getLedgerId())) { - if (log.isDebugEnabled()) { - log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", - topic.getName(), abortTxnSegments.firstKey()); - } - PositionImpl positionNeedToDelete = abortTxnSegments.firstKey(); - persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment, - () -> persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); - } - } - - private String buildKey(long sequenceId) { - return "multiple-" + sequenceId + "-" + this.topic.getName(); - } - - @Override - public CompletableFuture takeAbortedTxnSnapshot(PositionImpl maxReadPosition) { - ConcurrentOpenHashSet aborts = unsealedAbortedTxnIdSegment; - CompletableFuture completableFuture = new CompletableFuture<>(); - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, - () -> persistentWorker - .updateIndexMetadataForTheLastSnapshot(maxReadPosition, aborts) - .thenRun(() -> completableFuture.complete(null)) - .exceptionally(e -> { - completableFuture.completeExceptionally(e); - return null; - })); - return completableFuture; - } - - @Override - public CompletableFuture recoverFromSnapshot() { - return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotIndexService() - .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { - PositionImpl startReadCursorPosition = null; - boolean hasIndex = false; - try { - //Read Index to recover the sequenceID, indexes, lastAbortedTxns and maxReadPosition. - while (reader.hasMoreEvents()) { - Message message = reader.readNext(); - if (topic.getName().equals(message.getKey())) { - TransactionBufferSnapshotIndexes transactionBufferSnapshotIndexes = message.getValue(); - if (transactionBufferSnapshotIndexes != null) { - hasIndex = true; - this.persistentSnapshotIndexes = transactionBufferSnapshotIndexes; - startReadCursorPosition = PositionImpl.get( - transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionLedgerId(), - transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); - } - } - } - } catch (Exception ex) { - log.error("[{}] Transaction buffer recover fail when read " - + "transactionBufferSnapshot!", topic.getName(), ex); - return FutureUtil.failedFuture(ex); - } finally { - closeReader(reader); - } - PositionImpl finalStartReadCursorPosition = startReadCursorPosition; - if (!hasIndex) { - return CompletableFuture.completedFuture(null); - } else { - persistentSnapshotIndexes.getIndexList() - .forEach(transactionBufferSnapshotIndex -> - indexes.put(new PositionImpl( - transactionBufferSnapshotIndex.persistentPositionLedgerID, - transactionBufferSnapshotIndex.persistentPositionEntryID), - transactionBufferSnapshotIndex)); - this.unsealedAbortedTxnIdSegment = deserializationFotSnapshotSegment(persistentSnapshotIndexes - .getSnapshot().getAborts()); - if (indexes.size() != 0) { - persistentWorker.sequenceID.set(indexes.lastEntry().getValue().sequenceID + 1); - } - } - //Read snapshot segment to recover aborts. - ArrayList> completableFutures = new ArrayList<>(); - CompletableFuture openManagedLedgerFuture = new CompletableFuture<>(); - AtomicLong invalidIndex = new AtomicLong(0); - AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks - .OpenReadOnlyManagedLedgerCallback() { - @Override - public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { - persistentSnapshotIndexes.getIndexList().forEach(index -> { - CompletableFuture handleSegmentFuture = new CompletableFuture<>(); - completableFutures.add(handleSegmentFuture); - readOnlyManagedLedger.asyncReadEntry( - new PositionImpl(index.getPersistentPositionLedgerID(), - index.getPersistentPositionEntryID()), - new AsyncCallbacks.ReadEntryCallback() { - @Override - public void readEntryComplete(Entry entry, Object ctx) { - //Remove invalid index - if (entry == null) { - indexes.remove(new PositionImpl( - index.getMaxReadPositionLedgerID(), - index.getMaxReadPositionEntryID())); - handleSegmentFuture.complete(null); - invalidIndex.getAndIncrement(); - return; - } - handleSnapshotSegmentEntry(entry); - handleSegmentFuture.complete(null); - } - - @Override - public void readEntryFailed(ManagedLedgerException exception, Object ctx) { - handleSegmentFuture.completeExceptionally(exception); - } - }, null); - }); - openManagedLedgerFuture.complete(null); - } - - @Override - public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx) { - log.error("[{}] Failed to open readOnly managed ledger", topic, exception); - openManagedLedgerFuture.completeExceptionally(exception); - } - }; - - TopicName snapshotIndexTopicName = TopicName.get(TopicDomain.persistent.toString(), - TopicName.get(topic.getName()).getNamespaceObject(), - EventType.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS.toString()); - this.topic.getBrokerService().getPulsar().getManagedLedgerFactory() - .asyncOpenReadOnlyManagedLedger(snapshotIndexTopicName - .getPersistenceNamingEncoding(), callback, - topic.getManagedLedger().getConfig(), - null); - //Wait the processor recover completely and the allow TB to recover the messages - // after the startReadCursorPosition. - - return openManagedLedgerFuture.thenCompose((ignore) -> { - return FutureUtil.waitForAll(completableFutures).thenCompose((i) -> { - if (invalidIndex.get() != 0 ) { - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () - -> persistentWorker.updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), - indexes.values().stream().toList())); - } - return CompletableFuture.completedFuture(finalStartReadCursorPosition); - }); - }).exceptionally(ex -> { - log.error("[{}] Failed to recover snapshot segment", this.topic.getName(), ex); - return null; - }); - - - }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() - .getExecutor(this)); - } - - @Override - public CompletableFuture deleteAbortedTxnSnapshot() { - CompletableFuture completableFuture = new CompletableFuture<>(); - persistentWorker.appendTask(PersistentWorker.OperationType.Close, - () -> persistentWorker.clearSnapshotSegmentAndIndexes() - .thenRun(() -> { - completableFuture.thenCompose(null); - }).exceptionally(e -> { - completableFuture.completeExceptionally(e); - return null; - })); - return completableFuture; - } - - @Override - public long getLastSnapshotTimestamps() { - return this.lastSnapshotTimestamps; - } - - @Override - public CompletableFuture closeAsync() { - return persistentWorker.closeAsync(); - } - - private void handleSnapshotSegmentEntry(Entry entry) { - //decode snapshot from entry - ByteBuf headersAndPayload = entry.getDataBuffer(); - //skip metadata - Commands.parseMessageMetadata(headersAndPayload); - TransactionBufferSnapshotSegment snapshotSegment = Schema.AVRO(TransactionBufferSnapshotSegment.class) - .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer()); - abortTxnSegments.put(new PositionImpl(snapshotSegment.getMaxReadPositionLedgerId(), - snapshotSegment.getMaxReadPositionEntryId()), deserializationFotSnapshotSegment( - snapshotSegment.getAborts())); - - } - - private void closeReader(SystemTopicClient.Reader reader) { - reader.closeAsync().exceptionally(e -> { - log.error("[{}]Transaction buffer snapshot reader close error!", topic.getName(), e); - return null; - }); - } - - private class PersistentWorker { - protected final AtomicLong sequenceID = new AtomicLong(0); - - private final PersistentTopic topic; - - //Persistent snapshot segment and index at the single thread. - private final CompletableFuture> - snapshotSegmentsWriterFuture; - private final CompletableFuture> - snapshotIndexWriterFuture; - - private enum OperationState { - None, - UpdatingIndex, - WritingSegment, - DeletingSegment, - Closing, - Closed - } - private static final AtomicReferenceFieldUpdater - STATE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(PersistentWorker.class, - PersistentWorker.OperationState.class, "operationState"); - - public enum OperationType { - UpdateIndex, - WriteSegment, - DeleteSegment, - Close - } - - private volatile OperationState operationState = OperationState.None; - - ConcurrentLinkedDeque>>> taskQueue = - new ConcurrentLinkedDeque<>(); - private CompletableFuture lastOperationFuture; - private final Timer timer; - - public PersistentWorker(PersistentTopic topic) { - this.topic = topic; - this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); - this.snapshotSegmentsWriterFuture = this.topic.getBrokerService().getPulsar() - .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName())); - this.snapshotIndexWriterFuture = this.topic.getBrokerService().getPulsar() - .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName())); - - } - - public void appendTask(OperationType operationType, Supplier> task) { - switch (operationType) { - case UpdateIndex -> { - if (!taskQueue.isEmpty()) { - return; - } else if(STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.UpdatingIndex)) { - lastOperationFuture = task.get(); - lastOperationFuture.whenComplete((ignore, throwable) -> { - if (throwable != null && log.isDebugEnabled()) { - log.debug("[{}] Failed to update index snapshot", topic.getName(), throwable); - } - - STATE_UPDATER.compareAndSet(this, OperationState.UpdatingIndex, OperationState.None); - }); - } - } - case WriteSegment, DeleteSegment -> { - taskQueue.add(new MutablePair<>(operationType, task)); - executeTask(); - } - case Close -> { - STATE_UPDATER.set(this, OperationState.Closing); - taskQueue.clear(); - lastOperationFuture.thenRun(() -> { - lastOperationFuture = task.get(); - lastOperationFuture.thenRun(() -> - STATE_UPDATER.compareAndSet(this, OperationState.Closing, OperationState.Closed)); - }); - } - } - } - - private void executeTask() { - OperationType operationType = taskQueue.getFirst().getKey(); - switch (operationType) { - case WriteSegment -> { - if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.WritingSegment)) { - if (taskQueue.getFirst().getKey() == OperationType.WriteSegment) { - lastOperationFuture = taskQueue.getFirst().getValue().get(); - lastOperationFuture.whenComplete((ignore, throwable) -> { - if (throwable != null) { - if (log.isDebugEnabled()) { - log.debug("[{}] Failed to write snapshot segment", topic.getName(), throwable); - } - timer.newTimeout(timeout -> executeTask(), - takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); - } else { - taskQueue.removeFirst(); - } - STATE_UPDATER.compareAndSet(this, OperationState.WritingSegment, OperationState.None); - }); - } - } - } - case DeleteSegment -> { - if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.DeletingSegment)) { - if (taskQueue.getFirst().getKey() == OperationType.DeleteSegment) { - lastOperationFuture = taskQueue.getFirst().getValue().get(); - lastOperationFuture.whenComplete((ignore, throwable) -> { - if (throwable != null) { - if (log.isDebugEnabled()) { - log.debug("[{}] Failed to delete snapshot segment", topic.getName(), throwable); - } - timer.newTimeout(timeout -> executeTask(), - takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); - } else { - taskQueue.removeFirst(); - } - - STATE_UPDATER.compareAndSet(this, OperationState.DeletingSegment, OperationState.None); - }); - } - } - } - } - } - - private CompletableFuture takeSnapshotSegmentAsync(ConcurrentOpenHashSet sealedAbortedTxnIdSegment, - PositionImpl maxReadPosition) { - return writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, maxReadPosition).thenRun(() -> { - if (log.isDebugEnabled()) { - log.debug("Successes to take snapshot segment [{}] at maxReadPosition [{}] " - + "for the topic [{}], and the size of the segment is [{}]", - this.sequenceID, maxReadPosition, topic.getName(), sealedAbortedTxnIdSegment.size()); - } - this.sequenceID.getAndIncrement(); - }).exceptionally(e -> { - //Just log the error, and the processor will try to take snapshot again when the transactionBuffer - //append aborted txn nex time. - log.error("Failed to take snapshot segment [{}] at maxReadPosition [{}] " - + "for the topic [{}], and the size of the segment is [{}]", - this.sequenceID, maxReadPosition, topic.getName(), sealedAbortedTxnIdSegment.size(), e); - return null; - }); - } - - private CompletableFuture writeSnapshotSegmentAsync(ConcurrentOpenHashSet segment, - PositionImpl maxReadPosition) { - TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = new TransactionBufferSnapshotSegment(); - transactionBufferSnapshotSegment.setAborts(serializationForSegment(segment)); - transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); - transactionBufferSnapshotSegment.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); - transactionBufferSnapshotSegment.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); - - return snapshotSegmentsWriterFuture.thenCompose(segmentWriter -> { - transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get()); - return segmentWriter.writeAsync(buildKey(this.sequenceID.get()), transactionBufferSnapshotSegment); - }).thenCompose((messageId) -> { - //Build index for this segment - TransactionBufferSnapshotIndex index = new TransactionBufferSnapshotIndex(); - index.setSequenceID(transactionBufferSnapshotSegment.getSequenceId()); - index.setMaxReadPositionLedgerID(maxReadPosition.getLedgerId()); - index.setMaxReadPositionEntryID(maxReadPosition.getEntryId()); - index.setPersistentPositionLedgerID(((MessageIdImpl) messageId).getLedgerId()); - index.setPersistentPositionEntryID(((MessageIdImpl) messageId).getEntryId()); - - indexes.put(maxReadPosition, index); - //update snapshot segment index. - return updateSnapshotIndex(new TransactionBufferSnapshotIndexesMetadata( - maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), new HashSet<>()), - indexes.values().stream().toList()); - }); - } - - private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedToDelete) { - long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); - return snapshotSegmentsWriterFuture.thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) - .thenRun(() -> { - if (log.isDebugEnabled()) { - log.debug("[{}] Successes to delete the snapshot segment, " - + "whose sequenceId is [{}] and maxReadPosition is [{}]", - this.topic.getName(), this.sequenceID, positionNeedToDelete); - } - abortTxnSegments.remove(positionNeedToDelete); - //The process will check whether the snapshot segment is null, and update index when recovered. - indexes.remove(positionNeedToDelete); - //Keep index snapshot and update index - updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), - indexes.values().stream().toList()); - }).exceptionally(e -> { - log.warn("[{}] Failed to delete the snapshot segment, " - + "whose sequenceId is [{}] and maxReadPosition is [{}]", - this.topic.getName(), this.sequenceID, positionNeedToDelete, e); - return null; - }); - } - - //Update the indexes with the giving index snapshot and indexlist in the transactionBufferSnapshotIndexe. - private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotIndexesMetadata snapshotSegment, - List indexList) { - TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); - return snapshotIndexWriterFuture - .thenCompose((indexesWriter) -> { - snapshotIndexes.setIndexList(indexList); - snapshotIndexes.setSnapshot(snapshotSegment); - return indexesWriter.writeAsync(topic.getName(), snapshotIndexes); - }) - .thenRun(() -> { - persistentSnapshotIndexes = snapshotIndexes; - lastSnapshotTimestamps = System.currentTimeMillis(); - }) - .exceptionally(e -> { - log.error("[{}] Failed to update snapshot segment index", snapshotIndexes.getTopicName(), e); - return null; - }); - } - - //Only update the metadata in the transactionBufferSnapshotIndexes. - private CompletableFuture updateIndexMetadataForTheLastSnapshot(PositionImpl maxReadPosition, - ConcurrentOpenHashSet abortedTxns) { - TransactionBufferSnapshotIndexesMetadata metadata = new TransactionBufferSnapshotIndexesMetadata( - maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), serializationForSegment(abortedTxns)); - - return updateSnapshotIndex(metadata, persistentSnapshotIndexes.getIndexList()); - } - - private CompletableFuture clearSnapshotSegmentAndIndexes() { - ArrayList> completableFutures = new ArrayList<>(); - //Delete all segment - while (!abortTxnSegments.isEmpty()) { - if (log.isDebugEnabled()) { - log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", - topic.getName(), abortTxnSegments.firstKey()); - } - PositionImpl positionNeedToDelete = abortTxnSegments.firstKey(); - completableFutures.add(persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); - } - //Delete index - return FutureUtil.waitForAll(completableFutures) - .thenCompose((ignore) -> snapshotIndexWriterFuture - .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) - .thenRun(() -> { - log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", - topic.getName()); - - }) - .exceptionally(e -> { - log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", - topic.getName(), e); - - return null; - }); - } - - - CompletableFuture closeAsync() { - return CompletableFuture.allOf( - this.snapshotIndexWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync), - this.snapshotSegmentsWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync)); - } - } - - private ConcurrentOpenHashSet deserializationFotSnapshotSegment(Set snapshotSegment) { - ConcurrentOpenHashSet set = new ConcurrentOpenHashSet<>(); - snapshotSegment.forEach(txnIDData -> { - set.add(new TxnID(txnIDData.getMostSigBits(), txnIDData.getLeastSigBits())); - }); - return set; - } - - private Set serializationForSegment(ConcurrentOpenHashSet segment) { - Set set = new HashSet<>(); - segment.forEach(txnID -> { - set.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits())); - }); - return set; - } - -} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index a00e09db1308e..9327a6effe177 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -111,11 +111,7 @@ public TopicTransactionBuffer(PersistentTopic topic) { this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); - if (topic.getBrokerService().getPulsar().getConfiguration().isTransactionBufferSegmentedSnapshotEnabled()) { - snapshotAbortedTxnProcessor = new SnapshotSegmentAbortedTxnProcessorImpl(topic); - } else { - snapshotAbortedTxnProcessor = new SingleSnapshotAbortedTxnProcessorImpl(topic); - } + this.snapshotAbortedTxnProcessor = new SingleSnapshotAbortedTxnProcessorImpl(topic); this.recover(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 876e30a7cee1b..8a42ddc8e65e3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -735,7 +735,8 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob ByteBuf headersAndPayload = entry.getDataBuffer(); //skip metadata MessageMetadata msgMetadata = Commands.parseMessageMetadata(headersAndPayload); - snapshot = Schema.AVRO(TransactionBufferSnapshotSegment.class).decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer()); + snapshot = Schema.AVRO(TransactionBufferSnapshotSegment.class) + .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer()); //verify snapshot assertEquals(snapshot.getTopicName(), snapshotTopic); @@ -745,80 +746,4 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob assertEquals(snapshot.getAborts().toArray()[0], new TxnIDData(1, 1)); } - @Test - public void testSnapshotSegment() throws Exception { - String topic = NAMESPACE1 + "/testSnapshotSegment"; - String subName = "testSnapshotSegment"; - - LinkedMap ongoingTxns = new LinkedMap<>(); - LinkedList abortedTxns = new LinkedList<>(); - - this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); - this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSnapshotSegmentSize(10); - this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSnapshotMaxTransactionCount(3); - - Producer producer = pulsarClient.newProducer(Schema.INT32) - .topic(topic) - .enableBatching(false) - .create(); - - Consumer consumer = pulsarClient.newConsumer(Schema.INT32) - .topic(topic) - .subscriptionName(subName) - .subscriptionType(SubscriptionType.Exclusive) - .subscribe(); - - for (int i = 0; i < 10; i++) { - int maxReadMessage = 19; - int abortedTxnSize = 0; - for (int j = 0; j < 20; j++) { - Transaction transaction = pulsarClient.newTransaction() - .withTransactionTimeout(5, TimeUnit.MINUTES).build().get(); - //half common message and half transaction message. - //the transaction message have a half which are aborted. - if (RandomUtils.nextInt() % 2 == 0) { - MessageId messageId = producer.newMessage(transaction).value(i * 10 + j).send(); - if (RandomUtils.nextInt() % 2 == 0) { - transaction.abort().get(); - abortedTxns.add(messageId); - abortedTxnSize++; - } else { - ongoingTxns.put(transaction, messageId); - if (maxReadMessage == 19) { - //The except number of the messages that can be read - maxReadMessage = j - abortedTxnSize; - } - } - } else { - MessageId messageId = producer.newMessage().value(i * 10 + j).send(); - transaction.commit().get(); - } - } - for (int k = 0; k < maxReadMessage; k++) { - Message message = consumer.receive(2, TimeUnit.SECONDS); - assertNotNull(message); - assertFalse(abortedTxns.contains(message.getMessageId())); - } - Message message = consumer.receive(2, TimeUnit.SECONDS); - assertNull(message); - - for (Transaction ongoingTxn: ongoingTxns.keySet()) { - ongoingTxn.commit().get(); - } - ongoingTxns.clear(); - for (int k = maxReadMessage; k < 20 - abortedTxnSize; k++) { - message = consumer.receive(2, TimeUnit.SECONDS); - assertNotNull(message); - assertFalse(abortedTxns.contains(message.getMessageId())); - } - } - - admin.topics().unload(topic); - - for (int i = 0; i < 200 - abortedTxns.size(); i++) { - Message message = consumer.receive(2, TimeUnit.SECONDS); - assertNotNull(message); - assertFalse(abortedTxns.contains(message.getMessageId())); - } - } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 29165516ff7c4..e512ea9693398 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -93,7 +93,6 @@ import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.TransactionBuffer; -import org.apache.pulsar.broker.transaction.buffer.impl.SingleSnapshotAbortedTxnProcessorImpl; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferProvider; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferRecoverCallBack; @@ -663,13 +662,11 @@ public void testMaxReadPositionForNormalPublish() throws Exception { Field field = transactionBufferStateClass.getDeclaredField("state"); field.setAccessible(true); Class topicTransactionBufferClass = TopicTransactionBuffer.class; - Field processorField = topicTransactionBufferClass.getDeclaredField("snapshotAbortedTxnProcessor"); - processorField.setAccessible(true); - AbortedTxnProcessor abortedTxnProcessor = (AbortedTxnProcessor) processorField.get(topicTransactionBuffer); - + Field maxReadPositionField = topicTransactionBufferClass.getDeclaredField("maxReadPosition"); + maxReadPositionField.setAccessible(true); field.set(topicTransactionBuffer, TopicTransactionBufferState.State.Initializing); MessageIdImpl messageId5 = (MessageIdImpl) normalProducer.newMessage().value("normal message").send(); - PositionImpl position5 = topicTransactionBuffer.getMaxReadPosition(); + PositionImpl position5 = (PositionImpl) maxReadPositionField.get(topicTransactionBuffer); Assert.assertEquals(position5.getLedgerId(), messageId4.getLedgerId()); Assert.assertEquals(position5.getEntryId(), messageId4.getEntryId()); } @@ -1036,10 +1033,10 @@ public void testNotChangeMaxReadPositionAndAddAbortTimesWhenCheckIfNoSnapshot() processorField.setAccessible(true); AbortedTxnProcessor abortedTxnProcessor = (AbortedTxnProcessor) processorField.get(buffer); - Field changeTimeField = SingleSnapshotAbortedTxnProcessorImpl + Field changeTimeField = TopicTransactionBuffer .class.getDeclaredField("changeMaxReadPositionAndAddAbortTimes"); changeTimeField.setAccessible(true); - AtomicLong changeMaxReadPositionAndAddAbortTimes = (AtomicLong) changeTimeField.get(abortedTxnProcessor); + AtomicLong changeMaxReadPositionAndAddAbortTimes = (AtomicLong) changeTimeField.get(buffer); Field field1 = TopicTransactionBufferState.class.getDeclaredField("state"); field1.setAccessible(true); From 1a070cbe14a781f494a51f08254e072f3231c010 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Mon, 31 Oct 2022 20:15:14 +0800 Subject: [PATCH 28/32] fix some comments --- .../broker/transaction/buffer/AbortedTxnProcessor.java | 4 ++-- .../buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java | 2 +- .../transaction/buffer/impl/TopicTransactionBuffer.java | 6 +++--- .../transaction/TopicTransactionBufferRecoverTest.java | 5 +---- .../apache/pulsar/broker/transaction/TransactionTest.java | 2 +- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index cb05b7e2d7782..cf98bc7af362e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -61,10 +61,10 @@ public interface AbortedTxnProcessor { CompletableFuture deleteAbortedTxnSnapshot(); /** - * Take the frist snapshot if the topic has no snapshot before. + * Take aborted transactions snapshot. * @return a completableFuture. */ - CompletableFuture takeAbortedTxnSnapshot(PositionImpl maxReadPosition); + CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosition); /** * Get the lastSnapshotTimestamps. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index b23fab6592987..704fd3cf8fe3a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -125,7 +125,7 @@ public CompletableFuture deleteAbortedTxnSnapshot() { } @Override - public CompletableFuture takeAbortedTxnSnapshot(PositionImpl maxReadPosition) { + public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosition) { return takeSnapshotWriter.thenCompose(writer -> { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); snapshot.setTopicName(topic.getName()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 9327a6effe177..2cbd39b1af9f9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -208,7 +208,7 @@ public CompletableFuture checkIfTBRecoverCompletely(boolean isTxnEnabled) CompletableFuture completableFuture = new CompletableFuture<>(); transactionBufferFuture.thenRun(() -> { if (checkIfNoSnapshot()) { - snapshotAbortedTxnProcessor.takeAbortedTxnSnapshot(maxReadPosition).thenRun(() -> { + snapshotAbortedTxnProcessor.takeAbortedTxnsSnapshot(maxReadPosition).thenRun(() -> { if (changeToReadyStateFromNoSnapshot()) { timer.newTimeout(TopicTransactionBuffer.this, takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); @@ -421,14 +421,14 @@ private void handleLowWaterMark(TxnID txnID, long lowWaterMark) { private void takeSnapshotByChangeTimes() { if (changeMaxReadPositionAndAddAbortTimes.get() >= takeSnapshotIntervalNumber) { this.changeMaxReadPositionAndAddAbortTimes.set(0); - this.snapshotAbortedTxnProcessor.takeAbortedTxnSnapshot(this.maxReadPosition); + this.snapshotAbortedTxnProcessor.takeAbortedTxnsSnapshot(this.maxReadPosition); } } private void takeSnapshotByTimeout() { if (changeMaxReadPositionAndAddAbortTimes.get() > 0) { this.changeMaxReadPositionAndAddAbortTimes.set(0); - this.snapshotAbortedTxnProcessor.takeAbortedTxnSnapshot(this.maxReadPosition); + this.snapshotAbortedTxnProcessor.takeAbortedTxnsSnapshot(this.maxReadPosition); } this.timer.newTimeout(TopicTransactionBuffer.this, takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 8a42ddc8e65e3..370c10fa42061 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -31,14 +31,12 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.lang.reflect.Field; -import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.NavigableMap; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; @@ -77,7 +75,6 @@ import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderBuilder; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -451,7 +448,7 @@ public void clearTransactionBufferSnapshotTest(Boolean enableSnapshotSegment) th abortedTxnProcessorField.setAccessible(true); AbortedTxnProcessor abortedTxnProcessor = (AbortedTxnProcessor) abortedTxnProcessorField.get(topicTransactionBuffer); - abortedTxnProcessor.takeAbortedTxnSnapshot(topicTransactionBuffer.getMaxReadPosition()); + abortedTxnProcessor.takeAbortedTxnsSnapshot(topicTransactionBuffer.getMaxReadPosition()); TopicName transactionBufferTopicName = NamespaceEventsSystemTopicFactory.getSystemTopicName( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index e512ea9693398..cbdf3ab33c912 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -946,7 +946,7 @@ public void testNoEntryCanBeReadWhenRecovery() throws Exception { Field processorField = TopicTransactionBuffer.class.getDeclaredField("snapshotAbortedTxnProcessor"); processorField.setAccessible(true); AbortedTxnProcessor abortedTxnProcessor = (AbortedTxnProcessor) processorField.get(topicTransactionBuffer); - CompletableFuture completableFuture = abortedTxnProcessor.takeAbortedTxnSnapshot( + CompletableFuture completableFuture = abortedTxnProcessor.takeAbortedTxnsSnapshot( topicTransactionBuffer.getMaxReadPosition()); completableFuture.get(); From 99abe3fe28711a3749ea76e955d4c6389c41e817 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Mon, 31 Oct 2022 20:47:24 +0800 Subject: [PATCH 29/32] checkstyle --- .../buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java | 2 -- .../broker/transaction/buffer/impl/TopicTransactionBuffer.java | 1 - .../buffer/impl/TopicTransactionBufferRecoverCallBack.java | 1 - 3 files changed, 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 704fd3cf8fe3a..159d697a50472 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -18,11 +18,9 @@ */ package org.apache.pulsar.broker.transaction.buffer.impl; -import io.netty.util.Timer; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 2cbd39b1af9f9..9544a69116f6b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -23,7 +23,6 @@ import io.netty.util.Timeout; import io.netty.util.Timer; import io.netty.util.TimerTask; - import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoverCallBack.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoverCallBack.java index 324806296b750..768c0b3abd550 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoverCallBack.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBufferRecoverCallBack.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.transaction.buffer.impl; import org.apache.bookkeeper.mledger.Entry; -import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; public interface TopicTransactionBufferRecoverCallBack { From 5725b3483497a23b4977512efeb8fe6b764b2652 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Mon, 31 Oct 2022 21:51:35 +0800 Subject: [PATCH 30/32] license header --- .../pulsar/broker/transaction/buffer/AbortedTxnProcessor.java | 2 +- .../buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java | 2 +- .../metadata/v2/TransactionBufferSnapshotIndexesMetadata.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index cf98bc7af362e..e436e1df24972 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 159d697a50472..a13dd0499a6bc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java index 59fbea0e35769..9a468d250bbbf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information From 8f73634bf4a2ed76ed2037e8e3b4403fff869304 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Mon, 31 Oct 2022 23:26:39 +0800 Subject: [PATCH 31/32] fix test --- .../buffer/metadata/v2/TransactionBufferSnapshotSegment.java | 4 ++-- .../broker/transaction/TopicTransactionBufferRecoverTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java index 1ffd5cad6e785..77bb546880dbe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.broker.transaction.buffer.metadata.v2; -import java.util.Set; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -31,5 +31,5 @@ public class TransactionBufferSnapshotSegment { private long sequenceId; private long maxReadPositionLedgerId; private long maxReadPositionEntryId; - private Set aborts; + private List aborts; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 1a323e9b9758f..89427cf29b630 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -687,7 +687,7 @@ public void testTransactionBufferSegmentSystemTopic() throws Exception { snapshot.setSequenceId(1L); snapshot.setMaxReadPositionLedgerId(2L); snapshot.setMaxReadPositionEntryId(3L); - HashSet txnIDSet = new HashSet<>(); + LinkedList txnIDSet = new LinkedList<>(); txnIDSet.add(new TxnIDData(1, 1)); snapshot.setAborts(txnIDSet ); From 7b1b817d7c292dac86afff3d725f28a70ad024b8 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Mon, 31 Oct 2022 23:40:14 +0800 Subject: [PATCH 32/32] unused imported --- .../broker/transaction/TopicTransactionBufferRecoverTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 89427cf29b630..39c324d92f38c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -31,7 +31,6 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.lang.reflect.Field; -import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.NavigableMap;