From de37e61a9c7a5977222ae4399e4dd0a7d329b4a4 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Tue, 1 Nov 2022 10:49:30 +0800 Subject: [PATCH 01/41] [feat][txn] implement the SnapshotSegmentAbortedTxnProcessor --- ...napshotSegmentAbortedTxnProcessorImpl.java | 605 ++++++++++++++++++ .../buffer/impl/TopicTransactionBuffer.java | 6 +- ...nsactionBufferSnapshotIndexesMetadata.java | 4 +- .../TopicTransactionBufferRecoverTest.java | 78 +++ 4 files changed, 690 insertions(+), 3 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java new file mode 100644 index 0000000000000..1ca7a09e79192 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -0,0 +1,605 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing,2 + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.transaction.buffer.impl; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.util.Timer; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.function.Supplier; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.AsyncCallbacks; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.systopic.SystemTopicClient; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexesMetadata; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TxnIDData; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.common.events.EventType; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; + +@Slf4j +public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { + + private ConcurrentOpenHashSet unsealedAbortedTxnIdSegment = new ConcurrentOpenHashSet<>(); + + //Store the fixed aborted transaction segment + private final ConcurrentSkipListMap> abortTxnSegments + = new ConcurrentSkipListMap<>(); + + private final ConcurrentSkipListMap indexes + = new ConcurrentSkipListMap<>(); + //The latest persistent snapshot index. This is used to combine new segment indexes with the latest metadata and + // indexes. + private TransactionBufferSnapshotIndexes persistentSnapshotIndexes = new TransactionBufferSnapshotIndexes(); + + private final PersistentTopic topic; + + private volatile long lastSnapshotTimestamps; + + private final int takeSnapshotIntervalTime; + + private final int transactionBufferMaxAbortedTxnsOfSnapshotSegment; + private final PersistentWorker persistentWorker; + + public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { + this.topic = topic; + this.persistentWorker = new PersistentWorker(topic); + this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() + .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); + this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = topic.getBrokerService().getPulsar() + .getConfiguration().getTransactionBufferSnapshotSegmentSize(); + } + + @Override + public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl maxReadPosition) { + unsealedAbortedTxnIdSegment.add(abortedTxnId); + //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. + if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { + abortTxnSegments.put(maxReadPosition, unsealedAbortedTxnIdSegment); + persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> + persistentWorker.takeSnapshotSegmentAsync(unsealedAbortedTxnIdSegment, maxReadPosition)); + unsealedAbortedTxnIdSegment = new ConcurrentOpenHashSet<>(); + } + } + + @Override + public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { + if (readPosition == null) { + return abortTxnSegments.values().stream() + .anyMatch(list -> list.contains(txnID)) || unsealedAbortedTxnIdSegment.contains(txnID); + } else { + PositionImpl maxReadPosition = abortTxnSegments.ceilingKey((PositionImpl) readPosition); + if (maxReadPosition != null) { + return abortTxnSegments.keySet().stream() + .filter((position) -> position.compareTo(maxReadPosition) <= 0) + .anyMatch((position -> abortTxnSegments.get(position).contains(txnID))); + } else { + return abortTxnSegments.keySet().stream() + .filter((position) -> position.compareTo((PositionImpl) readPosition) <= 0) + .anyMatch((position -> abortTxnSegments.get(position).contains(txnID))) + || unsealedAbortedTxnIdSegment.contains(txnID); + } + } + } + + //In this implementation, we adopt snapshot segments. And then we clear invalid segment by its max read position. + @Override + public void trimExpiredAbortedTxns() { + //Checking whether there are some segment expired. + while (!abortTxnSegments.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) + .ledgerExists(abortTxnSegments.firstKey().getLedgerId())) { + if (log.isDebugEnabled()) { + log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", + topic.getName(), abortTxnSegments.firstKey()); + } + PositionImpl positionNeedToDelete = abortTxnSegments.firstKey(); + persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment, + () -> persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); + } + } + + private String buildKey(long sequenceId) { + return "multiple-" + sequenceId + "-" + this.topic.getName(); + } + + @Override + public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosition) { + ConcurrentOpenHashSet aborts = unsealedAbortedTxnIdSegment; + CompletableFuture completableFuture = new CompletableFuture<>(); + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, + () -> persistentWorker + .updateIndexMetadataForTheLastSnapshot(maxReadPosition, aborts) + .thenRun(() -> completableFuture.complete(null)) + .exceptionally(e -> { + completableFuture.completeExceptionally(e); + return null; + })); + return completableFuture; + } + + @Override + public CompletableFuture recoverFromSnapshot() { + return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotIndexService() + .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { + PositionImpl startReadCursorPosition = null; + boolean hasIndex = false; + try { + //Read Index to recover the sequenceID, indexes, lastAbortedTxns and maxReadPosition. + while (reader.hasMoreEvents()) { + Message message = reader.readNext(); + if (topic.getName().equals(message.getKey())) { + TransactionBufferSnapshotIndexes transactionBufferSnapshotIndexes = message.getValue(); + if (transactionBufferSnapshotIndexes != null) { + hasIndex = true; + this.persistentSnapshotIndexes = transactionBufferSnapshotIndexes; + startReadCursorPosition = PositionImpl.get( + transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionLedgerId(), + transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); + } + } + } + } catch (Exception ex) { + log.error("[{}] Transaction buffer recover fail when read " + + "transactionBufferSnapshot!", topic.getName(), ex); + return FutureUtil.failedFuture(ex); + } finally { + closeReader(reader); + } + PositionImpl finalStartReadCursorPosition = startReadCursorPosition; + if (!hasIndex) { + return CompletableFuture.completedFuture(null); + } else { + persistentSnapshotIndexes.getIndexList() + .forEach(transactionBufferSnapshotIndex -> + indexes.put(new PositionImpl( + transactionBufferSnapshotIndex.persistentPositionLedgerID, + transactionBufferSnapshotIndex.persistentPositionEntryID), + transactionBufferSnapshotIndex)); + this.unsealedAbortedTxnIdSegment = deserializationFotSnapshotSegment(persistentSnapshotIndexes + .getSnapshot().getAborts()); + if (indexes.size() != 0) { + persistentWorker.sequenceID.set(indexes.lastEntry().getValue().sequenceID + 1); + } + } + //Read snapshot segment to recover aborts. + ArrayList> completableFutures = new ArrayList<>(); + CompletableFuture openManagedLedgerFuture = new CompletableFuture<>(); + AtomicLong invalidIndex = new AtomicLong(0); + AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks + .OpenReadOnlyManagedLedgerCallback() { + @Override + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { + persistentSnapshotIndexes.getIndexList().forEach(index -> { + CompletableFuture handleSegmentFuture = new CompletableFuture<>(); + completableFutures.add(handleSegmentFuture); + readOnlyManagedLedger.asyncReadEntry( + new PositionImpl(index.getPersistentPositionLedgerID(), + index.getPersistentPositionEntryID()), + new AsyncCallbacks.ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + //Remove invalid index + if (entry == null) { + indexes.remove(new PositionImpl( + index.getMaxReadPositionLedgerID(), + index.getMaxReadPositionEntryID())); + handleSegmentFuture.complete(null); + invalidIndex.getAndIncrement(); + return; + } + handleSnapshotSegmentEntry(entry); + handleSegmentFuture.complete(null); + } + + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + handleSegmentFuture.completeExceptionally(exception); + } + }, null); + }); + openManagedLedgerFuture.complete(null); + } + + @Override + public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx) { + log.error("[{}] Failed to open readOnly managed ledger", topic, exception); + openManagedLedgerFuture.completeExceptionally(exception); + } + }; + + TopicName snapshotIndexTopicName = TopicName.get(TopicDomain.persistent.toString(), + TopicName.get(topic.getName()).getNamespaceObject(), + EventType.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS.toString()); + this.topic.getBrokerService().getPulsar().getManagedLedgerFactory() + .asyncOpenReadOnlyManagedLedger(snapshotIndexTopicName + .getPersistenceNamingEncoding(), callback, + topic.getManagedLedger().getConfig(), + null); + //Wait the processor recover completely and the allow TB to recover the messages + // after the startReadCursorPosition. + + return openManagedLedgerFuture.thenCompose((ignore) -> { + return FutureUtil.waitForAll(completableFutures).thenCompose((i) -> { + if (invalidIndex.get() != 0 ) { + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () + -> persistentWorker.updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), + indexes.values().stream().toList())); + } + return CompletableFuture.completedFuture(finalStartReadCursorPosition); + }); + }).exceptionally(ex -> { + log.error("[{}] Failed to recover snapshot segment", this.topic.getName(), ex); + return null; + }); + + + }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this)); + } + + @Override + public CompletableFuture deleteAbortedTxnSnapshot() { + CompletableFuture completableFuture = new CompletableFuture<>(); + persistentWorker.appendTask(PersistentWorker.OperationType.Close, + () -> persistentWorker.clearSnapshotSegmentAndIndexes() + .thenRun(() -> { + completableFuture.thenCompose(null); + }).exceptionally(e -> { + completableFuture.completeExceptionally(e); + return null; + })); + return completableFuture; + } + + @Override + public long getLastSnapshotTimestamps() { + return this.lastSnapshotTimestamps; + } + + @Override + public CompletableFuture closeAsync() { + return persistentWorker.closeAsync(); + } + + private void handleSnapshotSegmentEntry(Entry entry) { + //decode snapshot from entry + ByteBuf headersAndPayload = entry.getDataBuffer(); + //skip metadata + Commands.parseMessageMetadata(headersAndPayload); + TransactionBufferSnapshotSegment snapshotSegment = Schema.AVRO(TransactionBufferSnapshotSegment.class) + .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer()); + abortTxnSegments.put(new PositionImpl(snapshotSegment.getMaxReadPositionLedgerId(), + snapshotSegment.getMaxReadPositionEntryId()), deserializationFotSnapshotSegment( + snapshotSegment.getAborts())); + + } + + private void closeReader(SystemTopicClient.Reader reader) { + reader.closeAsync().exceptionally(e -> { + log.error("[{}]Transaction buffer snapshot reader close error!", topic.getName(), e); + return null; + }); + } + + private class PersistentWorker { + protected final AtomicLong sequenceID = new AtomicLong(0); + + private final PersistentTopic topic; + + //Persistent snapshot segment and index at the single thread. + private final CompletableFuture> + snapshotSegmentsWriterFuture; + private final CompletableFuture> + snapshotIndexWriterFuture; + + private enum OperationState { + None, + UpdatingIndex, + WritingSegment, + DeletingSegment, + Closing, + Closed + } + private static final AtomicReferenceFieldUpdater + STATE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(PersistentWorker.class, + PersistentWorker.OperationState.class, "operationState"); + + public enum OperationType { + UpdateIndex, + WriteSegment, + DeleteSegment, + Close + } + + private volatile OperationState operationState = OperationState.None; + + ConcurrentLinkedDeque>>> taskQueue = + new ConcurrentLinkedDeque<>(); + private CompletableFuture lastOperationFuture; + private final Timer timer; + + public PersistentWorker(PersistentTopic topic) { + this.topic = topic; + this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); + this.snapshotSegmentsWriterFuture = this.topic.getBrokerService().getPulsar() + .getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName())); + this.snapshotIndexWriterFuture = this.topic.getBrokerService().getPulsar() + .getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName())); + + } + + public void appendTask(OperationType operationType, Supplier> task) { + switch (operationType) { + case UpdateIndex -> { + if (!taskQueue.isEmpty()) { + return; + } else if(STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.UpdatingIndex)) { + lastOperationFuture = task.get(); + lastOperationFuture.whenComplete((ignore, throwable) -> { + if (throwable != null && log.isDebugEnabled()) { + log.debug("[{}] Failed to update index snapshot", topic.getName(), throwable); + } + + STATE_UPDATER.compareAndSet(this, OperationState.UpdatingIndex, OperationState.None); + }); + } + } + case WriteSegment, DeleteSegment -> { + taskQueue.add(new MutablePair<>(operationType, task)); + executeTask(); + } + case Close -> { + STATE_UPDATER.set(this, OperationState.Closing); + taskQueue.clear(); + lastOperationFuture.thenRun(() -> { + lastOperationFuture = task.get(); + lastOperationFuture.thenRun(() -> + STATE_UPDATER.compareAndSet(this, OperationState.Closing, OperationState.Closed)); + }); + } + } + } + + private void executeTask() { + OperationType operationType = taskQueue.getFirst().getKey(); + switch (operationType) { + case WriteSegment -> { + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.WritingSegment)) { + if (taskQueue.getFirst().getKey() == OperationType.WriteSegment) { + lastOperationFuture = taskQueue.getFirst().getValue().get(); + lastOperationFuture.whenComplete((ignore, throwable) -> { + if (throwable != null) { + if (log.isDebugEnabled()) { + log.debug("[{}] Failed to write snapshot segment", topic.getName(), throwable); + } + timer.newTimeout(timeout -> executeTask(), + takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); + } else { + taskQueue.removeFirst(); + } + STATE_UPDATER.compareAndSet(this, OperationState.WritingSegment, OperationState.None); + }); + } + } + } + case DeleteSegment -> { + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.DeletingSegment)) { + if (taskQueue.getFirst().getKey() == OperationType.DeleteSegment) { + lastOperationFuture = taskQueue.getFirst().getValue().get(); + lastOperationFuture.whenComplete((ignore, throwable) -> { + if (throwable != null) { + if (log.isDebugEnabled()) { + log.debug("[{}] Failed to delete snapshot segment", topic.getName(), throwable); + } + timer.newTimeout(timeout -> executeTask(), + takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); + } else { + taskQueue.removeFirst(); + } + + STATE_UPDATER.compareAndSet(this, OperationState.DeletingSegment, OperationState.None); + }); + } + } + } + } + } + + private CompletableFuture takeSnapshotSegmentAsync(ConcurrentOpenHashSet sealedAbortedTxnIdSegment, + PositionImpl maxReadPosition) { + return writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, maxReadPosition).thenRun(() -> { + if (log.isDebugEnabled()) { + log.debug("Successes to take snapshot segment [{}] at maxReadPosition [{}] " + + "for the topic [{}], and the size of the segment is [{}]", + this.sequenceID, maxReadPosition, topic.getName(), sealedAbortedTxnIdSegment.size()); + } + this.sequenceID.getAndIncrement(); + }).exceptionally(e -> { + //Just log the error, and the processor will try to take snapshot again when the transactionBuffer + //append aborted txn nex time. + log.error("Failed to take snapshot segment [{}] at maxReadPosition [{}] " + + "for the topic [{}], and the size of the segment is [{}]", + this.sequenceID, maxReadPosition, topic.getName(), sealedAbortedTxnIdSegment.size(), e); + return null; + }); + } + + private CompletableFuture writeSnapshotSegmentAsync(ConcurrentOpenHashSet segment, + PositionImpl maxReadPosition) { + TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = new TransactionBufferSnapshotSegment(); + transactionBufferSnapshotSegment.setAborts(serializationForSegment(segment)); + transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); + transactionBufferSnapshotSegment.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); + transactionBufferSnapshotSegment.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); + + return snapshotSegmentsWriterFuture.thenCompose(segmentWriter -> { + transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get()); + return segmentWriter.writeAsync(buildKey(this.sequenceID.get()), transactionBufferSnapshotSegment); + }).thenCompose((messageId) -> { + //Build index for this segment + TransactionBufferSnapshotIndex index = new TransactionBufferSnapshotIndex(); + index.setSequenceID(transactionBufferSnapshotSegment.getSequenceId()); + index.setMaxReadPositionLedgerID(maxReadPosition.getLedgerId()); + index.setMaxReadPositionEntryID(maxReadPosition.getEntryId()); + index.setPersistentPositionLedgerID(((MessageIdImpl) messageId).getLedgerId()); + index.setPersistentPositionEntryID(((MessageIdImpl) messageId).getEntryId()); + + indexes.put(maxReadPosition, index); + //update snapshot segment index. + return updateSnapshotIndex(new TransactionBufferSnapshotIndexesMetadata( + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), new LinkedList<>()), + indexes.values().stream().toList()); + }); + } + + private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedToDelete) { + long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); + return snapshotSegmentsWriterFuture.thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) + .thenRun(() -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Successes to delete the snapshot segment, " + + "whose sequenceId is [{}] and maxReadPosition is [{}]", + this.topic.getName(), this.sequenceID, positionNeedToDelete); + } + abortTxnSegments.remove(positionNeedToDelete); + //The process will check whether the snapshot segment is null, and update index when recovered. + indexes.remove(positionNeedToDelete); + //Keep index snapshot and update index + updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), + indexes.values().stream().toList()); + }).exceptionally(e -> { + log.warn("[{}] Failed to delete the snapshot segment, " + + "whose sequenceId is [{}] and maxReadPosition is [{}]", + this.topic.getName(), this.sequenceID, positionNeedToDelete, e); + return null; + }); + } + + //Update the indexes with the giving index snapshot and indexlist in the transactionBufferSnapshotIndexe. + private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotIndexesMetadata snapshotSegment, + List indexList) { + TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); + return snapshotIndexWriterFuture + .thenCompose((indexesWriter) -> { + snapshotIndexes.setIndexList(indexList); + snapshotIndexes.setSnapshot(snapshotSegment); + return indexesWriter.writeAsync(topic.getName(), snapshotIndexes); + }) + .thenRun(() -> { + persistentSnapshotIndexes = snapshotIndexes; + lastSnapshotTimestamps = System.currentTimeMillis(); + }) + .exceptionally(e -> { + log.error("[{}] Failed to update snapshot segment index", snapshotIndexes.getTopicName(), e); + return null; + }); + } + + //Only update the metadata in the transactionBufferSnapshotIndexes. + private CompletableFuture updateIndexMetadataForTheLastSnapshot(PositionImpl maxReadPosition, + ConcurrentOpenHashSet abortedTxns) { + TransactionBufferSnapshotIndexesMetadata metadata = new TransactionBufferSnapshotIndexesMetadata( + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), serializationForSegment(abortedTxns)); + + return updateSnapshotIndex(metadata, persistentSnapshotIndexes.getIndexList()); + } + + private CompletableFuture clearSnapshotSegmentAndIndexes() { + ArrayList> completableFutures = new ArrayList<>(); + //Delete all segment + while (!abortTxnSegments.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", + topic.getName(), abortTxnSegments.firstKey()); + } + PositionImpl positionNeedToDelete = abortTxnSegments.firstKey(); + completableFutures.add(persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); + } + //Delete index + return FutureUtil.waitForAll(completableFutures) + .thenCompose((ignore) -> snapshotIndexWriterFuture + .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) + .thenRun(() -> { + log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", + topic.getName()); + + }) + .exceptionally(e -> { + log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", + topic.getName(), e); + + return null; + }); + } + + + CompletableFuture closeAsync() { + return CompletableFuture.allOf( + this.snapshotIndexWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync), + this.snapshotSegmentsWriterFuture.thenCompose(SystemTopicClient.Writer::closeAsync)); + } + } + + private ConcurrentOpenHashSet deserializationFotSnapshotSegment(List snapshotSegment) { + ConcurrentOpenHashSet set = new ConcurrentOpenHashSet<>(); + snapshotSegment.forEach(txnIDData -> { + set.add(new TxnID(txnIDData.getMostSigBits(), txnIDData.getLeastSigBits())); + }); + return set; + } + + private List serializationForSegment(ConcurrentOpenHashSet segment) { + List set = new LinkedList<>(); + segment.forEach(txnID -> { + set.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits())); + }); + return set; + } + +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index f3bf4f95923cd..c40e868739199 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -110,7 +110,11 @@ public TopicTransactionBuffer(PersistentTopic topic) { this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); this.maxReadPosition = (PositionImpl) topic.getManagedLedger().getLastConfirmedEntry(); - this.snapshotAbortedTxnProcessor = new SingleSnapshotAbortedTxnProcessorImpl(topic); + if (topic.getBrokerService().getPulsar().getConfiguration().isTransactionBufferSegmentedSnapshotEnabled()) { + snapshotAbortedTxnProcessor = new SnapshotSegmentAbortedTxnProcessorImpl(topic); + } else { + snapshotAbortedTxnProcessor = new SingleSnapshotAbortedTxnProcessorImpl(topic); + } this.recover(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java index 9a468d250bbbf..f9c28a818f8b8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndexesMetadata.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.broker.transaction.buffer.metadata.v2; -import java.util.Set; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -29,5 +29,5 @@ public class TransactionBufferSnapshotIndexesMetadata { private long maxReadPositionLedgerId; private long maxReadPositionEntryId; - private Set aborts; + private List aborts; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 39c324d92f38c..2fcc26762a41f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -74,6 +74,7 @@ import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderBuilder; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -742,4 +743,81 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob assertEquals(snapshot.getAborts().toArray()[0], new TxnIDData(1, 1)); } + @Test + public void testSnapshotSegment() throws Exception { + String topic = NAMESPACE1 + "/testSnapshotSegment"; + String subName = "testSnapshotSegment"; + + LinkedMap ongoingTxns = new LinkedMap<>(); + LinkedList abortedTxns = new LinkedList<>(); + + this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSnapshotSegmentSize(10); + this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSnapshotMaxTransactionCount(3); + + Producer producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .enableBatching(false) + .create(); + + Consumer consumer = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Exclusive) + .subscribe(); + + for (int i = 0; i < 10; i++) { + int maxReadMessage = 19; + int abortedTxnSize = 0; + for (int j = 0; j < 20; j++) { + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.MINUTES).build().get(); + //half common message and half transaction message. + //the transaction message have a half which are aborted. + if (RandomUtils.nextInt() % 2 == 0) { + MessageId messageId = producer.newMessage(transaction).value(i * 10 + j).send(); + if (RandomUtils.nextInt() % 2 == 0) { + transaction.abort().get(); + abortedTxns.add(messageId); + abortedTxnSize++; + } else { + ongoingTxns.put(transaction, messageId); + if (maxReadMessage == 19) { + //The except number of the messages that can be read + maxReadMessage = j - abortedTxnSize; + } + } + } else { + MessageId messageId = producer.newMessage().value(i * 10 + j).send(); + transaction.commit().get(); + } + } + for (int k = 0; k < maxReadMessage; k++) { + Message message = consumer.receive(2, TimeUnit.SECONDS); + assertNotNull(message); + assertFalse(abortedTxns.contains(message.getMessageId())); + } + Message message = consumer.receive(2, TimeUnit.SECONDS); + assertNull(message); + + for (Transaction ongoingTxn: ongoingTxns.keySet()) { + ongoingTxn.commit().get(); + } + ongoingTxns.clear(); + for (int k = maxReadMessage; k < 20 - abortedTxnSize; k++) { + message = consumer.receive(2, TimeUnit.SECONDS); + assertNotNull(message); + assertFalse(abortedTxns.contains(message.getMessageId())); + } + } + + admin.topics().unload(topic); + + for (int i = 0; i < 200 - abortedTxns.size(); i++) { + Message message = consumer.receive(2, TimeUnit.SECONDS); + assertNotNull(message); + assertFalse(abortedTxns.contains(message.getMessageId())); + } + } + } From 6cc07cbaa5e9deb3be71b09ef8f926fc29d285a3 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Tue, 1 Nov 2022 11:10:21 +0800 Subject: [PATCH 02/41] optimize --- ...napshotSegmentAbortedTxnProcessorImpl.java | 71 +++++++++---------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 1ca7a09e79192..3102a01936aab 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -147,11 +147,13 @@ private String buildKey(long sequenceId) { @Override public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosition) { - ConcurrentOpenHashSet aborts = unsealedAbortedTxnIdSegment; + TransactionBufferSnapshotIndexesMetadata metadata = new TransactionBufferSnapshotIndexesMetadata( + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), + serializationForSegment(unsealedAbortedTxnIdSegment)); CompletableFuture completableFuture = new CompletableFuture<>(); persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> persistentWorker - .updateIndexMetadataForTheLastSnapshot(maxReadPosition, aborts) + .updateSnapshotIndex(metadata, persistentSnapshotIndexes.getIndexList()) .thenRun(() -> completableFuture.complete(null)) .exceptionally(e -> { completableFuture.completeExceptionally(e); @@ -262,22 +264,21 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob //Wait the processor recover completely and the allow TB to recover the messages // after the startReadCursorPosition. - return openManagedLedgerFuture.thenCompose((ignore) -> { - return FutureUtil.waitForAll(completableFutures).thenCompose((i) -> { - if (invalidIndex.get() != 0 ) { - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () - -> persistentWorker.updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), - indexes.values().stream().toList())); - } - return CompletableFuture.completedFuture(finalStartReadCursorPosition); - }); - }).exceptionally(ex -> { - log.error("[{}] Failed to recover snapshot segment", this.topic.getName(), ex); - return null; - }); - + return openManagedLedgerFuture + .thenCompose((ignore) -> FutureUtil.waitForAll(completableFutures).thenCompose((i) -> { + if (invalidIndex.get() != 0 ) { + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () + -> persistentWorker + .updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), + indexes.values().stream().toList())); + } + return CompletableFuture.completedFuture(finalStartReadCursorPosition); + })).exceptionally(ex -> { + log.error("[{}] Failed to recover snapshot segment", this.topic.getName(), ex); + return null; + }); - }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + }, topic.getBrokerService().getPulsar().getTransactionExecutorProvider() .getExecutor(this)); } @@ -423,7 +424,8 @@ private void executeTask() { } else { taskQueue.removeFirst(); } - STATE_UPDATER.compareAndSet(this, OperationState.WritingSegment, OperationState.None); + STATE_UPDATER.compareAndSet(this, + OperationState.WritingSegment, OperationState.None); }); } } @@ -443,7 +445,8 @@ private void executeTask() { taskQueue.removeFirst(); } - STATE_UPDATER.compareAndSet(this, OperationState.DeletingSegment, OperationState.None); + STATE_UPDATER.compareAndSet(this, + OperationState.DeletingSegment, OperationState.None); }); } } @@ -500,7 +503,8 @@ private CompletableFuture writeSnapshotSegmentAsync(ConcurrentOpenHashSet< private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedToDelete) { long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); - return snapshotSegmentsWriterFuture.thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) + return snapshotSegmentsWriterFuture + .thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) .thenRun(() -> { if (log.isDebugEnabled()) { log.debug("[{}] Successes to delete the snapshot segment, " @@ -521,7 +525,7 @@ private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedT }); } - //Update the indexes with the giving index snapshot and indexlist in the transactionBufferSnapshotIndexe. + //Update the indexes with the giving index snapshot and index list in the transactionBufferSnapshotIndexe. private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotIndexesMetadata snapshotSegment, List indexList) { TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); @@ -541,15 +545,6 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd }); } - //Only update the metadata in the transactionBufferSnapshotIndexes. - private CompletableFuture updateIndexMetadataForTheLastSnapshot(PositionImpl maxReadPosition, - ConcurrentOpenHashSet abortedTxns) { - TransactionBufferSnapshotIndexesMetadata metadata = new TransactionBufferSnapshotIndexesMetadata( - maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), serializationForSegment(abortedTxns)); - - return updateSnapshotIndex(metadata, persistentSnapshotIndexes.getIndexList()); - } - private CompletableFuture clearSnapshotSegmentAndIndexes() { ArrayList> completableFutures = new ArrayList<>(); //Delete all segment @@ -587,19 +582,19 @@ CompletableFuture closeAsync() { } private ConcurrentOpenHashSet deserializationFotSnapshotSegment(List snapshotSegment) { - ConcurrentOpenHashSet set = new ConcurrentOpenHashSet<>(); + ConcurrentOpenHashSet abortedTxns = new ConcurrentOpenHashSet<>(); snapshotSegment.forEach(txnIDData -> { - set.add(new TxnID(txnIDData.getMostSigBits(), txnIDData.getLeastSigBits())); + abortedTxns.add(new TxnID(txnIDData.getMostSigBits(), txnIDData.getLeastSigBits())); }); - return set; + return abortedTxns; } - private List serializationForSegment(ConcurrentOpenHashSet segment) { - List set = new LinkedList<>(); - segment.forEach(txnID -> { - set.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits())); + private List serializationForSegment(ConcurrentOpenHashSet abortedTxns) { + List segment = new LinkedList<>(); + abortedTxns.forEach(txnID -> { + segment.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits())); }); - return set; + return segment; } } \ No newline at end of file From 08c2da3654a00db01bd5c8bee86ccc6ce2a3377d Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Tue, 1 Nov 2022 14:36:25 +0800 Subject: [PATCH 03/41] checkstyle --- .../SnapshotSegmentAbortedTxnProcessorImpl.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 3102a01936aab..2206e798de710 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -66,11 +66,11 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess private ConcurrentOpenHashSet unsealedAbortedTxnIdSegment = new ConcurrentOpenHashSet<>(); //Store the fixed aborted transaction segment - private final ConcurrentSkipListMap> abortTxnSegments - = new ConcurrentSkipListMap<>(); + private final ConcurrentSkipListMap> abortTxnSegments = + new ConcurrentSkipListMap<>(); - private final ConcurrentSkipListMap indexes - = new ConcurrentSkipListMap<>(); + private final ConcurrentSkipListMap indexes = + new ConcurrentSkipListMap<>(); //The latest persistent snapshot index. This is used to combine new segment indexes with the latest metadata and // indexes. private TransactionBufferSnapshotIndexes persistentSnapshotIndexes = new TransactionBufferSnapshotIndexes(); @@ -214,7 +214,8 @@ public CompletableFuture recoverFromSnapshot() { AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks .OpenReadOnlyManagedLedgerCallback() { @Override - public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, + Object ctx) { persistentSnapshotIndexes.getIndexList().forEach(index -> { CompletableFuture handleSegmentFuture = new CompletableFuture<>(); completableFutures.add(handleSegmentFuture); @@ -266,7 +267,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob return openManagedLedgerFuture .thenCompose((ignore) -> FutureUtil.waitForAll(completableFutures).thenCompose((i) -> { - if (invalidIndex.get() != 0 ) { + if (invalidIndex.get() != 0) { persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> persistentWorker .updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), @@ -380,7 +381,7 @@ public void appendTask(OperationType operationType, Supplier { if (!taskQueue.isEmpty()) { return; - } else if(STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.UpdatingIndex)) { + } else if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.UpdatingIndex)) { lastOperationFuture = task.get(); lastOperationFuture.whenComplete((ignore, throwable) -> { if (throwable != null && log.isDebugEnabled()) { @@ -525,7 +526,7 @@ private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedT }); } - //Update the indexes with the giving index snapshot and index list in the transactionBufferSnapshotIndexe. + //Update the indexes with the giving index snapshot and index list in the transactionBufferSnapshotIndexes. private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotIndexesMetadata snapshotSegment, List indexList) { TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); From b298b0ee6825dbcdf422161d862fef3fe6f2b279 Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Thu, 3 Nov 2022 20:47:52 +0800 Subject: [PATCH 04/41] optimize --- .../impl/SnapshotSegmentAbortedTxnProcessorImpl.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 2206e798de710..0e9ed2e537c66 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -63,7 +63,7 @@ @Slf4j public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { - private ConcurrentOpenHashSet unsealedAbortedTxnIdSegment = new ConcurrentOpenHashSet<>(); + private ConcurrentOpenHashSet unsealedAbortedTxnIdSegment; //Store the fixed aborted transaction segment private final ConcurrentSkipListMap> abortTxnSegments = @@ -91,6 +91,8 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotSegmentSize(); + this.unsealedAbortedTxnIdSegment = + new ConcurrentOpenHashSet<>(this.transactionBufferMaxAbortedTxnsOfSnapshotSegment); } @Override @@ -98,9 +100,10 @@ public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl maxReadPos unsealedAbortedTxnIdSegment.add(abortedTxnId); //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { - abortTxnSegments.put(maxReadPosition, unsealedAbortedTxnIdSegment); + ConcurrentOpenHashSet abortedSegment = unsealedAbortedTxnIdSegment; + abortTxnSegments.put(maxReadPosition, abortedSegment); persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> - persistentWorker.takeSnapshotSegmentAsync(unsealedAbortedTxnIdSegment, maxReadPosition)); + persistentWorker.takeSnapshotSegmentAsync(abortedSegment, maxReadPosition)); unsealedAbortedTxnIdSegment = new ConcurrentOpenHashSet<>(); } } @@ -400,8 +403,7 @@ public void appendTask(OperationType operationType, Supplier { - lastOperationFuture = task.get(); - lastOperationFuture.thenRun(() -> + task.get().thenRun(() -> STATE_UPDATER.compareAndSet(this, OperationState.Closing, OperationState.Closed)); }); } From b6f8b475df66a377ecdb2a6525c73a532cecf8fe Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Sat, 5 Nov 2022 21:39:55 +0800 Subject: [PATCH 05/41] store the max persistent position of aborted txn marker in a segment for each segment --- .../buffer/AbortedTxnProcessor.java | 4 ++-- ...SingleSnapshotAbortedTxnProcessorImpl.java | 4 ++-- ...napshotSegmentAbortedTxnProcessorImpl.java | 20 ++++++++----------- .../buffer/impl/TopicTransactionBuffer.java | 2 +- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index e436e1df24972..9c173be8a1a70 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -30,9 +30,9 @@ public interface AbortedTxnProcessor { * After the transaction buffer writes a transaction aborted marker to the topic, * the transaction buffer will put the aborted txnID and the aborted marker position to AbortedTxnProcessor. * @param txnID aborted transaction ID. - * @param position the position of the abort txnID + * @param abortedMarkerPersistentPosition the position of the abort txn marker. */ - void putAbortedTxnAndPosition(TxnID txnID, PositionImpl position); + void putAbortedTxnAndPosition(TxnID txnID, PositionImpl abortedMarkerPersistentPosition); /** * Clean up invalid aborted transactions. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index a13dd0499a6bc..3fcc2257cc8ad 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -56,8 +56,8 @@ public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { } @Override - public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl position) { - aborts.put(abortedTxnId, position); + public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMarkerPersistentPosition) { + aborts.put(abortedTxnId, this.topic.getMaxReadPosition()); } //In this implementation we clear the invalid aborted txn ID one by one. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 0e9ed2e537c66..e4283d6dc2dd3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -92,19 +92,20 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotSegmentSize(); this.unsealedAbortedTxnIdSegment = - new ConcurrentOpenHashSet<>(this.transactionBufferMaxAbortedTxnsOfSnapshotSegment); + new ConcurrentOpenHashSet<>(this.transactionBufferMaxAbortedTxnsOfSnapshotSegment, 1); } @Override - public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl maxReadPosition) { + public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMarkerPersistentPosition) { unsealedAbortedTxnIdSegment.add(abortedTxnId); //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { ConcurrentOpenHashSet abortedSegment = unsealedAbortedTxnIdSegment; - abortTxnSegments.put(maxReadPosition, abortedSegment); + abortTxnSegments.put(abortedMarkerPersistentPosition, abortedSegment); persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> - persistentWorker.takeSnapshotSegmentAsync(abortedSegment, maxReadPosition)); - unsealedAbortedTxnIdSegment = new ConcurrentOpenHashSet<>(); + persistentWorker.takeSnapshotSegmentAsync(abortedSegment, this.topic.getMaxReadPosition())); + this.unsealedAbortedTxnIdSegment = + new ConcurrentOpenHashSet<>(this.transactionBufferMaxAbortedTxnsOfSnapshotSegment, 1); } } @@ -116,14 +117,9 @@ public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { } else { PositionImpl maxReadPosition = abortTxnSegments.ceilingKey((PositionImpl) readPosition); if (maxReadPosition != null) { - return abortTxnSegments.keySet().stream() - .filter((position) -> position.compareTo(maxReadPosition) <= 0) - .anyMatch((position -> abortTxnSegments.get(position).contains(txnID))); + return abortTxnSegments.get(maxReadPosition).contains(txnID); } else { - return abortTxnSegments.keySet().stream() - .filter((position) -> position.compareTo((PositionImpl) readPosition) <= 0) - .anyMatch((position -> abortTxnSegments.get(position).contains(txnID))) - || unsealedAbortedTxnIdSegment.contains(txnID); + return unsealedAbortedTxnIdSegment.contains(txnID); } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index c40e868739199..66b377934bddd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -354,7 +354,7 @@ public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { updateMaxReadPosition(txnID); - snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, maxReadPosition); + snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, (PositionImpl) position); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); takeSnapshotByChangeTimes(); } From eca6d336dcee661671f0d3ef2c5d901d8800cedb Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Sat, 5 Nov 2022 22:27:59 +0800 Subject: [PATCH 06/41] optimize test --- .../TopicTransactionBufferRecoverTest.java | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 2fcc26762a41f..b940a11fda466 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -87,6 +87,7 @@ import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; @@ -750,11 +751,14 @@ public void testSnapshotSegment() throws Exception { LinkedMap ongoingTxns = new LinkedMap<>(); LinkedList abortedTxns = new LinkedList<>(); - + // 0. Modify the configurations + int theSizeOfSegment = 10; + int theCountOfSnapshotMaxTxnCount = 3; this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); - this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSnapshotSegmentSize(10); - this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSnapshotMaxTransactionCount(3); - + this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSnapshotSegmentSize(theSizeOfSegment); + this.getPulsarServiceList().get(0).getConfig() + .setTransactionBufferSnapshotMaxTransactionCount(theCountOfSnapshotMaxTxnCount); + // 1. Build prodcuer and consumer Producer producer = pulsarClient.newProducer(Schema.INT32) .topic(topic) .enableBatching(false) @@ -766,14 +770,16 @@ public void testSnapshotSegment() throws Exception { .subscriptionType(SubscriptionType.Exclusive) .subscribe(); + // 2. Check the AbortedTxnProcessor workflow 10 times for (int i = 0; i < 10; i++) { - int maxReadMessage = 19; + MessageId maxReadMessage = null; int abortedTxnSize = 0; - for (int j = 0; j < 20; j++) { + // The number of aborted transaction = 30 / 2, that is more than the size of snapshot segment 10. + for (int j = 0; j < theSizeOfSegment * 4; j++) { Transaction transaction = pulsarClient.newTransaction() .withTransactionTimeout(5, TimeUnit.MINUTES).build().get(); - //half common message and half transaction message. - //the transaction message have a half which are aborted. + //Half common message and half transaction message. + //And the transaction message have a half which are aborted. if (RandomUtils.nextInt() % 2 == 0) { MessageId messageId = producer.newMessage(transaction).value(i * 10 + j).send(); if (RandomUtils.nextInt() % 2 == 0) { @@ -782,9 +788,10 @@ public void testSnapshotSegment() throws Exception { abortedTxnSize++; } else { ongoingTxns.put(transaction, messageId); - if (maxReadMessage == 19) { + if (maxReadMessage == null) { + log.info("Max read Position in test: [{}]", messageId); //The except number of the messages that can be read - maxReadMessage = j - abortedTxnSize; + maxReadMessage = messageId; } } } else { @@ -792,25 +799,30 @@ public void testSnapshotSegment() throws Exception { transaction.commit().get(); } } - for (int k = 0; k < maxReadMessage; k++) { + // 2.1 Check the updating of the maxReadPosition + int hasReceived = 0; + while (true) { Message message = consumer.receive(2, TimeUnit.SECONDS); - assertNotNull(message); - assertFalse(abortedTxns.contains(message.getMessageId())); + if (message != null) { + Assert.assertTrue(message.getMessageId().compareTo(maxReadMessage) < 0); + hasReceived ++; + } else { + break; + } } - Message message = consumer.receive(2, TimeUnit.SECONDS); - assertNull(message); for (Transaction ongoingTxn: ongoingTxns.keySet()) { ongoingTxn.commit().get(); } ongoingTxns.clear(); - for (int k = maxReadMessage; k < 20 - abortedTxnSize; k++) { - message = consumer.receive(2, TimeUnit.SECONDS); + // 2.2 Check the aborted txn + for (int k = hasReceived; k < theSizeOfSegment * 4 - abortedTxnSize; k++) { + Message message = consumer.receive(2, TimeUnit.SECONDS); assertNotNull(message); assertFalse(abortedTxns.contains(message.getMessageId())); } } - + // 3. Test recover admin.topics().unload(topic); for (int i = 0; i < 200 - abortedTxns.size(); i++) { From 564b6cdfbc4f30d4c26950204e63d98cd4ffcf7e Mon Sep 17 00:00:00 2001 From: liangyepianzhou Date: Mon, 14 Nov 2022 19:38:45 +0800 Subject: [PATCH 07/41] fix some comments --- ...napshotSegmentAbortedTxnProcessorImpl.java | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index e4283d6dc2dd3..a858c555fab78 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentSkipListMap; @@ -66,7 +67,7 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess private ConcurrentOpenHashSet unsealedAbortedTxnIdSegment; //Store the fixed aborted transaction segment - private final ConcurrentSkipListMap> abortTxnSegments = + private final ConcurrentSkipListMap> segmentMap = new ConcurrentSkipListMap<>(); private final ConcurrentSkipListMap indexes = @@ -101,7 +102,7 @@ public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMar //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { ConcurrentOpenHashSet abortedSegment = unsealedAbortedTxnIdSegment; - abortTxnSegments.put(abortedMarkerPersistentPosition, abortedSegment); + segmentMap.put(abortedMarkerPersistentPosition, abortedSegment); persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> persistentWorker.takeSnapshotSegmentAsync(abortedSegment, this.topic.getMaxReadPosition())); this.unsealedAbortedTxnIdSegment = @@ -112,12 +113,13 @@ public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMar @Override public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { if (readPosition == null) { - return abortTxnSegments.values().stream() + return segmentMap.values().stream() .anyMatch(list -> list.contains(txnID)) || unsealedAbortedTxnIdSegment.contains(txnID); } else { - PositionImpl maxReadPosition = abortTxnSegments.ceilingKey((PositionImpl) readPosition); - if (maxReadPosition != null) { - return abortTxnSegments.get(maxReadPosition).contains(txnID); + Map.Entry> cellingEntry = + segmentMap.ceilingEntry((PositionImpl) readPosition); + if (cellingEntry != null) { + return cellingEntry.getValue().contains(txnID); } else { return unsealedAbortedTxnIdSegment.contains(txnID); } @@ -128,13 +130,13 @@ public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { @Override public void trimExpiredAbortedTxns() { //Checking whether there are some segment expired. - while (!abortTxnSegments.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) - .ledgerExists(abortTxnSegments.firstKey().getLedgerId())) { + while (!segmentMap.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) + .ledgerExists(segmentMap.firstKey().getLedgerId())) { if (log.isDebugEnabled()) { log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", - topic.getName(), abortTxnSegments.firstKey()); + topic.getName(), segmentMap.firstKey()); } - PositionImpl positionNeedToDelete = abortTxnSegments.firstKey(); + PositionImpl positionNeedToDelete = segmentMap.firstKey(); persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment, () -> persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); } @@ -148,7 +150,7 @@ private String buildKey(long sequenceId) { public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosition) { TransactionBufferSnapshotIndexesMetadata metadata = new TransactionBufferSnapshotIndexesMetadata( maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), - serializationForSegment(unsealedAbortedTxnIdSegment)); + convertTypeToTxnIDData(unsealedAbortedTxnIdSegment)); CompletableFuture completableFuture = new CompletableFuture<>(); persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> persistentWorker @@ -200,7 +202,7 @@ public CompletableFuture recoverFromSnapshot() { transactionBufferSnapshotIndex.persistentPositionLedgerID, transactionBufferSnapshotIndex.persistentPositionEntryID), transactionBufferSnapshotIndex)); - this.unsealedAbortedTxnIdSegment = deserializationFotSnapshotSegment(persistentSnapshotIndexes + this.unsealedAbortedTxnIdSegment = convertTypeToTxnID(persistentSnapshotIndexes .getSnapshot().getAborts()); if (indexes.size() != 0) { persistentWorker.sequenceID.set(indexes.lastEntry().getValue().sequenceID + 1); @@ -313,8 +315,8 @@ private void handleSnapshotSegmentEntry(Entry entry) { Commands.parseMessageMetadata(headersAndPayload); TransactionBufferSnapshotSegment snapshotSegment = Schema.AVRO(TransactionBufferSnapshotSegment.class) .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer()); - abortTxnSegments.put(new PositionImpl(snapshotSegment.getMaxReadPositionLedgerId(), - snapshotSegment.getMaxReadPositionEntryId()), deserializationFotSnapshotSegment( + segmentMap.put(new PositionImpl(snapshotSegment.getMaxReadPositionLedgerId(), + snapshotSegment.getMaxReadPositionEntryId()), convertTypeToTxnID( snapshotSegment.getAborts())); } @@ -475,7 +477,7 @@ private CompletableFuture takeSnapshotSegmentAsync(ConcurrentOpenHashSet writeSnapshotSegmentAsync(ConcurrentOpenHashSet segment, PositionImpl maxReadPosition) { TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = new TransactionBufferSnapshotSegment(); - transactionBufferSnapshotSegment.setAborts(serializationForSegment(segment)); + transactionBufferSnapshotSegment.setAborts(convertTypeToTxnIDData(segment)); transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); transactionBufferSnapshotSegment.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); transactionBufferSnapshotSegment.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); @@ -510,7 +512,7 @@ private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedT + "whose sequenceId is [{}] and maxReadPosition is [{}]", this.topic.getName(), this.sequenceID, positionNeedToDelete); } - abortTxnSegments.remove(positionNeedToDelete); + segmentMap.remove(positionNeedToDelete); //The process will check whether the snapshot segment is null, and update index when recovered. indexes.remove(positionNeedToDelete); //Keep index snapshot and update index @@ -547,14 +549,13 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd private CompletableFuture clearSnapshotSegmentAndIndexes() { ArrayList> completableFutures = new ArrayList<>(); //Delete all segment - while (!abortTxnSegments.isEmpty()) { + segmentMap.keySet().forEach(positionNeedToDelete -> { if (log.isDebugEnabled()) { log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", - topic.getName(), abortTxnSegments.firstKey()); + topic.getName(), positionNeedToDelete); } - PositionImpl positionNeedToDelete = abortTxnSegments.firstKey(); completableFutures.add(persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); - } + }); //Delete index return FutureUtil.waitForAll(completableFutures) .thenCompose((ignore) -> snapshotIndexWriterFuture @@ -580,7 +581,7 @@ CompletableFuture closeAsync() { } } - private ConcurrentOpenHashSet deserializationFotSnapshotSegment(List snapshotSegment) { + private ConcurrentOpenHashSet convertTypeToTxnID(List snapshotSegment) { ConcurrentOpenHashSet abortedTxns = new ConcurrentOpenHashSet<>(); snapshotSegment.forEach(txnIDData -> { abortedTxns.add(new TxnID(txnIDData.getMostSigBits(), txnIDData.getLeastSigBits())); @@ -588,7 +589,7 @@ private ConcurrentOpenHashSet deserializationFotSnapshotSegment(List serializationForSegment(ConcurrentOpenHashSet abortedTxns) { + private List convertTypeToTxnIDData(ConcurrentOpenHashSet abortedTxns) { List segment = new LinkedList<>(); abortedTxns.forEach(txnID -> { segment.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits())); From 52ad0ae668ed296fea362bfd7d5eb412f984c361 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Tue, 20 Dec 2022 12:01:48 +0800 Subject: [PATCH 08/41] fix some comments --- .../impl/SnapshotSegmentAbortedTxnProcessorImpl.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index a858c555fab78..90c5b6a35fc76 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -85,6 +85,8 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess private final int transactionBufferMaxAbortedTxnsOfSnapshotSegment; private final PersistentWorker persistentWorker; + private static final String SNAPSHOT_PREFIX = "multiple-"; + public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; this.persistentWorker = new PersistentWorker(topic); @@ -113,8 +115,8 @@ public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMar @Override public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { if (readPosition == null) { - return segmentMap.values().stream() - .anyMatch(list -> list.contains(txnID)) || unsealedAbortedTxnIdSegment.contains(txnID); + return unsealedAbortedTxnIdSegment.contains(txnID) || segmentMap.values().stream() + .anyMatch(list -> list.contains(txnID)); } else { Map.Entry> cellingEntry = segmentMap.ceilingEntry((PositionImpl) readPosition); @@ -143,7 +145,7 @@ public void trimExpiredAbortedTxns() { } private String buildKey(long sequenceId) { - return "multiple-" + sequenceId + "-" + this.topic.getName(); + return SNAPSHOT_PREFIX + sequenceId + "-" + this.topic.getName(); } @Override From 291d9e28313626f650e4cf5d2be1b13460ee98ac Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Tue, 20 Dec 2022 16:56:40 +0800 Subject: [PATCH 09/41] use single set to check aborted transaction ID. --- ...napshotSegmentAbortedTxnProcessorImpl.java | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 90c5b6a35fc76..c9bd3eddd4882 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -22,9 +22,9 @@ import io.netty.buffer.Unpooled; import io.netty.util.Timer; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentSkipListMap; @@ -70,6 +70,8 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess private final ConcurrentSkipListMap> segmentMap = new ConcurrentSkipListMap<>(); + private final LinkedHashSet aborts = new LinkedHashSet<>(); + private final ConcurrentSkipListMap indexes = new ConcurrentSkipListMap<>(); //The latest persistent snapshot index. This is used to combine new segment indexes with the latest metadata and @@ -101,6 +103,7 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { @Override public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMarkerPersistentPosition) { unsealedAbortedTxnIdSegment.add(abortedTxnId); + aborts.add(abortedTxnId); //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { ConcurrentOpenHashSet abortedSegment = unsealedAbortedTxnIdSegment; @@ -114,18 +117,7 @@ public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMar @Override public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { - if (readPosition == null) { - return unsealedAbortedTxnIdSegment.contains(txnID) || segmentMap.values().stream() - .anyMatch(list -> list.contains(txnID)); - } else { - Map.Entry> cellingEntry = - segmentMap.ceilingEntry((PositionImpl) readPosition); - if (cellingEntry != null) { - return cellingEntry.getValue().contains(txnID); - } else { - return unsealedAbortedTxnIdSegment.contains(txnID); - } - } + return aborts.contains(txnID); } //In this implementation, we adopt snapshot segments. And then we clear invalid segment by its max read position. @@ -139,6 +131,7 @@ public void trimExpiredAbortedTxns() { topic.getName(), segmentMap.firstKey()); } PositionImpl positionNeedToDelete = segmentMap.firstKey(); + segmentMap.get(positionNeedToDelete).forEach(aborts::remove); persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment, () -> persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); } @@ -320,7 +313,7 @@ private void handleSnapshotSegmentEntry(Entry entry) { segmentMap.put(new PositionImpl(snapshotSegment.getMaxReadPositionLedgerId(), snapshotSegment.getMaxReadPositionEntryId()), convertTypeToTxnID( snapshotSegment.getAborts())); - + convertTypeToTxnID(snapshotSegment.getAborts()).forEach(aborts::add); } private void closeReader(SystemTopicClient.Reader reader) { From 20bcd501e500bd816be2b8c47a8318a09b5154f0 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Tue, 20 Dec 2022 19:20:31 +0800 Subject: [PATCH 10/41] use single set to check aborted transaction ID. --- ...napshotSegmentAbortedTxnProcessorImpl.java | 73 ++++++++++--------- .../v2/TransactionBufferSnapshotSegment.java | 4 +- .../TopicTransactionBufferRecoverTest.java | 8 +- 3 files changed, 43 insertions(+), 42 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index c9bd3eddd4882..6c10eef3dd54a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -22,7 +22,6 @@ import io.netty.buffer.Unpooled; import io.netty.util.Timer; import java.util.ArrayList; -import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -59,18 +58,16 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; @Slf4j public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { - private ConcurrentOpenHashSet unsealedAbortedTxnIdSegment; + private LinkedList unsealedAbortedTxnIdSegment; //Store the fixed aborted transaction segment - private final ConcurrentSkipListMap> segmentMap = - new ConcurrentSkipListMap<>(); + private final ConcurrentSkipListMap segmentIndex = new ConcurrentSkipListMap<>(); - private final LinkedHashSet aborts = new LinkedHashSet<>(); + private final LinkedList aborts = new LinkedList<>(); private final ConcurrentSkipListMap indexes = new ConcurrentSkipListMap<>(); @@ -96,8 +93,7 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotSegmentSize(); - this.unsealedAbortedTxnIdSegment = - new ConcurrentOpenHashSet<>(this.transactionBufferMaxAbortedTxnsOfSnapshotSegment, 1); + this.unsealedAbortedTxnIdSegment = new LinkedList<>(); } @Override @@ -106,12 +102,11 @@ public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMar aborts.add(abortedTxnId); //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { - ConcurrentOpenHashSet abortedSegment = unsealedAbortedTxnIdSegment; - segmentMap.put(abortedMarkerPersistentPosition, abortedSegment); + LinkedList abortedSegment = unsealedAbortedTxnIdSegment; + segmentIndex.put(abortedMarkerPersistentPosition, abortedTxnId); persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> - persistentWorker.takeSnapshotSegmentAsync(abortedSegment, this.topic.getMaxReadPosition())); - this.unsealedAbortedTxnIdSegment = - new ConcurrentOpenHashSet<>(this.transactionBufferMaxAbortedTxnsOfSnapshotSegment, 1); + persistentWorker.takeSnapshotSegmentAsync(abortedSegment, abortedMarkerPersistentPosition)); + this.unsealedAbortedTxnIdSegment = new LinkedList<>(); } } @@ -124,14 +119,13 @@ public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { @Override public void trimExpiredAbortedTxns() { //Checking whether there are some segment expired. - while (!segmentMap.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) - .ledgerExists(segmentMap.firstKey().getLedgerId())) { + while (!segmentIndex.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) + .ledgerExists(segmentIndex.firstKey().getLedgerId())) { if (log.isDebugEnabled()) { log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", - topic.getName(), segmentMap.firstKey()); + topic.getName(), segmentIndex.firstKey()); } - PositionImpl positionNeedToDelete = segmentMap.firstKey(); - segmentMap.get(positionNeedToDelete).forEach(aborts::remove); + PositionImpl positionNeedToDelete = segmentIndex.firstKey(); persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment, () -> persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); } @@ -310,10 +304,12 @@ private void handleSnapshotSegmentEntry(Entry entry) { Commands.parseMessageMetadata(headersAndPayload); TransactionBufferSnapshotSegment snapshotSegment = Schema.AVRO(TransactionBufferSnapshotSegment.class) .decode(Unpooled.wrappedBuffer(headersAndPayload).nioBuffer()); - segmentMap.put(new PositionImpl(snapshotSegment.getMaxReadPositionLedgerId(), - snapshotSegment.getMaxReadPositionEntryId()), convertTypeToTxnID( - snapshotSegment.getAborts())); - convertTypeToTxnID(snapshotSegment.getAborts()).forEach(aborts::add); + + TxnIDData lastTxn = snapshotSegment.getAborts().get(snapshotSegment.getAborts().size() - 1); + segmentIndex.put(new PositionImpl(snapshotSegment.getPersistentPositionLedgerId(), + snapshotSegment.getPersistentPositionEntryId()), + new TxnID(lastTxn.getMostSigBits(), lastTxn.getLeastSigBits())); + aborts.addAll(convertTypeToTxnID(snapshotSegment.getAborts())); } private void closeReader(SystemTopicClient.Reader reader) { @@ -450,13 +446,13 @@ private void executeTask() { } } - private CompletableFuture takeSnapshotSegmentAsync(ConcurrentOpenHashSet sealedAbortedTxnIdSegment, - PositionImpl maxReadPosition) { - return writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, maxReadPosition).thenRun(() -> { + private CompletableFuture takeSnapshotSegmentAsync(LinkedList sealedAbortedTxnIdSegment, + PositionImpl abortedMarkerPersistentPosition) { + return writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, abortedMarkerPersistentPosition).thenRun(() -> { if (log.isDebugEnabled()) { log.debug("Successes to take snapshot segment [{}] at maxReadPosition [{}] " + "for the topic [{}], and the size of the segment is [{}]", - this.sequenceID, maxReadPosition, topic.getName(), sealedAbortedTxnIdSegment.size()); + this.sequenceID, abortedMarkerPersistentPosition, topic.getName(), sealedAbortedTxnIdSegment.size()); } this.sequenceID.getAndIncrement(); }).exceptionally(e -> { @@ -464,23 +460,24 @@ private CompletableFuture takeSnapshotSegmentAsync(ConcurrentOpenHashSet writeSnapshotSegmentAsync(ConcurrentOpenHashSet segment, - PositionImpl maxReadPosition) { + private CompletableFuture writeSnapshotSegmentAsync(LinkedList segment, + PositionImpl abortedMarkerPersistentPosition) { TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = new TransactionBufferSnapshotSegment(); transactionBufferSnapshotSegment.setAborts(convertTypeToTxnIDData(segment)); transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); - transactionBufferSnapshotSegment.setMaxReadPositionEntryId(maxReadPosition.getEntryId()); - transactionBufferSnapshotSegment.setMaxReadPositionLedgerId(maxReadPosition.getLedgerId()); + transactionBufferSnapshotSegment.setPersistentPositionEntryId(abortedMarkerPersistentPosition.getEntryId()); + transactionBufferSnapshotSegment.setPersistentPositionLedgerId(abortedMarkerPersistentPosition.getLedgerId()); return snapshotSegmentsWriterFuture.thenCompose(segmentWriter -> { transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get()); return segmentWriter.writeAsync(buildKey(this.sequenceID.get()), transactionBufferSnapshotSegment); }).thenCompose((messageId) -> { + PositionImpl maxReadPosition = topic.getMaxReadPosition(); //Build index for this segment TransactionBufferSnapshotIndex index = new TransactionBufferSnapshotIndex(); index.setSequenceID(transactionBufferSnapshotSegment.getSequenceId()); @@ -507,7 +504,11 @@ private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedT + "whose sequenceId is [{}] and maxReadPosition is [{}]", this.topic.getName(), this.sequenceID, positionNeedToDelete); } - segmentMap.remove(positionNeedToDelete); + TxnID theLatestDeletedTxnID = segmentIndex.remove(positionNeedToDelete); + while (!aborts.getFirst().equals(theLatestDeletedTxnID)) { + aborts.removeFirst(); + } + aborts.remove(theLatestDeletedTxnID); //The process will check whether the snapshot segment is null, and update index when recovered. indexes.remove(positionNeedToDelete); //Keep index snapshot and update index @@ -544,7 +545,7 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd private CompletableFuture clearSnapshotSegmentAndIndexes() { ArrayList> completableFutures = new ArrayList<>(); //Delete all segment - segmentMap.keySet().forEach(positionNeedToDelete -> { + segmentIndex.keySet().forEach(positionNeedToDelete -> { if (log.isDebugEnabled()) { log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", topic.getName(), positionNeedToDelete); @@ -576,15 +577,15 @@ CompletableFuture closeAsync() { } } - private ConcurrentOpenHashSet convertTypeToTxnID(List snapshotSegment) { - ConcurrentOpenHashSet abortedTxns = new ConcurrentOpenHashSet<>(); + private LinkedList convertTypeToTxnID(List snapshotSegment) { + LinkedList abortedTxns = new LinkedList<>(); snapshotSegment.forEach(txnIDData -> { abortedTxns.add(new TxnID(txnIDData.getMostSigBits(), txnIDData.getLeastSigBits())); }); return abortedTxns; } - private List convertTypeToTxnIDData(ConcurrentOpenHashSet abortedTxns) { + private List convertTypeToTxnIDData(LinkedList abortedTxns) { List segment = new LinkedList<>(); abortedTxns.forEach(txnID -> { segment.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits())); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java index 77bb546880dbe..7ca828cc3e64f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotSegment.java @@ -29,7 +29,7 @@ public class TransactionBufferSnapshotSegment { private String topicName; private long sequenceId; - private long maxReadPositionLedgerId; - private long maxReadPositionEntryId; + private long persistentPositionLedgerId; + private long persistentPositionEntryId; private List aborts; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index b940a11fda466..601b86f1810ae 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -686,8 +686,8 @@ public void testTransactionBufferSegmentSystemTopic() throws Exception { //build and send snapshot snapshot.setTopicName(snapshotTopic); snapshot.setSequenceId(1L); - snapshot.setMaxReadPositionLedgerId(2L); - snapshot.setMaxReadPositionEntryId(3L); + snapshot.setPersistentPositionLedgerId(2L); + snapshot.setPersistentPositionEntryId(3L); LinkedList txnIDSet = new LinkedList<>(); txnIDSet.add(new TxnIDData(1, 1)); snapshot.setAborts(txnIDSet ); @@ -739,8 +739,8 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob //verify snapshot assertEquals(snapshot.getTopicName(), snapshotTopic); assertEquals(snapshot.getSequenceId(), 2L); - assertEquals(snapshot.getMaxReadPositionLedgerId(), 2L); - assertEquals(snapshot.getMaxReadPositionEntryId(), 3L); + assertEquals(snapshot.getPersistentPositionLedgerId(), 2L); + assertEquals(snapshot.getPersistentPositionEntryId(), 3L); assertEquals(snapshot.getAborts().toArray()[0], new TxnIDData(1, 1)); } From 80d87b9fcc1023fa488ba8b6b5068462bbbb9193 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Thu, 29 Dec 2022 16:36:08 +0800 Subject: [PATCH 11/41] fix some comments --- ...napshotSegmentAbortedTxnProcessorImpl.java | 94 +++++++++++-------- 1 file changed, 56 insertions(+), 38 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 6c10eef3dd54a..c218b75052f79 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -28,6 +28,8 @@ import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.function.Supplier; @@ -41,6 +43,7 @@ import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; @@ -53,6 +56,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.events.EventType; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; @@ -119,6 +123,7 @@ public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { @Override public void trimExpiredAbortedTxns() { //Checking whether there are some segment expired. + List positionsNeedToDelete = new ArrayList<>(); while (!segmentIndex.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) .ledgerExists(segmentIndex.firstKey().getLedgerId())) { if (log.isDebugEnabled()) { @@ -126,9 +131,10 @@ public void trimExpiredAbortedTxns() { topic.getName(), segmentIndex.firstKey()); } PositionImpl positionNeedToDelete = segmentIndex.firstKey(); - persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment, - () -> persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); + positionsNeedToDelete.add(positionNeedToDelete); } + persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment, + () -> persistentWorker.deleteSnapshotSegment(positionsNeedToDelete)); } private String buildKey(long sequenceId) { @@ -162,7 +168,8 @@ public CompletableFuture recoverFromSnapshot() { try { //Read Index to recover the sequenceID, indexes, lastAbortedTxns and maxReadPosition. while (reader.hasMoreEvents()) { - Message message = reader.readNext(); + Message message = reader.readNextAsync() + .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); if (topic.getName().equals(message.getKey())) { TransactionBufferSnapshotIndexes transactionBufferSnapshotIndexes = message.getValue(); if (transactionBufferSnapshotIndexes != null) { @@ -174,6 +181,13 @@ public CompletableFuture recoverFromSnapshot() { } } } + } catch (TimeoutException ex) { + Throwable t = FutureUtil.unwrapCompletionException(ex); + String errorMessage = String.format("[%s] Transaction buffer recover fail by read " + + "transactionBufferSnapshot timeout!", topic.getName()); + log.error(errorMessage, t); + return FutureUtil.failedFuture( + new BrokerServiceException.ServiceUnitNotReadyException(errorMessage, t)); } catch (Exception ex) { log.error("[{}] Transaction buffer recover fail when read " + "transactionBufferSnapshot!", topic.getName(), ex); @@ -200,7 +214,7 @@ public CompletableFuture recoverFromSnapshot() { //Read snapshot segment to recover aborts. ArrayList> completableFutures = new ArrayList<>(); CompletableFuture openManagedLedgerFuture = new CompletableFuture<>(); - AtomicLong invalidIndex = new AtomicLong(0); + AtomicBoolean hasInvalidIndex = new AtomicBoolean(false); AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks .OpenReadOnlyManagedLedgerCallback() { @Override @@ -221,7 +235,7 @@ public void readEntryComplete(Entry entry, Object ctx) { index.getMaxReadPositionLedgerID(), index.getMaxReadPositionEntryID())); handleSegmentFuture.complete(null); - invalidIndex.getAndIncrement(); + hasInvalidIndex.set(true); return; } handleSnapshotSegmentEntry(entry); @@ -257,7 +271,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob return openManagedLedgerFuture .thenCompose((ignore) -> FutureUtil.waitForAll(completableFutures).thenCompose((i) -> { - if (invalidIndex.get() != 0) { + if (hasInvalidIndex.get()) { persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> persistentWorker .updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), @@ -312,6 +326,11 @@ private void handleSnapshotSegmentEntry(Entry entry) { aborts.addAll(convertTypeToTxnID(snapshotSegment.getAborts())); } + private long getSystemClientOperationTimeoutMs() throws Exception { + PulsarClientImpl pulsarClient = (PulsarClientImpl) topic.getBrokerService().getPulsar().getClient(); + return pulsarClient.getConfiguration().getOperationTimeoutMs(); + } + private void closeReader(SystemTopicClient.Reader reader) { reader.closeAsync().exceptionally(e -> { log.error("[{}]Transaction buffer snapshot reader close error!", topic.getName(), e); @@ -494,32 +513,37 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm }); } - private CompletableFuture deleteSnapshotSegment(PositionImpl positionNeedToDelete) { - long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); - return snapshotSegmentsWriterFuture - .thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) - .thenRun(() -> { - if (log.isDebugEnabled()) { - log.debug("[{}] Successes to delete the snapshot segment, " + private CompletableFuture deleteSnapshotSegment(List positionNeedToDeletes) { + List> results = new ArrayList<>(); + for (PositionImpl positionNeedToDelete : positionNeedToDeletes) { + long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); + results.add(snapshotSegmentsWriterFuture + .thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) + .thenRun(() -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Successes to delete the snapshot segment, " + + "whose sequenceId is [{}] and maxReadPosition is [{}]", + this.topic.getName(), this.sequenceID, positionNeedToDelete); + } + TxnID theLatestDeletedTxnID = segmentIndex.remove(positionNeedToDelete); + while (!aborts.getFirst().equals(theLatestDeletedTxnID)) { + aborts.removeFirst(); + } + aborts.remove(theLatestDeletedTxnID); + //The process will check whether the snapshot segment is null, and update index when + // recovered. + indexes.remove(positionNeedToDelete); + //Keep index snapshot and update index + updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), + indexes.values().stream().toList()); + }).exceptionally(e -> { + log.warn("[{}] Failed to delete the snapshot segment, " + "whose sequenceId is [{}] and maxReadPosition is [{}]", - this.topic.getName(), this.sequenceID, positionNeedToDelete); - } - TxnID theLatestDeletedTxnID = segmentIndex.remove(positionNeedToDelete); - while (!aborts.getFirst().equals(theLatestDeletedTxnID)) { - aborts.removeFirst(); - } - aborts.remove(theLatestDeletedTxnID); - //The process will check whether the snapshot segment is null, and update index when recovered. - indexes.remove(positionNeedToDelete); - //Keep index snapshot and update index - updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), - indexes.values().stream().toList()); - }).exceptionally(e -> { - log.warn("[{}] Failed to delete the snapshot segment, " - + "whose sequenceId is [{}] and maxReadPosition is [{}]", - this.topic.getName(), this.sequenceID, positionNeedToDelete, e); - return null; - }); + this.topic.getName(), this.sequenceID, positionNeedToDelete, e); + return null; + })); + } + return FutureUtil.waitForAll(results); } //Update the indexes with the giving index snapshot and index list in the transactionBufferSnapshotIndexes. @@ -545,13 +569,7 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd private CompletableFuture clearSnapshotSegmentAndIndexes() { ArrayList> completableFutures = new ArrayList<>(); //Delete all segment - segmentIndex.keySet().forEach(positionNeedToDelete -> { - if (log.isDebugEnabled()) { - log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", - topic.getName(), positionNeedToDelete); - } - completableFutures.add(persistentWorker.deleteSnapshotSegment(positionNeedToDelete)); - }); + completableFutures.add(persistentWorker.deleteSnapshotSegment(segmentIndex.keySet().stream().toList())); //Delete index return FutureUtil.waitForAll(completableFutures) .thenCompose((ignore) -> snapshotIndexWriterFuture From e2664cfe5892553c311373976c89a65909743e31 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Thu, 29 Dec 2022 18:12:13 +0800 Subject: [PATCH 12/41] fix some comments --- .../buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 1614d2485e41f..82843d8997cd7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -61,7 +61,7 @@ public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { @Override public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMarkerPersistentPosition) { - aborts.put(abortedTxnId, this.topic.getMaxReadPosition()); + aborts.put(abortedTxnId, abortedMarkerPersistentPosition); } //In this implementation we clear the invalid aborted txn ID one by one. From 906d9426243f12f7a8107b9720d637d0df22beeb Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Thu, 29 Dec 2022 22:23:50 +0800 Subject: [PATCH 13/41] optimize data struct --- ...napshotSegmentAbortedTxnProcessorImpl.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index c218b75052f79..1140ba324fca3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -41,6 +41,7 @@ import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; +import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -66,15 +67,18 @@ @Slf4j public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { + //Stored the unsealed aborted txn id, it will be persistent as a snapshot segment and reinit + // when its size reach the capital of a snapshot segment. private LinkedList unsealedAbortedTxnIdSegment; - //Store the fixed aborted transaction segment - private final ConcurrentSkipListMap segmentIndex = new ConcurrentSkipListMap<>(); + //A mapping form the latest txn mark persistent position in a segment to its txn ID. + //This is mainly used to trim expired snapshot segment. + private final LinkedMap segmentIndex = new LinkedMap<>(); - private final LinkedList aborts = new LinkedList<>(); - - private final ConcurrentSkipListMap indexes = - new ConcurrentSkipListMap<>(); + //Store all aborted txn IDs check whether a txn is an aborted txn. + private final LinkedMap aborts = new LinkedMap<>(); + //The indexes of the snapshot segments. + private final LinkedMap indexes = new LinkedMap<>(); //The latest persistent snapshot index. This is used to combine new segment indexes with the latest metadata and // indexes. private TransactionBufferSnapshotIndexes persistentSnapshotIndexes = new TransactionBufferSnapshotIndexes(); @@ -103,7 +107,7 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { @Override public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMarkerPersistentPosition) { unsealedAbortedTxnIdSegment.add(abortedTxnId); - aborts.add(abortedTxnId); + aborts.put(abortedTxnId, abortedTxnId); //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { LinkedList abortedSegment = unsealedAbortedTxnIdSegment; @@ -116,7 +120,7 @@ public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMar @Override public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { - return aborts.contains(txnID); + return aborts.containsKey(txnID); } //In this implementation, we adopt snapshot segments. And then we clear invalid segment by its max read position. @@ -208,7 +212,7 @@ public CompletableFuture recoverFromSnapshot() { this.unsealedAbortedTxnIdSegment = convertTypeToTxnID(persistentSnapshotIndexes .getSnapshot().getAborts()); if (indexes.size() != 0) { - persistentWorker.sequenceID.set(indexes.lastEntry().getValue().sequenceID + 1); + persistentWorker.sequenceID.set(indexes.get(indexes.lastKey()).sequenceID + 1); } } //Read snapshot segment to recover aborts. @@ -323,7 +327,7 @@ private void handleSnapshotSegmentEntry(Entry entry) { segmentIndex.put(new PositionImpl(snapshotSegment.getPersistentPositionLedgerId(), snapshotSegment.getPersistentPositionEntryId()), new TxnID(lastTxn.getMostSigBits(), lastTxn.getLeastSigBits())); - aborts.addAll(convertTypeToTxnID(snapshotSegment.getAborts())); + convertTypeToTxnID(snapshotSegment.getAborts()).forEach(txnID -> aborts.put(txnID, txnID)); } private long getSystemClientOperationTimeoutMs() throws Exception { @@ -526,8 +530,8 @@ private CompletableFuture deleteSnapshotSegment(List positio this.topic.getName(), this.sequenceID, positionNeedToDelete); } TxnID theLatestDeletedTxnID = segmentIndex.remove(positionNeedToDelete); - while (!aborts.getFirst().equals(theLatestDeletedTxnID)) { - aborts.removeFirst(); + while (!aborts.firstKey().equals(theLatestDeletedTxnID)) { + aborts.remove(aborts.firstKey()); } aborts.remove(theLatestDeletedTxnID); //The process will check whether the snapshot segment is null, and update index when From 78424f9e0c035fc8e02db84d6fd8324fab254448 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Tue, 3 Jan 2023 23:35:28 +0800 Subject: [PATCH 14/41] fix some comments --- ...onBufferSnapshotBaseSystemTopicClient.java | 1 + ...SingleSnapshotAbortedTxnProcessorImpl.java | 7 +- ...napshotSegmentAbortedTxnProcessorImpl.java | 132 ++++++++++-------- .../v2/TransactionBufferSnapshotIndex.java | 8 +- .../TopicTransactionBufferRecoverTest.java | 8 +- 5 files changed, 87 insertions(+), 69 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSnapshotBaseSystemTopicClient.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSnapshotBaseSystemTopicClient.java index b18bf552c3004..8efa983a64d73 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSnapshotBaseSystemTopicClient.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TransactionBufferSnapshotBaseSystemTopicClient.java @@ -188,6 +188,7 @@ public SystemTopicClient getSystemTopic() { protected CompletableFuture> newWriterAsyncInternal() { return client.newProducer(Schema.AVRO(schemaType)) .topic(topicName.toString()) + .enableBatching(false) .createAsync().thenApply(producer -> { if (log.isDebugEnabled()) { log.debug("[{}] A new {} writer is created", topicName, schemaType.getName()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 82843d8997cd7..496a530d64637 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -56,7 +56,12 @@ public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; this.takeSnapshotWriter = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName())); + .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName())) + .exceptionally((ex) -> { + log.error("{} Failed to create snapshot writer", topic.getName()); + topic.close(); + return null; + }); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 1140ba324fca3..ee874a563dbee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; @@ -71,13 +70,13 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess // when its size reach the capital of a snapshot segment. private LinkedList unsealedAbortedTxnIdSegment; - //A mapping form the latest txn mark persistent position in a segment to its txn ID. + //A mapping form the latest txn mark persistent position in a segment to its latest txn ID. //This is mainly used to trim expired snapshot segment. private final LinkedMap segmentIndex = new LinkedMap<>(); //Store all aborted txn IDs check whether a txn is an aborted txn. private final LinkedMap aborts = new LinkedMap<>(); - //The indexes of the snapshot segments. + //The indexes of the snapshot segments whose key is the aborted mark persistent position. private final LinkedMap indexes = new LinkedMap<>(); //The latest persistent snapshot index. This is used to combine new segment indexes with the latest metadata and // indexes. @@ -137,8 +136,15 @@ public void trimExpiredAbortedTxns() { PositionImpl positionNeedToDelete = segmentIndex.firstKey(); positionsNeedToDelete.add(positionNeedToDelete); } - persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment, - () -> persistentWorker.deleteSnapshotSegment(positionsNeedToDelete)); + //Batch delete the expired segment and then update segment index. + if (!positionsNeedToDelete.isEmpty()) { + persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment, + () -> persistentWorker.deleteSnapshotSegment(positionsNeedToDelete) + .thenRun(() -> { + persistentWorker.updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), + indexes.values().stream().toList()); + })); + } } private String buildKey(long sequenceId) { @@ -206,11 +212,12 @@ public CompletableFuture recoverFromSnapshot() { persistentSnapshotIndexes.getIndexList() .forEach(transactionBufferSnapshotIndex -> indexes.put(new PositionImpl( - transactionBufferSnapshotIndex.persistentPositionLedgerID, - transactionBufferSnapshotIndex.persistentPositionEntryID), + transactionBufferSnapshotIndex.abortedMarkLedgerID, + transactionBufferSnapshotIndex.abortedMarkEntryID), transactionBufferSnapshotIndex)); this.unsealedAbortedTxnIdSegment = convertTypeToTxnID(persistentSnapshotIndexes .getSnapshot().getAborts()); + //If the size of indexes is 0, the sequence ID will be the init value 0. if (indexes.size() != 0) { persistentWorker.sequenceID.set(indexes.get(indexes.lastKey()).sequenceID + 1); } @@ -228,27 +235,34 @@ public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnly CompletableFuture handleSegmentFuture = new CompletableFuture<>(); completableFutures.add(handleSegmentFuture); readOnlyManagedLedger.asyncReadEntry( - new PositionImpl(index.getPersistentPositionLedgerID(), - index.getPersistentPositionEntryID()), + new PositionImpl(index.getSegmentLedgerID(), + index.getSegmentEntryID()), new AsyncCallbacks.ReadEntryCallback() { @Override public void readEntryComplete(Entry entry, Object ctx) { - //Remove invalid index - if (entry == null) { - indexes.remove(new PositionImpl( - index.getMaxReadPositionLedgerID(), - index.getMaxReadPositionEntryID())); - handleSegmentFuture.complete(null); - hasInvalidIndex.set(true); - return; - } handleSnapshotSegmentEntry(entry); + entry.release(); handleSegmentFuture.complete(null); } @Override public void readEntryFailed(ManagedLedgerException exception, Object ctx) { - handleSegmentFuture.completeExceptionally(exception); + if (exception instanceof ManagedLedgerException + .NonRecoverableLedgerException) { + if (((ManagedLedgerImpl)topic.getManagedLedger()) + .ledgerExists(index.getAbortedMarkLedgerID())) { + log.error("[{}] Failed to read snapshot segment [{}:{}]", + topic.getName(), index.segmentLedgerID, + index.segmentEntryID, exception); + topic.close(); + handleSegmentFuture.completeExceptionally(exception); + } else { + indexes.remove(new PositionImpl( + index.getAbortedMarkLedgerID(), + index.getAbortedMarkEntryID())); + hasInvalidIndex.set(true); + } + } } }, null); }); @@ -294,7 +308,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob @Override public CompletableFuture deleteAbortedTxnSnapshot() { CompletableFuture completableFuture = new CompletableFuture<>(); - persistentWorker.appendTask(PersistentWorker.OperationType.Close, + persistentWorker.appendTask(PersistentWorker.OperationType.Clear, () -> persistentWorker.clearSnapshotSegmentAndIndexes() .thenRun(() -> { completableFuture.thenCompose(null); @@ -355,11 +369,7 @@ private class PersistentWorker { private enum OperationState { None, - UpdatingIndex, - WritingSegment, - DeletingSegment, - Closing, - Closed + Operating } private static final AtomicReferenceFieldUpdater STATE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(PersistentWorker.class, @@ -369,7 +379,7 @@ public enum OperationType { UpdateIndex, WriteSegment, DeleteSegment, - Close + Clear } private volatile OperationState operationState = OperationState.None; @@ -377,33 +387,38 @@ public enum OperationType { ConcurrentLinkedDeque>>> taskQueue = new ConcurrentLinkedDeque<>(); private CompletableFuture lastOperationFuture; - private final Timer timer; public PersistentWorker(PersistentTopic topic) { this.topic = topic; - this.timer = topic.getBrokerService().getPulsar().getTransactionTimer(); this.snapshotSegmentsWriterFuture = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName())); this.snapshotIndexWriterFuture = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName())); - + .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName())) + .exceptionally((ex) -> { + log.error("{} Failed to create snapshot writer", topic.getName()); + topic.close(); + return null; + });; } public void appendTask(OperationType operationType, Supplier> task) { switch (operationType) { + //Update index is can be canceled when the task queue is not empty, so it should be executed immediately + // instead of taking in queue. If the task queue is not empty, execute the task from the queue. case UpdateIndex -> { if (!taskQueue.isEmpty()) { - return; - } else if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.UpdatingIndex)) { + topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this).submit(this::executeTask); + } else if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { lastOperationFuture = task.get(); lastOperationFuture.whenComplete((ignore, throwable) -> { if (throwable != null && log.isDebugEnabled()) { log.debug("[{}] Failed to update index snapshot", topic.getName(), throwable); } - STATE_UPDATER.compareAndSet(this, OperationState.UpdatingIndex, OperationState.None); + STATE_UPDATER.compareAndSet(this, OperationState.Operating, OperationState.None); }); } } @@ -411,22 +426,20 @@ public void appendTask(OperationType operationType, Supplier(operationType, task)); executeTask(); } - case Close -> { - STATE_UPDATER.set(this, OperationState.Closing); + //If the operation type is clear, all the operation in the queue is meaningless. + case Clear -> { + STATE_UPDATER.set(this, OperationState.Operating); taskQueue.clear(); - lastOperationFuture.thenRun(() -> { - task.get().thenRun(() -> - STATE_UPDATER.compareAndSet(this, OperationState.Closing, OperationState.Closed)); - }); } } } private void executeTask() { + if (taskQueue.isEmpty()) return; OperationType operationType = taskQueue.getFirst().getKey(); switch (operationType) { case WriteSegment -> { - if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.WritingSegment)) { + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { if (taskQueue.getFirst().getKey() == OperationType.WriteSegment) { lastOperationFuture = taskQueue.getFirst().getValue().get(); lastOperationFuture.whenComplete((ignore, throwable) -> { @@ -434,19 +447,18 @@ private void executeTask() { if (log.isDebugEnabled()) { log.debug("[{}] Failed to write snapshot segment", topic.getName(), throwable); } - timer.newTimeout(timeout -> executeTask(), - takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); } else { taskQueue.removeFirst(); } - STATE_UPDATER.compareAndSet(this, - OperationState.WritingSegment, OperationState.None); + STATE_UPDATER.compareAndSet(this, OperationState.Operating, OperationState.None); + topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this).submit(this::executeTask); }); } } } case DeleteSegment -> { - if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.DeletingSegment)) { + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { if (taskQueue.getFirst().getKey() == OperationType.DeleteSegment) { lastOperationFuture = taskQueue.getFirst().getValue().get(); lastOperationFuture.whenComplete((ignore, throwable) -> { @@ -454,14 +466,13 @@ private void executeTask() { if (log.isDebugEnabled()) { log.debug("[{}] Failed to delete snapshot segment", topic.getName(), throwable); } - timer.newTimeout(timeout -> executeTask(), - takeSnapshotIntervalTime, TimeUnit.MILLISECONDS); } else { taskQueue.removeFirst(); } - STATE_UPDATER.compareAndSet(this, - OperationState.DeletingSegment, OperationState.None); + STATE_UPDATER.compareAndSet(this, OperationState.Operating, OperationState.None); + topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this).submit(this::executeTask); }); } } @@ -504,12 +515,12 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm //Build index for this segment TransactionBufferSnapshotIndex index = new TransactionBufferSnapshotIndex(); index.setSequenceID(transactionBufferSnapshotSegment.getSequenceId()); - index.setMaxReadPositionLedgerID(maxReadPosition.getLedgerId()); - index.setMaxReadPositionEntryID(maxReadPosition.getEntryId()); - index.setPersistentPositionLedgerID(((MessageIdImpl) messageId).getLedgerId()); - index.setPersistentPositionEntryID(((MessageIdImpl) messageId).getEntryId()); + index.setAbortedMarkLedgerID(abortedMarkerPersistentPosition.getLedgerId()); + index.setAbortedMarkEntryID(abortedMarkerPersistentPosition.getEntryId()); + index.setSegmentLedgerID(((MessageIdImpl) messageId).getLedgerId()); + index.setSegmentEntryID(((MessageIdImpl) messageId).getEntryId()); - indexes.put(maxReadPosition, index); + indexes.put(abortedMarkerPersistentPosition, index); //update snapshot segment index. return updateSnapshotIndex(new TransactionBufferSnapshotIndexesMetadata( maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), new LinkedList<>()), @@ -517,6 +528,7 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm }); } + // update index after delete all segment. private CompletableFuture deleteSnapshotSegment(List positionNeedToDeletes) { List> results = new ArrayList<>(); for (PositionImpl positionNeedToDelete : positionNeedToDeletes) { @@ -530,16 +542,17 @@ private CompletableFuture deleteSnapshotSegment(List positio this.topic.getName(), this.sequenceID, positionNeedToDelete); } TxnID theLatestDeletedTxnID = segmentIndex.remove(positionNeedToDelete); + //The position is already deleted. + if (theLatestDeletedTxnID == null) { + return; + } while (!aborts.firstKey().equals(theLatestDeletedTxnID)) { aborts.remove(aborts.firstKey()); } aborts.remove(theLatestDeletedTxnID); - //The process will check whether the snapshot segment is null, and update index when - // recovered. + //The process will check whether the snapshot segment is null, + // and update index when recovered. indexes.remove(positionNeedToDelete); - //Keep index snapshot and update index - updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), - indexes.values().stream().toList()); }).exceptionally(e -> { log.warn("[{}] Failed to delete the snapshot segment, " + "whose sequenceId is [{}] and maxReadPosition is [{}]", @@ -581,7 +594,6 @@ private CompletableFuture clearSnapshotSegmentAndIndexes() { .thenRun(() -> { log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", topic.getName()); - }) .exceptionally(e -> { log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java index 118472397adda..3d761af8ebd06 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java @@ -29,8 +29,8 @@ @NoArgsConstructor public class TransactionBufferSnapshotIndex { public long sequenceID; - public long maxReadPositionLedgerID; - public long maxReadPositionEntryID; - public long persistentPositionLedgerID; - public long persistentPositionEntryID; + public long abortedMarkLedgerID; + public long abortedMarkEntryID; + public long segmentLedgerID; + public long segmentEntryID; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 28e9eea5e69f5..4acd2ca728150 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -723,10 +723,10 @@ public void testTransactionBufferIndexSystemTopic() throws Exception { TransactionBufferSnapshotIndex transactionBufferSnapshotIndex = transactionBufferTransactionBufferSnapshotIndexes.getIndexList().get(1); - assertEquals(transactionBufferSnapshotIndex.getMaxReadPositionLedgerID(), 1L); - assertEquals(transactionBufferSnapshotIndex.getMaxReadPositionEntryID(), 1L); - assertEquals(transactionBufferSnapshotIndex.getPersistentPositionLedgerID(), 1L); - assertEquals(transactionBufferSnapshotIndex.getPersistentPositionEntryID(), 1L); + assertEquals(transactionBufferSnapshotIndex.getAbortedMarkLedgerID(), 1L); + assertEquals(transactionBufferSnapshotIndex.getAbortedMarkEntryID(), 1L); + assertEquals(transactionBufferSnapshotIndex.getSegmentLedgerID(), 1L); + assertEquals(transactionBufferSnapshotIndex.getSegmentEntryID(), 1L); assertEquals(transactionBufferSnapshotIndex.getSequenceID(), 1L); } From 22ccc5b9b30feb509d16660cbe07346d93a360b2 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Mon, 9 Jan 2023 17:52:10 +0800 Subject: [PATCH 15/41] optimize --- ...napshotSegmentAbortedTxnProcessorImpl.java | 8 ++--- .../TopicTransactionBufferRecoverTest.java | 29 ++++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index ee874a563dbee..1f6e827debeff 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -86,8 +86,6 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess private volatile long lastSnapshotTimestamps; - private final int takeSnapshotIntervalTime; - private final int transactionBufferMaxAbortedTxnsOfSnapshotSegment; private final PersistentWorker persistentWorker; @@ -96,8 +94,6 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; this.persistentWorker = new PersistentWorker(topic); - this.takeSnapshotIntervalTime = topic.getBrokerService().getPulsar() - .getConfiguration().getTransactionBufferSnapshotMinTimeInMillis(); this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotSegmentSize(); this.unsealedAbortedTxnIdSegment = new LinkedList<>(); @@ -156,6 +152,10 @@ public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosit TransactionBufferSnapshotIndexesMetadata metadata = new TransactionBufferSnapshotIndexesMetadata( maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), convertTypeToTxnIDData(unsealedAbortedTxnIdSegment)); + return updateSnapshotIndexMetadata(metadata); + } + + private CompletableFuture updateSnapshotIndexMetadata(TransactionBufferSnapshotIndexesMetadata metadata) { CompletableFuture completableFuture = new CompletableFuture<>(); persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> persistentWorker diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 4acd2ca728150..e593c11a8bd97 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -832,14 +832,14 @@ public void testSnapshotSegment() throws Exception { LinkedMap ongoingTxns = new LinkedMap<>(); LinkedList abortedTxns = new LinkedList<>(); - // 0. Modify the configurations + // 0. Modify the configurations, enabling the segment snapshot and set the size of the snapshot segment. int theSizeOfSegment = 10; int theCountOfSnapshotMaxTxnCount = 3; this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSnapshotSegmentSize(theSizeOfSegment); this.getPulsarServiceList().get(0).getConfig() .setTransactionBufferSnapshotMaxTransactionCount(theCountOfSnapshotMaxTxnCount); - // 1. Build prodcuer and consumer + // 1. Build producer and consumer Producer producer = pulsarClient.newProducer(Schema.INT32) .topic(topic) .enableBatching(false) @@ -852,17 +852,17 @@ public void testSnapshotSegment() throws Exception { .subscribe(); // 2. Check the AbortedTxnProcessor workflow 10 times + int messageSize = theSizeOfSegment * 4; for (int i = 0; i < 10; i++) { MessageId maxReadMessage = null; int abortedTxnSize = 0; - // The number of aborted transaction = 30 / 2, that is more than the size of snapshot segment 10. - for (int j = 0; j < theSizeOfSegment * 4; j++) { + for (int j = 0; j < messageSize; j++) { Transaction transaction = pulsarClient.newTransaction() .withTransactionTimeout(5, TimeUnit.MINUTES).build().get(); //Half common message and half transaction message. - //And the transaction message have a half which are aborted. - if (RandomUtils.nextInt() % 2 == 0) { + if (j % 2 == 0) { MessageId messageId = producer.newMessage(transaction).value(i * 10 + j).send(); + //And the transaction message have a half which are aborted. if (RandomUtils.nextInt() % 2 == 0) { transaction.abort().get(); abortedTxns.add(messageId); @@ -876,11 +876,11 @@ public void testSnapshotSegment() throws Exception { } } } else { - MessageId messageId = producer.newMessage().value(i * 10 + j).send(); + producer.newMessage().value(i * 10 + j).send(); transaction.commit().get(); } } - // 2.1 Check the updating of the maxReadPosition + // 2.1 Receive all message before the maxReadPosition to verify the correctness of the max read position. int hasReceived = 0; while (true) { Message message = consumer.receive(2, TimeUnit.SECONDS); @@ -891,26 +891,27 @@ public void testSnapshotSegment() throws Exception { break; } } - + //2.2 Commit all ongoing transaction and verify that the consumer can receive all rest message + // expect for aborted txn message. for (Transaction ongoingTxn: ongoingTxns.keySet()) { ongoingTxn.commit().get(); } ongoingTxns.clear(); - // 2.2 Check the aborted txn - for (int k = hasReceived; k < theSizeOfSegment * 4 - abortedTxnSize; k++) { + for (int k = hasReceived; k < messageSize - abortedTxnSize; k++) { Message message = consumer.receive(2, TimeUnit.SECONDS); assertNotNull(message); assertFalse(abortedTxns.contains(message.getMessageId())); } } - // 3. Test recover + // 3. After the topic unload, the consumer can receive all the messages in the 10 tests + // expect for the aborted transaction messages. admin.topics().unload(topic); - - for (int i = 0; i < 200 - abortedTxns.size(); i++) { + for (int i = 0; i < messageSize * 10 - abortedTxns.size(); i++) { Message message = consumer.receive(2, TimeUnit.SECONDS); assertNotNull(message); assertFalse(abortedTxns.contains(message.getMessageId())); } + assertNull(consumer.receive(2, TimeUnit.SECONDS)); } } From 71fe2db870ca5f282a2a6b450dff41bcd4c4b0b2 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Mon, 9 Jan 2023 20:04:26 +0800 Subject: [PATCH 16/41] fix uncompleted future. --- .../service/BrokerServiceException.java | 6 ++++++ .../buffer/AbortedTxnProcessor.java | 2 +- ...SingleSnapshotAbortedTxnProcessorImpl.java | 2 +- ...napshotSegmentAbortedTxnProcessorImpl.java | 21 ++++++++++++++----- .../buffer/impl/TopicTransactionBuffer.java | 2 +- 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java index fd3a391bca34a..17caf7b119215 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java @@ -77,6 +77,12 @@ public TopicClosedException(Throwable t) { } } + public static class TransactionBufferClosedException extends BrokerServiceException { + public TransactionBufferClosedException(String s) { + super(s); + } + } + @Deprecated public static class AddEntryMetadataException extends BrokerServiceException { public AddEntryMetadataException(Throwable t) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index 9c173be8a1a70..87fc3e4d0498f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -58,7 +58,7 @@ public interface AbortedTxnProcessor { * Delete the transaction buffer aborted transaction snapshot. * @return a completableFuture. */ - CompletableFuture deleteAbortedTxnSnapshot(); + CompletableFuture clearAbortedTxnSnapshot(); /** * Take aborted transactions snapshot. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 496a530d64637..724bbe3c5dd0a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -132,7 +132,7 @@ public CompletableFuture recoverFromSnapshot() { } @Override - public CompletableFuture deleteAbortedTxnSnapshot() { + public CompletableFuture clearAbortedTxnSnapshot() { return this.takeSnapshotWriter.thenCompose(writer -> { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); snapshot.setTopicName(topic.getName()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 1f6e827debeff..30a81e0726e6b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -306,7 +306,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob } @Override - public CompletableFuture deleteAbortedTxnSnapshot() { + public CompletableFuture clearAbortedTxnSnapshot() { CompletableFuture completableFuture = new CompletableFuture<>(); persistentWorker.appendTask(PersistentWorker.OperationType.Clear, () -> persistentWorker.clearSnapshotSegmentAndIndexes() @@ -369,7 +369,8 @@ private class PersistentWorker { private enum OperationState { None, - Operating + Operating, + Closed } private static final AtomicReferenceFieldUpdater STATE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(PersistentWorker.class, @@ -409,6 +410,7 @@ public void appendTask(OperationType operationType, Supplier { if (!taskQueue.isEmpty()) { + task.get().complete(null); topic.getBrokerService().getPulsar().getTransactionExecutorProvider() .getExecutor(this).submit(this::executeTask); } else if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { @@ -423,12 +425,21 @@ public void appendTask(OperationType operationType, Supplier { - taskQueue.add(new MutablePair<>(operationType, task)); - executeTask(); + if (!STATE_UPDATER.get(this).equals(OperationState.Closed)) { + taskQueue.add(new MutablePair<>(operationType, task)); + executeTask(); + } } //If the operation type is clear, all the operation in the queue is meaningless. case Clear -> { - STATE_UPDATER.set(this, OperationState.Operating); + STATE_UPDATER.set(this, OperationState.Closed); + taskQueue.forEach(pair -> { + pair.getRight().get().completeExceptionally( + new BrokerServiceException.TransactionBufferClosedException( + String.format("Cancel the operation [%s] due to the" + + " transaction buffer of the topic[%s] already closed", + pair.getLeft().name(), this.topic.getName()))); + }); taskQueue.clear(); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 66b377934bddd..a1a74d4e6dbfc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -459,7 +459,7 @@ public CompletableFuture purgeTxns(List dataLedgers) { @Override public CompletableFuture clearSnapshot() { - return snapshotAbortedTxnProcessor.deleteAbortedTxnSnapshot(); + return snapshotAbortedTxnProcessor.clearAbortedTxnSnapshot(); } @Override From fc9ef84520b75804757f836d423f18d18182b4f2 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Tue, 10 Jan 2023 19:44:00 +0800 Subject: [PATCH 17/41] optimize and add an API test --- .../SystemTopicTxnBufferSnapshotService.java | 2 +- .../NamespaceEventsSystemTopicFactory.java | 8 +- .../buffer/AbortedTxnProcessor.java | 3 +- ...SingleSnapshotAbortedTxnProcessorImpl.java | 2 +- ...napshotSegmentAbortedTxnProcessorImpl.java | 82 +++++----- .../buffer/impl/TopicTransactionBuffer.java | 4 +- .../transaction/AbortTxnProcessorTest.java | 143 ++++++++++++++++++ .../TopicTransactionBufferRecoverTest.java | 2 +- 8 files changed, 198 insertions(+), 48 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java index 7be599c8c2781..a1b78d89a13eb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicTxnBufferSnapshotService.java @@ -72,7 +72,7 @@ protected CompletableFuture> getTransactionBufferSystemTopi } return CompletableFuture.completedFuture(clients.computeIfAbsent(systemTopicName, (v) -> namespaceEventsSystemTopicFactory - .createTransactionBufferSystemTopicClient(topicName.getNamespaceObject(), + .createTransactionBufferSystemTopicClient(systemTopicName, this, schemaType))); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicFactory.java index 8d30d1d140f5a..f5e6c7748d10b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicFactory.java @@ -44,12 +44,10 @@ public TopicPoliciesSystemTopicClient createTopicPoliciesSystemTopicClient(Names } public TransactionBufferSnapshotBaseSystemTopicClient createTransactionBufferSystemTopicClient( - NamespaceName namespaceName, SystemTopicTxnBufferSnapshotService + TopicName systemTopicName, SystemTopicTxnBufferSnapshotService systemTopicTxnBufferSnapshotService, Class schemaType) { - TopicName topicName = TopicName.get(TopicDomain.persistent.value(), namespaceName, - SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); - log.info("Create transaction buffer snapshot client, topicName : {}", topicName.toString()); - return new TransactionBufferSnapshotBaseSystemTopicClient(client, topicName, + log.info("Create transaction buffer snapshot client, topicName : {}", systemTopicName.toString()); + return new TransactionBufferSnapshotBaseSystemTopicClient(client, systemTopicName, systemTopicTxnBufferSnapshotService, schemaType); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index 87fc3e4d0498f..06b35e427e179 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -42,10 +42,9 @@ public interface AbortedTxnProcessor { /** * Check whether the transaction ID is an aborted transaction ID. * @param txnID the transaction ID that needs to be checked. - * @param readPosition the read position of the transaction message, can be used to find the segment. * @return a boolean, whether the transaction ID is an aborted transaction ID. */ - boolean checkAbortedTransaction(TxnID txnID, Position readPosition); + boolean checkAbortedTransaction(TxnID txnID); /** * Recover transaction buffer by transaction buffer snapshot. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 724bbe3c5dd0a..d0ca848fcbcec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -83,7 +83,7 @@ public void trimExpiredAbortedTxns() { } @Override - public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { + public boolean checkAbortedTransaction(TxnID txnID) { return aborts.containsKey(txnID); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 30a81e0726e6b..b28aff59fed36 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -20,7 +20,6 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; -import io.netty.util.Timer; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; @@ -31,12 +30,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import java.util.function.Supplier; +import java.util.function.Function; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; @@ -57,7 +55,7 @@ import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; -import org.apache.pulsar.common.events.EventType; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.Commands; @@ -71,7 +69,7 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess private LinkedList unsealedAbortedTxnIdSegment; //A mapping form the latest txn mark persistent position in a segment to its latest txn ID. - //This is mainly used to trim expired snapshot segment. + //This is mainly used to trim expired snapshot segment and remove them form aborts. private final LinkedMap segmentIndex = new LinkedMap<>(); //Store all aborted txn IDs check whether a txn is an aborted txn. @@ -107,14 +105,14 @@ public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMar if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { LinkedList abortedSegment = unsealedAbortedTxnIdSegment; segmentIndex.put(abortedMarkerPersistentPosition, abortedTxnId); - persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> + persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, (cancel) -> persistentWorker.takeSnapshotSegmentAsync(abortedSegment, abortedMarkerPersistentPosition)); this.unsealedAbortedTxnIdSegment = new LinkedList<>(); } } @Override - public boolean checkAbortedTransaction(TxnID txnID, Position readPosition) { + public boolean checkAbortedTransaction(TxnID txnID) { return aborts.containsKey(txnID); } @@ -131,15 +129,23 @@ public void trimExpiredAbortedTxns() { } PositionImpl positionNeedToDelete = segmentIndex.firstKey(); positionsNeedToDelete.add(positionNeedToDelete); + + TxnID theLatestDeletedTxnID = segmentIndex.remove(positionNeedToDelete); + //The position is already deleted. + if (theLatestDeletedTxnID == null) { + continue; + } + while (!aborts.firstKey().equals(theLatestDeletedTxnID)) { + aborts.remove(aborts.firstKey()); + } + aborts.remove(theLatestDeletedTxnID); } //Batch delete the expired segment and then update segment index. if (!positionsNeedToDelete.isEmpty()) { persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment, - () -> persistentWorker.deleteSnapshotSegment(positionsNeedToDelete) - .thenRun(() -> { - persistentWorker.updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), - indexes.values().stream().toList()); - })); + (cancel) -> persistentWorker.deleteSnapshotSegment(positionsNeedToDelete) + .thenRun(() -> persistentWorker.updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), + indexes.values().stream().toList()))); } } @@ -157,14 +163,20 @@ public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosit private CompletableFuture updateSnapshotIndexMetadata(TransactionBufferSnapshotIndexesMetadata metadata) { CompletableFuture completableFuture = new CompletableFuture<>(); - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, - () -> persistentWorker + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, (cancel) -> { + if (!cancel) { + return persistentWorker .updateSnapshotIndex(metadata, persistentSnapshotIndexes.getIndexList()) .thenRun(() -> completableFuture.complete(null)) .exceptionally(e -> { completableFuture.completeExceptionally(e); return null; - })); + }); + } else { + completableFuture.complete(null); + return completableFuture; + } + }); return completableFuture; } @@ -278,7 +290,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob TopicName snapshotIndexTopicName = TopicName.get(TopicDomain.persistent.toString(), TopicName.get(topic.getName()).getNamespaceObject(), - EventType.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS.toString()); + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS); this.topic.getBrokerService().getPulsar().getManagedLedgerFactory() .asyncOpenReadOnlyManagedLedger(snapshotIndexTopicName .getPersistenceNamingEncoding(), callback, @@ -289,12 +301,20 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob return openManagedLedgerFuture .thenCompose((ignore) -> FutureUtil.waitForAll(completableFutures).thenCompose((i) -> { + //This is a compensation mechanism for deleting the segment + // but not successfully updating the index. if (hasInvalidIndex.get()) { - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, (cancel) -> persistentWorker .updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), indexes.values().stream().toList())); } + //Append the aborted txn IDs in the index metadata + //can keep the order of the aborted txn in the aborts. + //So that we can trim the expired snapshot segment in aborts + // according to the latest txn IDs + convertTypeToTxnID(persistentSnapshotIndexes.getSnapshot().getAborts()) + .forEach(txnID -> aborts.put(txnID, txnID)); return CompletableFuture.completedFuture(finalStartReadCursorPosition); })).exceptionally(ex -> { log.error("[{}] Failed to recover snapshot segment", this.topic.getName(), ex); @@ -309,7 +329,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob public CompletableFuture clearAbortedTxnSnapshot() { CompletableFuture completableFuture = new CompletableFuture<>(); persistentWorker.appendTask(PersistentWorker.OperationType.Clear, - () -> persistentWorker.clearSnapshotSegmentAndIndexes() + (cancel) -> persistentWorker.clearSnapshotSegmentAndIndexes() .thenRun(() -> { completableFuture.thenCompose(null); }).exceptionally(e -> { @@ -356,7 +376,7 @@ private void closeReader(SystemTopicClient.Reader reader) { }); } - private class PersistentWorker { + public class PersistentWorker { protected final AtomicLong sequenceID = new AtomicLong(0); private final PersistentTopic topic; @@ -385,7 +405,7 @@ public enum OperationType { private volatile OperationState operationState = OperationState.None; - ConcurrentLinkedDeque>>> taskQueue = + ConcurrentLinkedDeque>>> taskQueue = new ConcurrentLinkedDeque<>(); private CompletableFuture lastOperationFuture; @@ -404,22 +424,21 @@ public PersistentWorker(PersistentTopic topic) { });; } - public void appendTask(OperationType operationType, Supplier> task) { + public void appendTask(OperationType operationType, Function> task) { switch (operationType) { //Update index is can be canceled when the task queue is not empty, so it should be executed immediately // instead of taking in queue. If the task queue is not empty, execute the task from the queue. case UpdateIndex -> { if (!taskQueue.isEmpty()) { - task.get().complete(null); + task.apply(Boolean.TRUE); topic.getBrokerService().getPulsar().getTransactionExecutorProvider() .getExecutor(this).submit(this::executeTask); } else if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { - lastOperationFuture = task.get(); + lastOperationFuture = task.apply(Boolean.FALSE); lastOperationFuture.whenComplete((ignore, throwable) -> { if (throwable != null && log.isDebugEnabled()) { log.debug("[{}] Failed to update index snapshot", topic.getName(), throwable); } - STATE_UPDATER.compareAndSet(this, OperationState.Operating, OperationState.None); }); } @@ -434,7 +453,7 @@ public void appendTask(OperationType operationType, Supplier { STATE_UPDATER.set(this, OperationState.Closed); taskQueue.forEach(pair -> { - pair.getRight().get().completeExceptionally( + pair.getRight().apply(Boolean.FALSE).completeExceptionally( new BrokerServiceException.TransactionBufferClosedException( String.format("Cancel the operation [%s] due to the" + " transaction buffer of the topic[%s] already closed", @@ -452,7 +471,7 @@ private void executeTask() { case WriteSegment -> { if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { if (taskQueue.getFirst().getKey() == OperationType.WriteSegment) { - lastOperationFuture = taskQueue.getFirst().getValue().get(); + lastOperationFuture = taskQueue.getFirst().getValue().apply(Boolean.FALSE); lastOperationFuture.whenComplete((ignore, throwable) -> { if (throwable != null) { if (log.isDebugEnabled()) { @@ -471,7 +490,7 @@ private void executeTask() { case DeleteSegment -> { if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { if (taskQueue.getFirst().getKey() == OperationType.DeleteSegment) { - lastOperationFuture = taskQueue.getFirst().getValue().get(); + lastOperationFuture = taskQueue.getFirst().getValue().apply(Boolean.FALSE); lastOperationFuture.whenComplete((ignore, throwable) -> { if (throwable != null) { if (log.isDebugEnabled()) { @@ -552,15 +571,6 @@ private CompletableFuture deleteSnapshotSegment(List positio + "whose sequenceId is [{}] and maxReadPosition is [{}]", this.topic.getName(), this.sequenceID, positionNeedToDelete); } - TxnID theLatestDeletedTxnID = segmentIndex.remove(positionNeedToDelete); - //The position is already deleted. - if (theLatestDeletedTxnID == null) { - return; - } - while (!aborts.firstKey().equals(theLatestDeletedTxnID)) { - aborts.remove(aborts.firstKey()); - } - aborts.remove(theLatestDeletedTxnID); //The process will check whether the snapshot segment is null, // and update index when recovered. indexes.remove(positionNeedToDelete); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index a1a74d4e6dbfc..7429c784d8b67 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -279,7 +279,7 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { private void handleTransactionMessage(TxnID txnId, Position position) { if (!ongoingTxns.containsKey(txnId) && !this.snapshotAbortedTxnProcessor - .checkAbortedTransaction(txnId, position)) { + .checkAbortedTransaction(txnId)) { ongoingTxns.put(txnId, (PositionImpl) position); PositionImpl firstPosition = ongoingTxns.get(ongoingTxns.firstKey()); //max read position is less than first ongoing transaction message position, so entryId -1 @@ -470,7 +470,7 @@ public CompletableFuture closeAsync() { @Override public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { - return snapshotAbortedTxnProcessor.checkAbortedTransaction(txnID, readPosition); + return snapshotAbortedTxnProcessor.checkAbortedTransaction(txnID); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java new file mode 100644 index 0000000000000..e863cf236182c --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java @@ -0,0 +1,143 @@ +package org.apache.pulsar.broker.transaction; + +import java.lang.reflect.Field; +import java.util.LinkedList; +import java.util.NavigableMap; +import java.util.Queue; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.commons.collections4.map.LinkedMap; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; +import org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl; +import org.apache.pulsar.client.api.transaction.TxnID; +import org.testcontainers.shaded.org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +public class AbortTxnProcessorTest extends TransactionTestBase { + + private static final String PROCESSOR_TOPIC = NAMESPACE1 + "/abortedTxnProcessor"; + private static final int SEGMENT_SIZE = 5; + private PulsarService pulsarService = null; + + @Override + @BeforeClass + protected void setup() throws Exception { + setUpBase(1, 1, PROCESSOR_TOPIC, 0); + this.pulsarService = getPulsarServiceList().get(0); + this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(SEGMENT_SIZE); + } + + @Override + @AfterClass + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + /** + * Test api: + * 1. putAbortedTxnAndPosition + * 2. checkAbortedTransaction + * 3. takeAbortedTxnsSnapshot + * 4. recoverFromSnapshot + * 5. trimExpiredAbortedTxns + * @throws Exception + */ + @Test + public void testPutAbortedTxnIntoProcessor() throws Exception { + PersistentTopic persistentTopic = (PersistentTopic) pulsarService.getBrokerService() + .getTopic(PROCESSOR_TOPIC, false).get().get(); + AbortedTxnProcessor processor = new SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic); + //1. prepare test data. + //1.1 Put 10 aborted txn IDs to persistent two sealed segments. + for (int i = 0; i < 10; i++) { + TxnID txnID = new TxnID(0, i); + PositionImpl position = new PositionImpl(0,i); + processor.putAbortedTxnAndPosition(txnID, position); + } + //1.2 Put 4 aborted txn IDs into the unsealed segment. + for (int i = 10; i < 14; i++) { + TxnID txnID = new TxnID(0, i); + PositionImpl position = new PositionImpl(0,i); + processor.putAbortedTxnAndPosition(txnID, position); + } + //1.3 Verify the common data flow + verifyAbortedTxnIDAndSegmentIndex(processor,0,14); + //2. Take the latest snapshot and verify recover from snapshot + AbortedTxnProcessor newProcessor = new SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic); + PositionImpl maxReadPosition = new PositionImpl(0, 14); + //2.1 Avoid update operation being canceled. + waitTaskExecuteCompletely(newProcessor); + //2.2 take the latest snapshot + processor.takeAbortedTxnsSnapshot(maxReadPosition).get(); + newProcessor.recoverFromSnapshot().get(); + //Verify the recovery data flow + verifyAbortedTxnIDAndSegmentIndex(newProcessor,0,14); + //3. Delete the ledgers and then verify the date. + Field ledgersField = ManagedLedgerImpl.class.getDeclaredField("ledgers"); + ledgersField.setAccessible(true); + NavigableMap ledgers = + (NavigableMap) + ledgersField.get(persistentTopic.getManagedLedger()); + ledgers.forEach((k, v) -> { + ledgers.remove(k); + }); + newProcessor.trimExpiredAbortedTxns(); + //4. Verify the two sealed segment will be deleted. + Awaitility.await().untilAsserted(() -> verifyAbortedTxnIDAndSegmentIndex(newProcessor, 11, 4)); + } + + private void waitTaskExecuteCompletely(AbortedTxnProcessor processor) throws Exception { + Field workerField = SnapshotSegmentAbortedTxnProcessorImpl.class.getDeclaredField("persistentWorker"); + workerField.setAccessible(true); + SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker persistentWorker = + (SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker) workerField.get(processor); + Field taskQueueField = SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker.class + .getDeclaredField("taskQueue"); + taskQueueField.setAccessible(true); + Queue queue = (Queue) taskQueueField.get(persistentWorker); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(queue.size(), 0)); + } + + private void verifyAbortedTxnIDAndSegmentIndex(AbortedTxnProcessor processor, int begin, int txnIdSize) + throws Exception { + //Verify the checking of the aborted txn IDs + for (int i = begin; i < txnIdSize; i++) { + Assert.assertTrue(processor.checkAbortedTransaction(new TxnID(0, i))); + } + //Verify there are 2 sealed segment and the unsealed segment size is 4. + Field unsealedSegmentField = SnapshotSegmentAbortedTxnProcessorImpl.class + .getDeclaredField("unsealedAbortedTxnIdSegment"); + Field indexField = SnapshotSegmentAbortedTxnProcessorImpl.class + .getDeclaredField("segmentIndex"); + unsealedSegmentField.setAccessible(true); + indexField.setAccessible(true); + LinkedList unsealedSegment = (LinkedList) unsealedSegmentField.get(processor); + LinkedMap indexes = (LinkedMap) indexField.get(processor); + Assert.assertEquals(unsealedSegment.size(), txnIdSize % SEGMENT_SIZE); + Assert.assertEquals(indexes.size(), txnIdSize / SEGMENT_SIZE); + } + + // Verify the update index future can be completed when the queue has other tasks. + public void verifyFuturesCanCompleteWithException(AbortedTxnProcessor processor) throws Exception { + Field workerField = SnapshotSegmentAbortedTxnProcessorImpl.class.getDeclaredField("persistentWorker"); + workerField.setAccessible(true); + SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker persistentWorker = + (SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker) workerField.get(processor); + Field taskQueueField = SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker.class + .getDeclaredField("taskQueue"); + taskQueueField.setAccessible(true); + Queue queue = (Queue) taskQueueField.get(persistentWorker); + queue.add(new Object()); + processor.takeAbortedTxnsSnapshot(new PositionImpl(1, 10)).get(2, TimeUnit.SECONDS); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index e593c11a8bd97..d563f8e06b8f0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -825,6 +825,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob assertEquals(snapshot.getAborts().toArray()[0], new TxnIDData(1, 1)); } + //Verify the snapshotSegmentProcessor end to end @Test public void testSnapshotSegment() throws Exception { String topic = NAMESPACE1 + "/testSnapshotSegment"; @@ -870,7 +871,6 @@ public void testSnapshotSegment() throws Exception { } else { ongoingTxns.put(transaction, messageId); if (maxReadMessage == null) { - log.info("Max read Position in test: [{}]", messageId); //The except number of the messages that can be read maxReadMessage = messageId; } From dfc30f04588b26b6f85bc35a257128e2b99d0bf0 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Fri, 13 Jan 2023 19:39:38 +0800 Subject: [PATCH 18/41] optimize some comments --- ...napshotSegmentAbortedTxnProcessorImpl.java | 188 +++++++----------- .../transaction/AbortTxnProcessorTest.java | 10 +- 2 files changed, 84 insertions(+), 114 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index b28aff59fed36..4e85ad9804459 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -30,7 +30,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import java.util.function.Function; +import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; @@ -92,8 +92,9 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; this.persistentWorker = new PersistentWorker(topic); - this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = topic.getBrokerService().getPulsar() - .getConfiguration().getTransactionBufferSnapshotSegmentSize(); + //Cumulative the segment capital according to its size configuration. + this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = (topic.getBrokerService().getPulsar() + .getConfiguration().getTransactionBufferSnapshotSegmentSize() - 8 - topic.getName().length()) / 3; this.unsealedAbortedTxnIdSegment = new LinkedList<>(); } @@ -105,7 +106,7 @@ public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMar if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { LinkedList abortedSegment = unsealedAbortedTxnIdSegment; segmentIndex.put(abortedMarkerPersistentPosition, abortedTxnId); - persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, (cancel) -> + persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> persistentWorker.takeSnapshotSegmentAsync(abortedSegment, abortedMarkerPersistentPosition)); this.unsealedAbortedTxnIdSegment = new LinkedList<>(); } @@ -131,21 +132,15 @@ public void trimExpiredAbortedTxns() { positionsNeedToDelete.add(positionNeedToDelete); TxnID theLatestDeletedTxnID = segmentIndex.remove(positionNeedToDelete); - //The position is already deleted. - if (theLatestDeletedTxnID == null) { - continue; - } while (!aborts.firstKey().equals(theLatestDeletedTxnID)) { aborts.remove(aborts.firstKey()); } aborts.remove(theLatestDeletedTxnID); } - //Batch delete the expired segment and then update segment index. + //Batch delete the expired segment if (!positionsNeedToDelete.isEmpty()) { persistentWorker.appendTask(PersistentWorker.OperationType.DeleteSegment, - (cancel) -> persistentWorker.deleteSnapshotSegment(positionsNeedToDelete) - .thenRun(() -> persistentWorker.updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), - indexes.values().stream().toList()))); + () -> persistentWorker.deleteSnapshotSegment(positionsNeedToDelete)); } } @@ -162,22 +157,8 @@ public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosit } private CompletableFuture updateSnapshotIndexMetadata(TransactionBufferSnapshotIndexesMetadata metadata) { - CompletableFuture completableFuture = new CompletableFuture<>(); - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, (cancel) -> { - if (!cancel) { - return persistentWorker - .updateSnapshotIndex(metadata, persistentSnapshotIndexes.getIndexList()) - .thenRun(() -> completableFuture.complete(null)) - .exceptionally(e -> { - completableFuture.completeExceptionally(e); - return null; - }); - } else { - completableFuture.complete(null); - return completableFuture; - } - }); - return completableFuture; + return persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> persistentWorker + .updateSnapshotIndex(metadata, persistentSnapshotIndexes.getIndexList())); } @Override @@ -304,7 +285,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob //This is a compensation mechanism for deleting the segment // but not successfully updating the index. if (hasInvalidIndex.get()) { - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, (cancel) + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> persistentWorker .updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), indexes.values().stream().toList())); @@ -327,16 +308,8 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob @Override public CompletableFuture clearAbortedTxnSnapshot() { - CompletableFuture completableFuture = new CompletableFuture<>(); - persistentWorker.appendTask(PersistentWorker.OperationType.Clear, - (cancel) -> persistentWorker.clearSnapshotSegmentAndIndexes() - .thenRun(() -> { - completableFuture.thenCompose(null); - }).exceptionally(e -> { - completableFuture.completeExceptionally(e); - return null; - })); - return completableFuture; + return persistentWorker.appendTask(PersistentWorker.OperationType.Clear, + persistentWorker::clearSnapshotSegmentAndIndexes); } @Override @@ -405,15 +378,19 @@ public enum OperationType { private volatile OperationState operationState = OperationState.None; - ConcurrentLinkedDeque>>> taskQueue = - new ConcurrentLinkedDeque<>(); - private CompletableFuture lastOperationFuture; + ConcurrentLinkedDeque, + Supplier>>>> taskQueue = new ConcurrentLinkedDeque<>(); public PersistentWorker(PersistentTopic topic) { this.topic = topic; this.snapshotSegmentsWriterFuture = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName())); + .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName())) + .exceptionally(ex -> { + log.error("{} Failed to create snapshot index writer", topic.getName()); + topic.close(); + return null; + }); this.snapshotIndexWriterFuture = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName())) @@ -421,45 +398,64 @@ public PersistentWorker(PersistentTopic topic) { log.error("{} Failed to create snapshot writer", topic.getName()); topic.close(); return null; - });; + }); } - public void appendTask(OperationType operationType, Function> task) { + public CompletableFuture appendTask(OperationType operationType, + Supplier> task) { + CompletableFuture taskExecutedResult = new CompletableFuture<>(); switch (operationType) { //Update index is can be canceled when the task queue is not empty, so it should be executed immediately // instead of taking in queue. If the task queue is not empty, execute the task from the queue. case UpdateIndex -> { if (!taskQueue.isEmpty()) { - task.apply(Boolean.TRUE); topic.getBrokerService().getPulsar().getTransactionExecutorProvider() .getExecutor(this).submit(this::executeTask); + return CompletableFuture.completedFuture(null); } else if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { - lastOperationFuture = task.apply(Boolean.FALSE); - lastOperationFuture.whenComplete((ignore, throwable) -> { + return task.get().whenComplete((ignore, throwable) -> { if (throwable != null && log.isDebugEnabled()) { log.debug("[{}] Failed to update index snapshot", topic.getName(), throwable); } STATE_UPDATER.compareAndSet(this, OperationState.Operating, OperationState.None); }); + } else { + return CompletableFuture.completedFuture(null); } } case WriteSegment, DeleteSegment -> { if (!STATE_UPDATER.get(this).equals(OperationState.Closed)) { - taskQueue.add(new MutablePair<>(operationType, task)); + taskQueue.add(new MutablePair<>(operationType, new MutablePair<>(taskExecutedResult, task))); executeTask(); + return taskExecutedResult; + } else { + return CompletableFuture.completedFuture(null); } } //If the operation type is clear, all the operation in the queue is meaningless. case Clear -> { - STATE_UPDATER.set(this, OperationState.Closed); - taskQueue.forEach(pair -> { - pair.getRight().apply(Boolean.FALSE).completeExceptionally( - new BrokerServiceException.TransactionBufferClosedException( - String.format("Cancel the operation [%s] due to the" - + " transaction buffer of the topic[%s] already closed", - pair.getLeft().name(), this.topic.getName()))); - }); - taskQueue.clear(); + //Do not clear the snapshots if the topic is used. + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Closed)) { + taskQueue.forEach(pair -> + pair.getRight().getRight().get().completeExceptionally( + new BrokerServiceException.TransactionBufferClosedException( + String.format("Cancel the operation [%s] due to the" + + " transaction buffer of the topic[%s] already closed", + pair.getLeft().name(), this.topic.getName())))); + taskQueue.clear(); + return task.get(); + } else { + return FutureUtil.failedFuture( + new BrokerServiceException.NotAllowedException( + String.format("Failed to clear the snapshot of topic [%s] due to " + + "the topic is used. Please stop the using of the topic " + + "and try it again", this.topic.getName()))); + } + } + default -> { + return FutureUtil.failedFuture(new BrokerServiceException + .NotAllowedException(String.format("Th operation [%s] is unsupported", + operationType.name()))); } } } @@ -467,46 +463,23 @@ public void appendTask(OperationType operationType, Function { - if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { - if (taskQueue.getFirst().getKey() == OperationType.WriteSegment) { - lastOperationFuture = taskQueue.getFirst().getValue().apply(Boolean.FALSE); - lastOperationFuture.whenComplete((ignore, throwable) -> { - if (throwable != null) { - if (log.isDebugEnabled()) { - log.debug("[{}] Failed to write snapshot segment", topic.getName(), throwable); - } - } else { - taskQueue.removeFirst(); - } - STATE_UPDATER.compareAndSet(this, OperationState.Operating, OperationState.None); - topic.getBrokerService().getPulsar().getTransactionExecutorProvider() - .getExecutor(this).submit(this::executeTask); - }); - } - } - } - case DeleteSegment -> { - if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { - if (taskQueue.getFirst().getKey() == OperationType.DeleteSegment) { - lastOperationFuture = taskQueue.getFirst().getValue().apply(Boolean.FALSE); - lastOperationFuture.whenComplete((ignore, throwable) -> { - if (throwable != null) { - if (log.isDebugEnabled()) { - log.debug("[{}] Failed to delete snapshot segment", topic.getName(), throwable); - } - } else { - taskQueue.removeFirst(); - } - - STATE_UPDATER.compareAndSet(this, OperationState.Operating, OperationState.None); - topic.getBrokerService().getPulsar().getTransactionExecutorProvider() - .getExecutor(this).submit(this::executeTask); - }); + if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { + taskQueue.getFirst().getValue().getRight().get().whenComplete((ignore, throwable) -> { + if (throwable != null) { + if (log.isDebugEnabled()) { + log.debug("[{}] Failed to do operation do operation of [{}]", + topic.getName(), operationType.name(), throwable); } + taskQueue.getFirst().getRight().getKey().completeExceptionally(throwable); + } else { + taskQueue.getFirst().getRight().getKey().complete(null); + taskQueue.removeFirst(); } - } + STATE_UPDATER.compareAndSet(this, OperationState.Operating, + OperationState.None); + topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this).submit(this::executeTask); + }); } } @@ -605,21 +578,17 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd } private CompletableFuture clearSnapshotSegmentAndIndexes() { - ArrayList> completableFutures = new ArrayList<>(); //Delete all segment - completableFutures.add(persistentWorker.deleteSnapshotSegment(segmentIndex.keySet().stream().toList())); - //Delete index - return FutureUtil.waitForAll(completableFutures) + return persistentWorker.deleteSnapshotSegment(segmentIndex.keySet().stream().toList()) + //Delete index .thenCompose((ignore) -> snapshotIndexWriterFuture .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) - .thenRun(() -> { - log.info("Successes to clear the snapshot segment and indexes for the topic [{}]", - topic.getName()); - }) + .thenRun(() -> + log.debug("Successes to clear the snapshot segment and indexes for the topic [{}]", + topic.getName())) .exceptionally(e -> { log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", topic.getName(), e); - return null; }); } @@ -634,17 +603,14 @@ CompletableFuture closeAsync() { private LinkedList convertTypeToTxnID(List snapshotSegment) { LinkedList abortedTxns = new LinkedList<>(); - snapshotSegment.forEach(txnIDData -> { - abortedTxns.add(new TxnID(txnIDData.getMostSigBits(), txnIDData.getLeastSigBits())); - }); + snapshotSegment.forEach(txnIDData -> + abortedTxns.add(new TxnID(txnIDData.getMostSigBits(), txnIDData.getLeastSigBits()))); return abortedTxns; } private List convertTypeToTxnIDData(LinkedList abortedTxns) { List segment = new LinkedList<>(); - abortedTxns.forEach(txnID -> { - segment.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits())); - }); + abortedTxns.forEach(txnID -> segment.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits()))); return segment; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java index e863cf236182c..12246790ab9ba 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java @@ -24,7 +24,7 @@ @Slf4j public class AbortTxnProcessorTest extends TransactionTestBase { - private static final String PROCESSOR_TOPIC = NAMESPACE1 + "/abortedTxnProcessor"; + private static final String PROCESSOR_TOPIC = "persistent://" + NAMESPACE1 + "/abortedTxnProcessor"; private static final int SEGMENT_SIZE = 5; private PulsarService pulsarService = null; @@ -34,7 +34,7 @@ protected void setup() throws Exception { setUpBase(1, 1, PROCESSOR_TOPIC, 0); this.pulsarService = getPulsarServiceList().get(0); this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); - this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(SEGMENT_SIZE); + this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 + PROCESSOR_TOPIC.length() + 5 * 3); } @Override @@ -128,7 +128,11 @@ private void verifyAbortedTxnIDAndSegmentIndex(AbortedTxnProcessor processor, in } // Verify the update index future can be completed when the queue has other tasks. - public void verifyFuturesCanCompleteWithException(AbortedTxnProcessor processor) throws Exception { + @Test + public void testFuturesCanCompleteWithException() throws Exception { + PersistentTopic persistentTopic = (PersistentTopic) pulsarService.getBrokerService() + .getTopic(PROCESSOR_TOPIC, false).get().get(); + AbortedTxnProcessor processor = new SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic); Field workerField = SnapshotSegmentAbortedTxnProcessorImpl.class.getDeclaredField("persistentWorker"); workerField.setAccessible(true); SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker persistentWorker = From 8418e34ba94f1ae6dbd6ed28eb9bfec18ff93874 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Fri, 13 Jan 2023 20:06:03 +0800 Subject: [PATCH 19/41] delete persistentSnapshotIndexes --- .../SnapshotSegmentAbortedTxnProcessorImpl.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 4e85ad9804459..b1617d032ee97 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -76,9 +76,6 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess private final LinkedMap aborts = new LinkedMap<>(); //The indexes of the snapshot segments whose key is the aborted mark persistent position. private final LinkedMap indexes = new LinkedMap<>(); - //The latest persistent snapshot index. This is used to combine new segment indexes with the latest metadata and - // indexes. - private TransactionBufferSnapshotIndexes persistentSnapshotIndexes = new TransactionBufferSnapshotIndexes(); private final PersistentTopic topic; @@ -158,7 +155,7 @@ public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosit private CompletableFuture updateSnapshotIndexMetadata(TransactionBufferSnapshotIndexesMetadata metadata) { return persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> persistentWorker - .updateSnapshotIndex(metadata, persistentSnapshotIndexes.getIndexList())); + .updateSnapshotIndex(metadata, indexes.values().stream().toList())); } @Override @@ -167,6 +164,7 @@ public CompletableFuture recoverFromSnapshot() { .getTxnBufferSnapshotIndexService() .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { PositionImpl startReadCursorPosition = null; + TransactionBufferSnapshotIndexes persistentSnapshotIndexes = null; boolean hasIndex = false; try { //Read Index to recover the sequenceID, indexes, lastAbortedTxns and maxReadPosition. @@ -177,7 +175,7 @@ public CompletableFuture recoverFromSnapshot() { TransactionBufferSnapshotIndexes transactionBufferSnapshotIndexes = message.getValue(); if (transactionBufferSnapshotIndexes != null) { hasIndex = true; - this.persistentSnapshotIndexes = transactionBufferSnapshotIndexes; + persistentSnapshotIndexes = transactionBufferSnapshotIndexes; startReadCursorPosition = PositionImpl.get( transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionLedgerId(), transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionEntryId()); @@ -199,6 +197,7 @@ public CompletableFuture recoverFromSnapshot() { closeReader(reader); } PositionImpl finalStartReadCursorPosition = startReadCursorPosition; + TransactionBufferSnapshotIndexes finalPersistentSnapshotIndexes = persistentSnapshotIndexes; if (!hasIndex) { return CompletableFuture.completedFuture(null); } else { @@ -224,7 +223,7 @@ public CompletableFuture recoverFromSnapshot() { @Override public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { - persistentSnapshotIndexes.getIndexList().forEach(index -> { + finalPersistentSnapshotIndexes.getIndexList().forEach(index -> { CompletableFuture handleSegmentFuture = new CompletableFuture<>(); completableFutures.add(handleSegmentFuture); readOnlyManagedLedger.asyncReadEntry( @@ -287,14 +286,14 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob if (hasInvalidIndex.get()) { persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> persistentWorker - .updateSnapshotIndex(persistentSnapshotIndexes.getSnapshot(), + .updateSnapshotIndex(finalPersistentSnapshotIndexes.getSnapshot(), indexes.values().stream().toList())); } //Append the aborted txn IDs in the index metadata //can keep the order of the aborted txn in the aborts. //So that we can trim the expired snapshot segment in aborts // according to the latest txn IDs - convertTypeToTxnID(persistentSnapshotIndexes.getSnapshot().getAborts()) + convertTypeToTxnID(finalPersistentSnapshotIndexes.getSnapshot().getAborts()) .forEach(txnID -> aborts.put(txnID, txnID)); return CompletableFuture.completedFuture(finalStartReadCursorPosition); })).exceptionally(ex -> { @@ -568,7 +567,6 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd return indexesWriter.writeAsync(topic.getName(), snapshotIndexes); }) .thenRun(() -> { - persistentSnapshotIndexes = snapshotIndexes; lastSnapshotTimestamps = System.currentTimeMillis(); }) .exceptionally(e -> { From c8a134412e5791b5b662718000d22f63ecc77944 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Fri, 13 Jan 2023 21:57:49 +0800 Subject: [PATCH 20/41] fix test --- .../buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java | 2 +- .../apache/pulsar/broker/transaction/AbortTxnProcessorTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index b1617d032ee97..1905b75efc99e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -100,7 +100,7 @@ public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMar unsealedAbortedTxnIdSegment.add(abortedTxnId); aborts.put(abortedTxnId, abortedTxnId); //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. - if (unsealedAbortedTxnIdSegment.size() == transactionBufferMaxAbortedTxnsOfSnapshotSegment) { + if (unsealedAbortedTxnIdSegment.size() >= transactionBufferMaxAbortedTxnsOfSnapshotSegment) { LinkedList abortedSegment = unsealedAbortedTxnIdSegment; segmentIndex.put(abortedMarkerPersistentPosition, abortedTxnId); persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java index 12246790ab9ba..4791f6c3c6a05 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java @@ -76,7 +76,7 @@ public void testPutAbortedTxnIntoProcessor() throws Exception { AbortedTxnProcessor newProcessor = new SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic); PositionImpl maxReadPosition = new PositionImpl(0, 14); //2.1 Avoid update operation being canceled. - waitTaskExecuteCompletely(newProcessor); + waitTaskExecuteCompletely(processor); //2.2 take the latest snapshot processor.takeAbortedTxnsSnapshot(maxReadPosition).get(); newProcessor.recoverFromSnapshot().get(); From 374f6008f346397139fdb31c8df31b4d9b84cd30 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Thu, 19 Jan 2023 22:48:23 +0800 Subject: [PATCH 21/41] fix some comments and add notes --- ...napshotSegmentAbortedTxnProcessorImpl.java | 208 +++++++++++++----- 1 file changed, 152 insertions(+), 56 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 1905b75efc99e..89a3bd02b9d4a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -64,24 +64,67 @@ @Slf4j public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcessor { - //Stored the unsealed aborted txn id, it will be persistent as a snapshot segment and reinit - // when its size reach the capital of a snapshot segment. - private LinkedList unsealedAbortedTxnIdSegment; - - //A mapping form the latest txn mark persistent position in a segment to its latest txn ID. - //This is mainly used to trim expired snapshot segment and remove them form aborts. + /** + * Stored the unsealed aborted transaction IDs Whose size is always less than the snapshotSegmentCapacity. + * It will be persistent as a snapshot segment when its size reach the configured capacity. + */ + private LinkedList unsealedTxnIds; + + /** + * The map is used to clear the aborted transaction IDs persistent in the expired ledger. + *

+ * The key PositionImpl {@link PositionImpl} is the persistent position of + * the latest transaction of a segment. + * The value TxnID {@link TxnID} is the latest Transaction ID in a segment. + *

+ * + *

+ * If the position is expired, the processor can get the according latest + * transaction ID in this map. And then the processor can clear all the + * transaction IDs in the aborts {@link SnapshotSegmentAbortedTxnProcessorImpl#aborts} + * that lower than the transaction ID. + * And then the processor can delete the segments persistently according to + * the positions. + *

+ */ private final LinkedMap segmentIndex = new LinkedMap<>(); - //Store all aborted txn IDs check whether a txn is an aborted txn. + /** + * This map is used to check whether a transaction is an aborted transaction. + *

+ * The transaction IDs is appended in order, so the processor can delete expired + * transaction IDs according to the latest expired transaction IDs in segmentIndex + * {@link SnapshotSegmentAbortedTxnProcessorImpl#segmentIndex}. + *

+ */ private final LinkedMap aborts = new LinkedMap<>(); - //The indexes of the snapshot segments whose key is the aborted mark persistent position. + /** + * This map stores the indexes of the snapshot segment. + *

+ * The key is the persistent position of the marker of the last transaction in the segment. + * The value TransactionBufferSnapshotIndex {@link TransactionBufferSnapshotIndex} is the + * indexes of the snapshot segment. + *

+ */ private final LinkedMap indexes = new LinkedMap<>(); private final PersistentTopic topic; private volatile long lastSnapshotTimestamps; - private final int transactionBufferMaxAbortedTxnsOfSnapshotSegment; + /** + * The number of the aborted transaction IDs in a segment. + * This is calculated according to the configured memory size. + */ + private final int snapshotSegmentCapacity; + /** + * Responsible for executing the persistent tasks. + *

Including:

+ *

Update segment index.

+ *

Write snapshot segment.

+ *

Delete snapshot segment.

+ *

Clear all snapshot segment.

+ */ private final PersistentWorker persistentWorker; private static final String SNAPSHOT_PREFIX = "multiple-"; @@ -89,23 +132,36 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; this.persistentWorker = new PersistentWorker(topic); - //Cumulative the segment capital according to its size configuration. - this.transactionBufferMaxAbortedTxnsOfSnapshotSegment = (topic.getBrokerService().getPulsar() + /** + * Calculate the segment capital according to its size configuration. + *

+ * The empty transaction segment size is 5. + * Adding a empty linkedList, the size increase to 6. + * Add the topic name the size increase to the 7 + topic.getName().length(). + * Add the aborted transaction IDs, the size increase to 8 + + * topic.getName().length() + 3 * aborted transaction ID size. + *

+ */ + this.snapshotSegmentCapacity = (topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotSegmentSize() - 8 - topic.getName().length()) / 3; - this.unsealedAbortedTxnIdSegment = new LinkedList<>(); + this.unsealedTxnIds = new LinkedList<>(); } @Override - public void putAbortedTxnAndPosition(TxnID abortedTxnId, PositionImpl abortedMarkerPersistentPosition) { - unsealedAbortedTxnIdSegment.add(abortedTxnId); - aborts.put(abortedTxnId, abortedTxnId); - //The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. - if (unsealedAbortedTxnIdSegment.size() >= transactionBufferMaxAbortedTxnsOfSnapshotSegment) { - LinkedList abortedSegment = unsealedAbortedTxnIdSegment; - segmentIndex.put(abortedMarkerPersistentPosition, abortedTxnId); + public void putAbortedTxnAndPosition(TxnID txnID, PositionImpl position) { + unsealedTxnIds.add(txnID); + aborts.put(txnID, txnID); + /** + * The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. + * Append a task to persistent the segment with the aborted transaction IDs and the latest + * transaction mark persistent position passed by param. + */ + if (unsealedTxnIds.size() >= snapshotSegmentCapacity) { + LinkedList abortedSegment = unsealedTxnIds; + segmentIndex.put(position, txnID); persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> - persistentWorker.takeSnapshotSegmentAsync(abortedSegment, abortedMarkerPersistentPosition)); - this.unsealedAbortedTxnIdSegment = new LinkedList<>(); + persistentWorker.takeSnapshotSegmentAsync(abortedSegment, position)); + this.unsealedTxnIds = new LinkedList<>(); } } @@ -114,7 +170,12 @@ public boolean checkAbortedTransaction(TxnID txnID) { return aborts.containsKey(txnID); } - //In this implementation, we adopt snapshot segments. And then we clear invalid segment by its max read position. + /** + * Check werther the position in segmentIndex {@link SnapshotSegmentAbortedTxnProcessorImpl#segmentIndex} + * is expired. If the position is not exist in the original topic, the according transaction is an invalid + * transaction. And the according segment is invalid, too. The transaction IDs before the transaction ID + * in the aborts are invalid, too. + */ @Override public void trimExpiredAbortedTxns() { //Checking whether there are some segment expired. @@ -128,11 +189,11 @@ public void trimExpiredAbortedTxns() { PositionImpl positionNeedToDelete = segmentIndex.firstKey(); positionsNeedToDelete.add(positionNeedToDelete); - TxnID theLatestDeletedTxnID = segmentIndex.remove(positionNeedToDelete); + TxnID theLatestDeletedTxnID = segmentIndex.remove(0); while (!aborts.firstKey().equals(theLatestDeletedTxnID)) { - aborts.remove(aborts.firstKey()); + aborts.remove(0); } - aborts.remove(theLatestDeletedTxnID); + aborts.remove(0); } //Batch delete the expired segment if (!positionsNeedToDelete.isEmpty()) { @@ -147,13 +208,10 @@ private String buildKey(long sequenceId) { @Override public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosition) { + //Store the latest aborted transaction IDs in unsealedTxnIDs and the according the latest max read position. TransactionBufferSnapshotIndexesMetadata metadata = new TransactionBufferSnapshotIndexesMetadata( maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), - convertTypeToTxnIDData(unsealedAbortedTxnIdSegment)); - return updateSnapshotIndexMetadata(metadata); - } - - private CompletableFuture updateSnapshotIndexMetadata(TransactionBufferSnapshotIndexesMetadata metadata) { + convertTypeToTxnIDData(unsealedTxnIds)); return persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> persistentWorker .updateSnapshotIndex(metadata, indexes.values().stream().toList())); } @@ -167,7 +225,15 @@ public CompletableFuture recoverFromSnapshot() { TransactionBufferSnapshotIndexes persistentSnapshotIndexes = null; boolean hasIndex = false; try { - //Read Index to recover the sequenceID, indexes, lastAbortedTxns and maxReadPosition. + /** + * Read the transaction snapshot segment index. + *

+ * The processor can get the sequence ID, unsealed transaction IDs, + * segment index list and max read position in the snapshot segment index. + * Then we can traverse the index list to read all aborted transaction IDs + * in segments to aborts. + *

+ */ while (reader.hasMoreEvents()) { Message message = reader.readNextAsync() .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); @@ -207,9 +273,11 @@ public CompletableFuture recoverFromSnapshot() { transactionBufferSnapshotIndex.abortedMarkLedgerID, transactionBufferSnapshotIndex.abortedMarkEntryID), transactionBufferSnapshotIndex)); - this.unsealedAbortedTxnIdSegment = convertTypeToTxnID(persistentSnapshotIndexes + this.unsealedTxnIds = convertTypeToTxnID(persistentSnapshotIndexes .getSnapshot().getAborts()); - //If the size of indexes is 0, the sequence ID will be the init value 0. + /** + * If there is no segment index, the persistent worker will write segment begin from 0. + */ if (indexes.size() != 0) { persistentWorker.sequenceID.set(indexes.get(indexes.lastKey()).sequenceID + 1); } @@ -241,12 +309,21 @@ public void readEntryComplete(Entry entry, Object ctx) { public void readEntryFailed(ManagedLedgerException exception, Object ctx) { if (exception instanceof ManagedLedgerException .NonRecoverableLedgerException) { + /** + * The logic flow of deleting expired segment is: + *

+ * 1. delete segment + * 2. update segment index + *

+ * If the worker delete segment successfully + * but failed to update segment index, + * the segment can not be read according to the index + */ if (((ManagedLedgerImpl)topic.getManagedLedger()) .ledgerExists(index.getAbortedMarkLedgerID())) { log.error("[{}] Failed to read snapshot segment [{}:{}]", topic.getName(), index.segmentLedgerID, index.segmentEntryID, exception); - topic.close(); handleSegmentFuture.completeExceptionally(exception); } else { indexes.remove(new PositionImpl( @@ -276,27 +353,32 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob .getPersistenceNamingEncoding(), callback, topic.getManagedLedger().getConfig(), null); - //Wait the processor recover completely and the allow TB to recover the messages - // after the startReadCursorPosition. - + /** + * Wait the processor recover completely and then allow TB + * to recover the messages after the startReadCursorPosition. + */ return openManagedLedgerFuture - .thenCompose((ignore) -> FutureUtil.waitForAll(completableFutures).thenCompose((i) -> { - //This is a compensation mechanism for deleting the segment - // but not successfully updating the index. + .thenCompose((ignore) -> FutureUtil.waitForAll(completableFutures)) + .thenCompose((i) -> { + /** + * Update the snapshot segment index if there exist invalid indexes. + */ if (hasInvalidIndex.get()) { persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> persistentWorker .updateSnapshotIndex(finalPersistentSnapshotIndexes.getSnapshot(), indexes.values().stream().toList())); } - //Append the aborted txn IDs in the index metadata - //can keep the order of the aborted txn in the aborts. - //So that we can trim the expired snapshot segment in aborts - // according to the latest txn IDs + /** + * Append the aborted txn IDs in the index metadata + * can keep the order of the aborted txn in the aborts. + * So that we can trim the expired snapshot segment in aborts + * according to the latest transaction IDs in the segmentIndex. + */ convertTypeToTxnID(finalPersistentSnapshotIndexes.getSnapshot().getAborts()) .forEach(txnID -> aborts.put(txnID, txnID)); return CompletableFuture.completedFuture(finalStartReadCursorPosition); - })).exceptionally(ex -> { + }).exceptionally(ex -> { log.error("[{}] Failed to recover snapshot segment", this.topic.getName(), ex); return null; }); @@ -404,9 +486,12 @@ public CompletableFuture appendTask(OperationType operationType, Supplier> task) { CompletableFuture taskExecutedResult = new CompletableFuture<>(); switch (operationType) { - //Update index is can be canceled when the task queue is not empty, so it should be executed immediately - // instead of taking in queue. If the task queue is not empty, execute the task from the queue. case UpdateIndex -> { + /** + * The update index operation can be canceled when the task queue is not empty, + * so it should be executed immediately instead of appending to the task queue. + * If the taskQueue is not empty, the worker will execute the task in the queue. + */ if (!taskQueue.isEmpty()) { topic.getBrokerService().getPulsar().getTransactionExecutorProvider() .getExecutor(this).submit(this::executeTask); @@ -422,6 +507,10 @@ public CompletableFuture appendTask(OperationType operationType, return CompletableFuture.completedFuture(null); } } + /** + * Only the operations of WriteSegment and DeleteSegment will be appended into the taskQueue. + * The operation will be canceled when the worker is close which means the topic is deleted. + */ case WriteSegment, DeleteSegment -> { if (!STATE_UPDATER.get(this).equals(OperationState.Closed)) { taskQueue.add(new MutablePair<>(operationType, new MutablePair<>(taskExecutedResult, task))); @@ -431,9 +520,11 @@ public CompletableFuture appendTask(OperationType operationType, return CompletableFuture.completedFuture(null); } } - //If the operation type is clear, all the operation in the queue is meaningless. case Clear -> { - //Do not clear the snapshots if the topic is used. + /** + * Do not clear the snapshots if the topic is used. + * If the users want to delete a topic, he should stop the usage of the topic. + */ if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Closed)) { taskQueue.forEach(pair -> pair.getRight().getRight().get().completeExceptionally( @@ -442,6 +533,9 @@ public CompletableFuture appendTask(OperationType operationType, + " transaction buffer of the topic[%s] already closed", pair.getLeft().name(), this.topic.getName())))); taskQueue.clear(); + /** + * The task of clear all snapshot segment and index is executed immediately too. + */ return task.get(); } else { return FutureUtil.failedFuture( @@ -461,21 +555,24 @@ public CompletableFuture appendTask(OperationType operationType, private void executeTask() { if (taskQueue.isEmpty()) return; - OperationType operationType = taskQueue.getFirst().getKey(); if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { - taskQueue.getFirst().getValue().getRight().get().whenComplete((ignore, throwable) -> { + Pair, Supplier>>> firstTask = + taskQueue.getFirst(); + if (firstTask == null) return; + firstTask.getValue().getRight().get().whenComplete((ignore, throwable) -> { if (throwable != null) { if (log.isDebugEnabled()) { log.debug("[{}] Failed to do operation do operation of [{}]", - topic.getName(), operationType.name(), throwable); + topic.getName(), firstTask.getKey().name(), throwable); } - taskQueue.getFirst().getRight().getKey().completeExceptionally(throwable); + firstTask.getRight().getKey().completeExceptionally(throwable); } else { - taskQueue.getFirst().getRight().getKey().complete(null); + firstTask.getRight().getKey().complete(null); taskQueue.removeFirst(); } STATE_UPDATER.compareAndSet(this, OperationState.Operating, OperationState.None); + //Execute the next task in the task queue. topic.getBrokerService().getPulsar().getTransactionExecutorProvider() .getExecutor(this).submit(this::executeTask); }); @@ -556,7 +653,6 @@ private CompletableFuture deleteSnapshotSegment(List positio return FutureUtil.waitForAll(results); } - //Update the indexes with the giving index snapshot and index list in the transactionBufferSnapshotIndexes. private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotIndexesMetadata snapshotSegment, List indexList) { TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); @@ -606,7 +702,7 @@ private LinkedList convertTypeToTxnID(List snapshotSegment) { return abortedTxns; } - private List convertTypeToTxnIDData(LinkedList abortedTxns) { + private List convertTypeToTxnIDData(List abortedTxns) { List segment = new LinkedList<>(); abortedTxns.forEach(txnID -> segment.add(new TxnIDData(txnID.getMostSigBits(), txnID.getLeastSigBits()))); return segment; From 96e34d519d0c650bc7fef2f737d0865703239936 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Thu, 19 Jan 2023 22:58:20 +0800 Subject: [PATCH 22/41] optimize recover indexes --- ...napshotSegmentAbortedTxnProcessorImpl.java | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 89a3bd02b9d4a..7cc7f7ba3082f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -267,20 +267,8 @@ public CompletableFuture recoverFromSnapshot() { if (!hasIndex) { return CompletableFuture.completedFuture(null); } else { - persistentSnapshotIndexes.getIndexList() - .forEach(transactionBufferSnapshotIndex -> - indexes.put(new PositionImpl( - transactionBufferSnapshotIndex.abortedMarkLedgerID, - transactionBufferSnapshotIndex.abortedMarkEntryID), - transactionBufferSnapshotIndex)); this.unsealedTxnIds = convertTypeToTxnID(persistentSnapshotIndexes .getSnapshot().getAborts()); - /** - * If there is no segment index, the persistent worker will write segment begin from 0. - */ - if (indexes.size() != 0) { - persistentWorker.sequenceID.set(indexes.get(indexes.lastKey()).sequenceID + 1); - } } //Read snapshot segment to recover aborts. ArrayList> completableFutures = new ArrayList<>(); @@ -301,6 +289,10 @@ public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnly @Override public void readEntryComplete(Entry entry, Object ctx) { handleSnapshotSegmentEntry(entry); + indexes.put(new PositionImpl( + index.abortedMarkLedgerID, + index.abortedMarkEntryID), + index); entry.release(); handleSegmentFuture.complete(null); } @@ -317,7 +309,8 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { *

* If the worker delete segment successfully * but failed to update segment index, - * the segment can not be read according to the index + * the segment can not be read according to the index. + * We update index again if there are invalid indexes. */ if (((ManagedLedgerImpl)topic.getManagedLedger()) .ledgerExists(index.getAbortedMarkLedgerID())) { @@ -326,9 +319,6 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { index.segmentEntryID, exception); handleSegmentFuture.completeExceptionally(exception); } else { - indexes.remove(new PositionImpl( - index.getAbortedMarkLedgerID(), - index.getAbortedMarkEntryID())); hasInvalidIndex.set(true); } } @@ -369,6 +359,12 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob .updateSnapshotIndex(finalPersistentSnapshotIndexes.getSnapshot(), indexes.values().stream().toList())); } + /** + * If there is no segment index, the persistent worker will write segment begin from 0. + */ + if (indexes.size() != 0) { + persistentWorker.sequenceID.set(indexes.get(indexes.lastKey()).sequenceID + 1); + } /** * Append the aborted txn IDs in the index metadata * can keep the order of the aborted txn in the aborts. From 8fa06062a8ba0e2826e4596d27a66952ff0379f2 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Sat, 28 Jan 2023 12:28:56 +0800 Subject: [PATCH 23/41] optimize notes --- ...napshotSegmentAbortedTxnProcessorImpl.java | 130 ++++++++++-------- 1 file changed, 72 insertions(+), 58 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 7cc7f7ba3082f..5675a2e9ccec1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -132,15 +132,15 @@ public class SnapshotSegmentAbortedTxnProcessorImpl implements AbortedTxnProcess public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; this.persistentWorker = new PersistentWorker(topic); - /** - * Calculate the segment capital according to its size configuration. - *

- * The empty transaction segment size is 5. - * Adding a empty linkedList, the size increase to 6. - * Add the topic name the size increase to the 7 + topic.getName().length(). - * Add the aborted transaction IDs, the size increase to 8 + - * topic.getName().length() + 3 * aborted transaction ID size. - *

+ /* + Calculate the segment capital according to its size configuration. +

+ The empty transaction segment size is 5. + Adding a empty linkedList, the size increase to 6. + Add the topic name the size increase to the 7 + topic.getName().length(). + Add the aborted transaction IDs, the size increase to 8 + + topic.getName().length() + 3 * aborted transaction ID size. +

*/ this.snapshotSegmentCapacity = (topic.getBrokerService().getPulsar() .getConfiguration().getTransactionBufferSnapshotSegmentSize() - 8 - topic.getName().length()) / 3; @@ -151,10 +151,10 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { public void putAbortedTxnAndPosition(TxnID txnID, PositionImpl position) { unsealedTxnIds.add(txnID); aborts.put(txnID, txnID); - /** - * The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. - * Append a task to persistent the segment with the aborted transaction IDs and the latest - * transaction mark persistent position passed by param. + /* + The size of lastAbortedTxns reaches the configuration of the size of snapshot segment. + Append a task to persistent the segment with the aborted transaction IDs and the latest + transaction mark persistent position passed by param. */ if (unsealedTxnIds.size() >= snapshotSegmentCapacity) { LinkedList abortedSegment = unsealedTxnIds; @@ -225,14 +225,14 @@ public CompletableFuture recoverFromSnapshot() { TransactionBufferSnapshotIndexes persistentSnapshotIndexes = null; boolean hasIndex = false; try { - /** - * Read the transaction snapshot segment index. - *

- * The processor can get the sequence ID, unsealed transaction IDs, - * segment index list and max read position in the snapshot segment index. - * Then we can traverse the index list to read all aborted transaction IDs - * in segments to aborts. - *

+ /* + Read the transaction snapshot segment index. +

+ The processor can get the sequence ID, unsealed transaction IDs, + segment index list and max read position in the snapshot segment index. + Then we can traverse the index list to read all aborted transaction IDs + in segments to aborts. +

*/ while (reader.hasMoreEvents()) { Message message = reader.readNextAsync() @@ -301,16 +301,16 @@ public void readEntryComplete(Entry entry, Object ctx) { public void readEntryFailed(ManagedLedgerException exception, Object ctx) { if (exception instanceof ManagedLedgerException .NonRecoverableLedgerException) { - /** - * The logic flow of deleting expired segment is: - *

- * 1. delete segment - * 2. update segment index - *

- * If the worker delete segment successfully - * but failed to update segment index, - * the segment can not be read according to the index. - * We update index again if there are invalid indexes. + /* + The logic flow of deleting expired segment is: +

+ 1. delete segment + 2. update segment index +

+ If the worker delete segment successfully + but failed to update segment index, + the segment can not be read according to the index. + We update index again if there are invalid indexes. */ if (((ManagedLedgerImpl)topic.getManagedLedger()) .ledgerExists(index.getAbortedMarkLedgerID())) { @@ -343,15 +343,15 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob .getPersistenceNamingEncoding(), callback, topic.getManagedLedger().getConfig(), null); - /** - * Wait the processor recover completely and then allow TB - * to recover the messages after the startReadCursorPosition. + /* + Wait the processor recover completely and then allow TB + to recover the messages after the startReadCursorPosition. */ return openManagedLedgerFuture .thenCompose((ignore) -> FutureUtil.waitForAll(completableFutures)) .thenCompose((i) -> { - /** - * Update the snapshot segment index if there exist invalid indexes. + /* + Update the snapshot segment index if there exist invalid indexes. */ if (hasInvalidIndex.get()) { persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () @@ -359,17 +359,17 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob .updateSnapshotIndex(finalPersistentSnapshotIndexes.getSnapshot(), indexes.values().stream().toList())); } - /** - * If there is no segment index, the persistent worker will write segment begin from 0. + /* + If there is no segment index, the persistent worker will write segment begin from 0. */ if (indexes.size() != 0) { persistentWorker.sequenceID.set(indexes.get(indexes.lastKey()).sequenceID + 1); } - /** - * Append the aborted txn IDs in the index metadata - * can keep the order of the aborted txn in the aborts. - * So that we can trim the expired snapshot segment in aborts - * according to the latest transaction IDs in the segmentIndex. + /* + Append the aborted txn IDs in the index metadata + can keep the order of the aborted txn in the aborts. + So that we can trim the expired snapshot segment in aborts + according to the latest transaction IDs in the segmentIndex. */ convertTypeToTxnID(finalPersistentSnapshotIndexes.getSnapshot().getAborts()) .forEach(txnID -> aborts.put(txnID, txnID)); @@ -426,6 +426,22 @@ private void closeReader(SystemTopicClient.Reader reader) { }); } + /** + * The PersistentWorker be responsible for executing the persistent tasks, including: + *

+ * 1. Write snapshot segment --- Encapsulate a sealed snapshot segment and persistent it. + * 2. Delete snapshot segment --- Evict expired snapshot segments. + * 3. Update snapshot indexes --- Update snapshot indexes after writing or deleting snapshot segment + * or update snapshot indexes metadata regularly. + * 4. Clear all snapshot segments and indexes. --- Executed when deleting this topic. + *

+ * * Task 1 and task 2 will be put into a task queue. The tasks in the queue will be executed in order. + * * If the task queue is empty, task 3 will be executed immediately when it is appended to the worker. + * Else, the worker will try to execute the tasks in the task queue. + * * When task 4 was appended into worker, the worker will change the operation state to closed + * and cancel all tasks in the task queue. finally, execute the task 4 (clear task). + * If there are race conditions, throw an Exception to let users try again. + */ public class PersistentWorker { protected final AtomicLong sequenceID = new AtomicLong(0); @@ -483,10 +499,10 @@ public CompletableFuture appendTask(OperationType operationType, CompletableFuture taskExecutedResult = new CompletableFuture<>(); switch (operationType) { case UpdateIndex -> { - /** - * The update index operation can be canceled when the task queue is not empty, - * so it should be executed immediately instead of appending to the task queue. - * If the taskQueue is not empty, the worker will execute the task in the queue. + /* + The update index operation can be canceled when the task queue is not empty, + so it should be executed immediately instead of appending to the task queue. + If the taskQueue is not empty, the worker will execute the tasks in the queue. */ if (!taskQueue.isEmpty()) { topic.getBrokerService().getPulsar().getTransactionExecutorProvider() @@ -503,9 +519,9 @@ public CompletableFuture appendTask(OperationType operationType, return CompletableFuture.completedFuture(null); } } - /** - * Only the operations of WriteSegment and DeleteSegment will be appended into the taskQueue. - * The operation will be canceled when the worker is close which means the topic is deleted. + /* + Only the operations of WriteSegment and DeleteSegment will be appended into the taskQueue. + The operation will be canceled when the worker is close which means the topic is deleted. */ case WriteSegment, DeleteSegment -> { if (!STATE_UPDATER.get(this).equals(OperationState.Closed)) { @@ -517,9 +533,9 @@ public CompletableFuture appendTask(OperationType operationType, } } case Clear -> { - /** - * Do not clear the snapshots if the topic is used. - * If the users want to delete a topic, he should stop the usage of the topic. + /* + Do not clear the snapshots if the topic is used. + If the users want to delete a topic, they should stop the usage of the topic. */ if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Closed)) { taskQueue.forEach(pair -> @@ -529,8 +545,8 @@ public CompletableFuture appendTask(OperationType operationType, + " transaction buffer of the topic[%s] already closed", pair.getLeft().name(), this.topic.getName())))); taskQueue.clear(); - /** - * The task of clear all snapshot segment and index is executed immediately too. + /* + The task of clear all snapshot segments and indexes is executed immediately. */ return task.get(); } else { @@ -658,9 +674,7 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd snapshotIndexes.setSnapshot(snapshotSegment); return indexesWriter.writeAsync(topic.getName(), snapshotIndexes); }) - .thenRun(() -> { - lastSnapshotTimestamps = System.currentTimeMillis(); - }) + .thenRun(() -> lastSnapshotTimestamps = System.currentTimeMillis()) .exceptionally(e -> { log.error("[{}] Failed to update snapshot segment index", snapshotIndexes.getTopicName(), e); return null; From 9e6af2a8bd88c14200a59517f64c4db2596778e0 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Sat, 28 Jan 2023 12:47:52 +0800 Subject: [PATCH 24/41] optimize notes --- .../buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 5675a2e9ccec1..61cff3e9a14db 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -136,7 +136,7 @@ public SnapshotSegmentAbortedTxnProcessorImpl(PersistentTopic topic) { Calculate the segment capital according to its size configuration.

The empty transaction segment size is 5. - Adding a empty linkedList, the size increase to 6. + Adding an empty linkedList, the size increase to 6. Add the topic name the size increase to the 7 + topic.getName().length(). Add the aborted transaction IDs, the size increase to 8 + topic.getName().length() + 3 * aborted transaction ID size. From 204a5ff8c749603a0dbd3ea2fcdb96579560e987 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Fri, 3 Feb 2023 12:02:21 +0800 Subject: [PATCH 25/41] fix some test --- .../transaction/AbortTxnProcessorTest.java | 2 +- .../TopicTransactionBufferRecoverTest.java | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java index 4791f6c3c6a05..6d8fef59b99c3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java @@ -116,7 +116,7 @@ private void verifyAbortedTxnIDAndSegmentIndex(AbortedTxnProcessor processor, in } //Verify there are 2 sealed segment and the unsealed segment size is 4. Field unsealedSegmentField = SnapshotSegmentAbortedTxnProcessorImpl.class - .getDeclaredField("unsealedAbortedTxnIdSegment"); + .getDeclaredField("unsealedTxnIds"); Field indexField = SnapshotSegmentAbortedTxnProcessorImpl.class .getDeclaredField("segmentIndex"); unsealedSegmentField.setAccessible(true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index d563f8e06b8f0..8d8063a9e250c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -80,6 +80,7 @@ import org.apache.pulsar.client.api.ReaderBuilder; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; @@ -532,9 +533,14 @@ public void clearTransactionBufferSnapshotTest(Boolean enableSnapshotSegment) th (AbortedTxnProcessor) abortedTxnProcessorField.get(topicTransactionBuffer); abortedTxnProcessor.takeAbortedTxnsSnapshot(topicTransactionBuffer.getMaxReadPosition()); - TopicName transactionBufferTopicName = - NamespaceEventsSystemTopicFactory.getSystemTopicName( - TopicName.get(topic).getNamespaceObject(), EventType.TRANSACTION_BUFFER_SNAPSHOT); + TopicName transactionBufferTopicName; + if (!enableSnapshotSegment) { + transactionBufferTopicName = NamespaceEventsSystemTopicFactory.getSystemTopicName( + TopicName.get(topic).getNamespaceObject(), EventType.TRANSACTION_BUFFER_SNAPSHOT); + } else { + transactionBufferTopicName = NamespaceEventsSystemTopicFactory.getSystemTopicName( + TopicName.get(topic).getNamespaceObject(), EventType.TRANSACTION_BUFFER_SNAPSHOT_INDEXES); + } PersistentTopic snapshotTopic = (PersistentTopic) getPulsarServiceList().get(0) .getBrokerService().getTopic(transactionBufferTopicName.toString(), false).get().get(); Field field = PersistentTopic.class.getDeclaredField("currentCompaction"); @@ -552,7 +558,7 @@ private void checkSnapshotCount(TopicName topicName, boolean hasSnapshot, CompletableFuture compactionFuture = (CompletableFuture) field.get(persistentTopic); Awaitility.await().untilAsserted(() -> assertTrue(compactionFuture.isDone())); - Reader reader = pulsarClient.newReader(Schema.AVRO(TransactionBufferSnapshot.class)) + Reader reader = pulsarClient.newReader(Schema.AUTO_CONSUME()) .readCompacted(true) .startMessageId(MessageId.earliest) .startMessageIdInclusive() @@ -561,7 +567,7 @@ private void checkSnapshotCount(TopicName topicName, boolean hasSnapshot, int count = 0; while (true) { - Message snapshotMsg = reader.readNext(2, TimeUnit.SECONDS); + Message snapshotMsg = reader.readNext(2, TimeUnit.SECONDS); if (snapshotMsg != null) { count++; } else { From b46e38b1bbf3f61306168801f2321f63f7692318 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Fri, 3 Feb 2023 12:39:00 +0800 Subject: [PATCH 26/41] fix some comment --- .../service/BrokerServiceException.java | 6 --- ...napshotSegmentAbortedTxnProcessorImpl.java | 51 +++++++++---------- ...va => SegmentAbortedTxnProcessorTest.java} | 5 +- .../TopicTransactionBufferRecoverTest.java | 5 +- 4 files changed, 30 insertions(+), 37 deletions(-) rename pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/{AbortTxnProcessorTest.java => SegmentAbortedTxnProcessorTest.java} (97%) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java index 17caf7b119215..fd3a391bca34a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java @@ -77,12 +77,6 @@ public TopicClosedException(Throwable t) { } } - public static class TransactionBufferClosedException extends BrokerServiceException { - public TransactionBufferClosedException(String s) { - super(s); - } - } - @Deprecated public static class AddEntryMetadataException extends BrokerServiceException { public AddEntryMetadataException(Throwable t) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 61cff3e9a14db..705a7b285dedf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -223,7 +223,6 @@ public CompletableFuture recoverFromSnapshot() { .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { PositionImpl startReadCursorPosition = null; TransactionBufferSnapshotIndexes persistentSnapshotIndexes = null; - boolean hasIndex = false; try { /* Read the transaction snapshot segment index. @@ -240,7 +239,6 @@ public CompletableFuture recoverFromSnapshot() { if (topic.getName().equals(message.getKey())) { TransactionBufferSnapshotIndexes transactionBufferSnapshotIndexes = message.getValue(); if (transactionBufferSnapshotIndexes != null) { - hasIndex = true; persistentSnapshotIndexes = transactionBufferSnapshotIndexes; startReadCursorPosition = PositionImpl.get( transactionBufferSnapshotIndexes.getSnapshot().getMaxReadPositionLedgerId(), @@ -264,7 +262,7 @@ public CompletableFuture recoverFromSnapshot() { } PositionImpl finalStartReadCursorPosition = startReadCursorPosition; TransactionBufferSnapshotIndexes finalPersistentSnapshotIndexes = persistentSnapshotIndexes; - if (!hasIndex) { + if (persistentSnapshotIndexes == null) { return CompletableFuture.completedFuture(null); } else { this.unsealedTxnIds = convertTypeToTxnID(persistentSnapshotIndexes @@ -299,28 +297,25 @@ public void readEntryComplete(Entry entry, Object ctx) { @Override public void readEntryFailed(ManagedLedgerException exception, Object ctx) { - if (exception instanceof ManagedLedgerException - .NonRecoverableLedgerException) { - /* - The logic flow of deleting expired segment is: -

- 1. delete segment - 2. update segment index -

- If the worker delete segment successfully - but failed to update segment index, - the segment can not be read according to the index. - We update index again if there are invalid indexes. - */ - if (((ManagedLedgerImpl)topic.getManagedLedger()) - .ledgerExists(index.getAbortedMarkLedgerID())) { - log.error("[{}] Failed to read snapshot segment [{}:{}]", - topic.getName(), index.segmentLedgerID, - index.segmentEntryID, exception); - handleSegmentFuture.completeExceptionally(exception); - } else { - hasInvalidIndex.set(true); - } + /* + The logic flow of deleting expired segment is: +

+ 1. delete segment + 2. update segment index +

+ If the worker delete segment successfully + but failed to update segment index, + the segment can not be read according to the index. + We update index again if there are invalid indexes. + */ + if (((ManagedLedgerImpl)topic.getManagedLedger()) + .ledgerExists(index.getAbortedMarkLedgerID())) { + log.error("[{}] Failed to read snapshot segment [{}:{}]", + topic.getName(), index.segmentLedgerID, + index.segmentEntryID, exception); + handleSegmentFuture.completeExceptionally(exception); + } else { + hasInvalidIndex.set(true); } } }, null); @@ -540,7 +535,7 @@ public CompletableFuture appendTask(OperationType operationType, if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Closed)) { taskQueue.forEach(pair -> pair.getRight().getRight().get().completeExceptionally( - new BrokerServiceException.TransactionBufferClosedException( + new BrokerServiceException.ServiceUnitNotReadyException( String.format("Cancel the operation [%s] due to the" + " transaction buffer of the topic[%s] already closed", pair.getLeft().name(), this.topic.getName())))); @@ -602,7 +597,7 @@ private CompletableFuture takeSnapshotSegmentAsync(LinkedList seale this.sequenceID.getAndIncrement(); }).exceptionally(e -> { //Just log the error, and the processor will try to take snapshot again when the transactionBuffer - //append aborted txn nex time. + //append aborted txn next time. log.error("Failed to take snapshot segment [{}] at maxReadPosition [{}] " + "for the topic [{}], and the size of the segment is [{}]", this.sequenceID, abortedMarkerPersistentPosition, topic.getName(), sealedAbortedTxnIdSegment.size(), e); @@ -633,6 +628,8 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm indexes.put(abortedMarkerPersistentPosition, index); //update snapshot segment index. + //If the index can not be written successfully, the snapshot segment wil be overwritten + // when the processor writes snapshot segment next time. return updateSnapshotIndex(new TransactionBufferSnapshotIndexesMetadata( maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), new LinkedList<>()), indexes.values().stream().toList()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java similarity index 97% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java index 6d8fef59b99c3..df58dce0c9e12 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/AbortTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java @@ -22,7 +22,7 @@ import org.testng.annotations.Test; @Slf4j -public class AbortTxnProcessorTest extends TransactionTestBase { +public class SegmentAbortedTxnProcessorTest extends TransactionTestBase { private static final String PROCESSOR_TOPIC = "persistent://" + NAMESPACE1 + "/abortedTxnProcessor"; private static final int SEGMENT_SIZE = 5; @@ -34,7 +34,8 @@ protected void setup() throws Exception { setUpBase(1, 1, PROCESSOR_TOPIC, 0); this.pulsarService = getPulsarServiceList().get(0); this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); - this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 + PROCESSOR_TOPIC.length() + 5 * 3); + this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 + PROCESSOR_TOPIC.length() + + SEGMENT_SIZE * 3); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 8d8063a9e250c..d4ddb26e014ca 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -834,7 +834,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob //Verify the snapshotSegmentProcessor end to end @Test public void testSnapshotSegment() throws Exception { - String topic = NAMESPACE1 + "/testSnapshotSegment"; + String topic ="persistent://" + NAMESPACE1 + "/testSnapshotSegment"; String subName = "testSnapshotSegment"; LinkedMap ongoingTxns = new LinkedMap<>(); @@ -843,7 +843,8 @@ public void testSnapshotSegment() throws Exception { int theSizeOfSegment = 10; int theCountOfSnapshotMaxTxnCount = 3; this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); - this.getPulsarServiceList().get(0).getConfig().setTransactionBufferSnapshotSegmentSize(theSizeOfSegment); + this.getPulsarServiceList().get(0).getConfig() + .setTransactionBufferSnapshotSegmentSize(8 + topic.length() + theSizeOfSegment * 3); this.getPulsarServiceList().get(0).getConfig() .setTransactionBufferSnapshotMaxTransactionCount(theCountOfSnapshotMaxTxnCount); // 1. Build producer and consumer From d2210e3a1fc477aaf3447a2f9e19d3ea05f49f29 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Fri, 3 Feb 2023 13:13:45 +0800 Subject: [PATCH 27/41] fix some comment --- .../impl/SnapshotSegmentAbortedTxnProcessorImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 705a7b285dedf..2f8d9a5daba48 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -500,8 +500,7 @@ public CompletableFuture appendTask(OperationType operationType, If the taskQueue is not empty, the worker will execute the tasks in the queue. */ if (!taskQueue.isEmpty()) { - topic.getBrokerService().getPulsar().getTransactionExecutorProvider() - .getExecutor(this).submit(this::executeTask); + executeTask(); return CompletableFuture.completedFuture(null); } else if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { return task.get().whenComplete((ignore, throwable) -> { @@ -563,9 +562,10 @@ public CompletableFuture appendTask(OperationType operationType, private void executeTask() { if (taskQueue.isEmpty()) return; if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { + //Double-check. Avoid NoSuchElementException due to the first task is completed by other thread. + if (taskQueue.isEmpty()) return; Pair, Supplier>>> firstTask = taskQueue.getFirst(); - if (firstTask == null) return; firstTask.getValue().getRight().get().whenComplete((ignore, throwable) -> { if (throwable != null) { if (log.isDebugEnabled()) { @@ -579,7 +579,7 @@ private void executeTask() { } STATE_UPDATER.compareAndSet(this, OperationState.Operating, OperationState.None); - //Execute the next task in the task queue. + //Execute the next task in the other thread. topic.getBrokerService().getPulsar().getTransactionExecutorProvider() .getExecutor(this).submit(this::executeTask); }); From ee8a67128ae8a04bb2b12c9f7ee9486d3de24cb7 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Fri, 3 Feb 2023 23:50:46 +0800 Subject: [PATCH 28/41] Optimize the logic of clearing and add test. --- ...napshotSegmentAbortedTxnProcessorImpl.java | 39 +++++++- .../SegmentAbortedTxnProcessorTest.java | 95 +++++++++++++++++++ 2 files changed, 131 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 2f8d9a5daba48..5231547fb7d1f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -679,9 +679,7 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd } private CompletableFuture clearSnapshotSegmentAndIndexes() { - //Delete all segment - return persistentWorker.deleteSnapshotSegment(segmentIndex.keySet().stream().toList()) - //Delete index + return persistentWorker.clearAllSnapshotSegments() .thenCompose((ignore) -> snapshotIndexWriterFuture .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) .thenRun(() -> @@ -694,6 +692,41 @@ private CompletableFuture clearSnapshotSegmentAndIndexes() { }); } + /** + * Because the operation of writing segment and index is not atomic, + * we cannot use segment index to clear the snapshot segments. + * If we use segment index to clear snapshot segments, there will case dirty data in the below case: + *

+ * 1. Write snapshot segment 1, 2, 3, update index (1, 2, 3) + * 2. Write snapshot 4, failing to update index + * 3. Trim expired snapshot segment 1, 2, 3, update index (empty) + * 4. Write snapshot segment 1, 2, update index (1, 2) + * 5. Delete topic, clear all snapshot segment (segment1. segment2). + * Segment 3 and segment 4 can not be cleared until this namespace being deleted. + *

+ */ + private CompletableFuture clearAllSnapshotSegments() { + return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotSegmentService() + .createReader(TopicName.get(topic.getName())).thenComposeAsync(reader -> { + try { + while (reader.hasMoreEvents()) { + Message message = reader.readNextAsync() + .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); + if (topic.getName().equals(message.getValue().getTopicName())) { + snapshotSegmentsWriterFuture.get().write(message.getKey(), null); + } + } + return CompletableFuture.completedFuture(null); + } catch (Exception ex) { + log.error("[{}] Transaction buffer clear snapshot segments fail!", topic.getName(), ex); + return FutureUtil.failedFuture(ex); + } finally { + closeReader(reader); + } + }); + } + CompletableFuture closeAsync() { return CompletableFuture.allOf( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java index df58dce0c9e12..114f403e243f3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java @@ -1,9 +1,11 @@ package org.apache.pulsar.broker.transaction; +import static org.testng.Assert.assertTrue; import java.lang.reflect.Field; import java.util.LinkedList; import java.util.NavigableMap; import java.util.Queue; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; @@ -12,9 +14,16 @@ import org.apache.commons.collections4.map.LinkedMap; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; +import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; import org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; +import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; +import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.common.events.EventType; +import org.apache.pulsar.common.naming.TopicName; import org.testcontainers.shaded.org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -145,4 +154,90 @@ public void testFuturesCanCompleteWithException() throws Exception { queue.add(new Object()); processor.takeAbortedTxnsSnapshot(new PositionImpl(1, 10)).get(2, TimeUnit.SECONDS); } + + @Test + public void testClearSnapshotSegments() throws Exception { + PersistentTopic persistentTopic = (PersistentTopic) pulsarService.getBrokerService() + .getTopic(PROCESSOR_TOPIC, false).get().get(); + AbortedTxnProcessor processor = new SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic); + //1. Write two snapshot segment. + for (int j = 0; j < SEGMENT_SIZE * 2; j++) { + TxnID txnID = new TxnID(0, j); + PositionImpl position = new PositionImpl(0, j); + processor.putAbortedTxnAndPosition(txnID, position); + } + //2. Close index writer, making the index can not be updated. + Field field = SnapshotSegmentAbortedTxnProcessorImpl.class.getDeclaredField("persistentWorker"); + field.setAccessible(true); + SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker worker = + (SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker) field.get(processor); + Field indexWriteFutureField = SnapshotSegmentAbortedTxnProcessorImpl + .PersistentWorker.class.getDeclaredField("snapshotIndexWriterFuture"); + indexWriteFutureField.setAccessible(true); + CompletableFuture> snapshotIndexWriterFuture = + (CompletableFuture>) + indexWriteFutureField.get(worker); + snapshotIndexWriterFuture.get().close(); + //3. Try to write a snapshot segment. + for (int j = 0; j < SEGMENT_SIZE; j++) { + TxnID txnID = new TxnID(0, j); + PositionImpl position = new PositionImpl(0, j); + processor.putAbortedTxnAndPosition(txnID, position); + } + //4. Wait writing segment completed. + Awaitility.await().untilAsserted(() -> verifySnapshotSegmentsSize(PROCESSOR_TOPIC, 3)); + //5. Clear all the snapshot segments and indexes. + processor.clearAbortedTxnSnapshot().get(); + //6. Do compaction and wait it completed. + TopicName segmentTopicName = NamespaceEventsSystemTopicFactory.getSystemTopicName( + TopicName.get(PROCESSOR_TOPIC).getNamespaceObject(), + EventType.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS); + TopicName indexTopicName = NamespaceEventsSystemTopicFactory.getSystemTopicName( + TopicName.get(PROCESSOR_TOPIC).getNamespaceObject(), + EventType.TRANSACTION_BUFFER_SNAPSHOT_INDEXES); + doCompaction(segmentTopicName); + doCompaction(indexTopicName); + + //6. Verify the snapshot segments and index after clearing. + verifySnapshotSegmentsSize(PROCESSOR_TOPIC, 0); + verifySnapshotSegmentsIndexSize(PROCESSOR_TOPIC, 0); + } + + private void verifySnapshotSegmentsSize(String topic, int size) throws Exception { + SystemTopicClient.Reader reader = + pulsarService.getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotSegmentService() + .createReader(TopicName.get(topic)).get(); + while (reader.hasMoreEvents()) { + Message message = reader.readNextAsync() + .get(5, TimeUnit.SECONDS); + if (topic.equals(message.getValue().getTopicName())) { + Assert.assertFalse(size-- < 0); + } + } + } + + private void verifySnapshotSegmentsIndexSize(String topic, int size) throws Exception { + SystemTopicClient.Reader reader = + pulsarService.getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotIndexService() + .createReader(TopicName.get(topic)).get(); + while (reader.hasMoreEvents()) { + Message message = reader.readNextAsync() + .get(5, TimeUnit.SECONDS); + if (topic.equals(message.getValue().getTopicName())) { + Assert.assertFalse(size-- < 0); + } + } + } + + private void doCompaction(TopicName topic) throws Exception { + PersistentTopic snapshotTopic = (PersistentTopic) pulsarService.getBrokerService() + .getTopic(topic.toString(), false).get().get(); + Field field = PersistentTopic.class.getDeclaredField("currentCompaction"); + field.setAccessible(true); + snapshotTopic.triggerCompaction(); + CompletableFuture compactionFuture = (CompletableFuture) field.get(snapshotTopic); + org.awaitility.Awaitility.await().untilAsserted(() -> assertTrue(compactionFuture.isDone())); + } } From 016d3f949cef0c68ff212dcd61614f4cb8033c7f Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Sat, 4 Feb 2023 01:11:53 +0800 Subject: [PATCH 29/41] license and checkstyle --- .../buffer/AbortedTxnProcessor.java | 1 - ...SingleSnapshotAbortedTxnProcessorImpl.java | 5 +-- ...napshotSegmentAbortedTxnProcessorImpl.java | 31 ++++++++++++------- .../SegmentAbortedTxnProcessorTest.java | 26 +++++++++++++--- 4 files changed, 42 insertions(+), 21 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java index 06b35e427e179..8223aa12b75ae 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/AbortedTxnProcessor.java @@ -19,7 +19,6 @@ package org.apache.pulsar.broker.transaction.buffer; import java.util.concurrent.CompletableFuture; -import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.client.api.transaction.TxnID; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index d0ca848fcbcec..71d22a185aace 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -24,7 +24,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.map.LinkedMap; @@ -137,9 +136,7 @@ public CompletableFuture clearAbortedTxnSnapshot() { TransactionBufferSnapshot snapshot = new TransactionBufferSnapshot(); snapshot.setTopicName(topic.getName()); return writer.deleteAsync(snapshot.getTopicName(), snapshot); - }).thenRun(() -> { - log.info("[{}] Successes to delete the aborted transaction snapshot", this.topic); - }); + }).thenRun(() -> log.info("[{}] Successes to delete the aborted transaction snapshot", this.topic)); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 5231547fb7d1f..11803442b01d0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -9,7 +9,7 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing,2 + * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the @@ -159,8 +159,8 @@ public void putAbortedTxnAndPosition(TxnID txnID, PositionImpl position) { if (unsealedTxnIds.size() >= snapshotSegmentCapacity) { LinkedList abortedSegment = unsealedTxnIds; segmentIndex.put(position, txnID); - persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, () -> - persistentWorker.takeSnapshotSegmentAsync(abortedSegment, position)); + persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, + () -> persistentWorker.takeSnapshotSegmentAsync(abortedSegment, position)); this.unsealedTxnIds = new LinkedList<>(); } } @@ -212,8 +212,8 @@ public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosit TransactionBufferSnapshotIndexesMetadata metadata = new TransactionBufferSnapshotIndexesMetadata( maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), convertTypeToTxnIDData(unsealedTxnIds)); - return persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () -> persistentWorker - .updateSnapshotIndex(metadata, indexes.values().stream().toList())); + return persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, + () -> persistentWorker.updateSnapshotIndex(metadata, indexes.values().stream().toList())); } @Override @@ -308,7 +308,7 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { the segment can not be read according to the index. We update index again if there are invalid indexes. */ - if (((ManagedLedgerImpl)topic.getManagedLedger()) + if (((ManagedLedgerImpl) topic.getManagedLedger()) .ledgerExists(index.getAbortedMarkLedgerID())) { log.error("[{}] Failed to read snapshot segment [{}:{}]", topic.getName(), index.segmentLedgerID, @@ -560,10 +560,14 @@ public CompletableFuture appendTask(OperationType operationType, } private void executeTask() { - if (taskQueue.isEmpty()) return; + if (taskQueue.isEmpty()) { + return; + } if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { //Double-check. Avoid NoSuchElementException due to the first task is completed by other thread. - if (taskQueue.isEmpty()) return; + if (taskQueue.isEmpty()) { + return; + } Pair, Supplier>>> firstTask = taskQueue.getFirst(); firstTask.getValue().getRight().get().whenComplete((ignore, throwable) -> { @@ -592,7 +596,8 @@ private CompletableFuture takeSnapshotSegmentAsync(LinkedList seale if (log.isDebugEnabled()) { log.debug("Successes to take snapshot segment [{}] at maxReadPosition [{}] " + "for the topic [{}], and the size of the segment is [{}]", - this.sequenceID, abortedMarkerPersistentPosition, topic.getName(), sealedAbortedTxnIdSegment.size()); + this.sequenceID, abortedMarkerPersistentPosition, topic.getName(), + sealedAbortedTxnIdSegment.size()); } this.sequenceID.getAndIncrement(); }).exceptionally(e -> { @@ -600,7 +605,8 @@ private CompletableFuture takeSnapshotSegmentAsync(LinkedList seale //append aborted txn next time. log.error("Failed to take snapshot segment [{}] at maxReadPosition [{}] " + "for the topic [{}], and the size of the segment is [{}]", - this.sequenceID, abortedMarkerPersistentPosition, topic.getName(), sealedAbortedTxnIdSegment.size(), e); + this.sequenceID, abortedMarkerPersistentPosition, topic.getName(), + sealedAbortedTxnIdSegment.size(), e); return null; }); } @@ -611,7 +617,8 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm transactionBufferSnapshotSegment.setAborts(convertTypeToTxnIDData(segment)); transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); transactionBufferSnapshotSegment.setPersistentPositionEntryId(abortedMarkerPersistentPosition.getEntryId()); - transactionBufferSnapshotSegment.setPersistentPositionLedgerId(abortedMarkerPersistentPosition.getLedgerId()); + transactionBufferSnapshotSegment.setPersistentPositionLedgerId( + abortedMarkerPersistentPosition.getLedgerId()); return snapshotSegmentsWriterFuture.thenCompose(segmentWriter -> { transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java index 114f403e243f3..b563c9882e1ec 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.pulsar.broker.transaction; import static org.testng.Assert.assertTrue; @@ -71,17 +89,17 @@ public void testPutAbortedTxnIntoProcessor() throws Exception { //1.1 Put 10 aborted txn IDs to persistent two sealed segments. for (int i = 0; i < 10; i++) { TxnID txnID = new TxnID(0, i); - PositionImpl position = new PositionImpl(0,i); + PositionImpl position = new PositionImpl(0, i); processor.putAbortedTxnAndPosition(txnID, position); } //1.2 Put 4 aborted txn IDs into the unsealed segment. for (int i = 10; i < 14; i++) { TxnID txnID = new TxnID(0, i); - PositionImpl position = new PositionImpl(0,i); + PositionImpl position = new PositionImpl(0, i); processor.putAbortedTxnAndPosition(txnID, position); } //1.3 Verify the common data flow - verifyAbortedTxnIDAndSegmentIndex(processor,0,14); + verifyAbortedTxnIDAndSegmentIndex(processor, 0, 14); //2. Take the latest snapshot and verify recover from snapshot AbortedTxnProcessor newProcessor = new SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic); PositionImpl maxReadPosition = new PositionImpl(0, 14); @@ -91,7 +109,7 @@ public void testPutAbortedTxnIntoProcessor() throws Exception { processor.takeAbortedTxnsSnapshot(maxReadPosition).get(); newProcessor.recoverFromSnapshot().get(); //Verify the recovery data flow - verifyAbortedTxnIDAndSegmentIndex(newProcessor,0,14); + verifyAbortedTxnIDAndSegmentIndex(newProcessor, 0, 14); //3. Delete the ledgers and then verify the date. Field ledgersField = ManagedLedgerImpl.class.getDeclaredField("ledgers"); ledgersField.setAccessible(true); From b498a2d238b5e46c774d082b1e729a3f28047467 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Sat, 4 Feb 2023 02:46:09 +0800 Subject: [PATCH 30/41] fix test --- .../broker/transaction/SegmentAbortedTxnProcessorTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java index b563c9882e1ec..7d67ce87920fc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java @@ -25,11 +25,13 @@ import java.util.Queue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.commons.collections4.map.LinkedMap; +import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; @@ -168,8 +170,10 @@ public void testFuturesCanCompleteWithException() throws Exception { Field taskQueueField = SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker.class .getDeclaredField("taskQueue"); taskQueueField.setAccessible(true); + Supplier task = CompletableFuture::new; Queue queue = (Queue) taskQueueField.get(persistentWorker); - queue.add(new Object()); + queue.add(new MutablePair<>(SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker.OperationType.WriteSegment, + new MutablePair<>(new CompletableFuture<>(), task))); processor.takeAbortedTxnsSnapshot(new PositionImpl(1, 10)).get(2, TimeUnit.SECONDS); } From 1edc5b2b61764cd9656845c14ba7a0c0466f24a9 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Wed, 8 Feb 2023 11:48:52 +0800 Subject: [PATCH 31/41] fix some comments --- .../impl/SnapshotSegmentAbortedTxnProcessorImpl.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 11803442b01d0..80720754aa657 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -270,7 +270,7 @@ public CompletableFuture recoverFromSnapshot() { } //Read snapshot segment to recover aborts. ArrayList> completableFutures = new ArrayList<>(); - CompletableFuture openManagedLedgerFuture = new CompletableFuture<>(); + CompletableFuture openManagedLedgerAndHandleSegmentsFuture = new CompletableFuture<>(); AtomicBoolean hasInvalidIndex = new AtomicBoolean(false); AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks .OpenReadOnlyManagedLedgerCallback() { @@ -320,21 +320,21 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { } }, null); }); - openManagedLedgerFuture.complete(null); + openManagedLedgerAndHandleSegmentsFuture.complete(null); } @Override public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}] Failed to open readOnly managed ledger", topic, exception); - openManagedLedgerFuture.completeExceptionally(exception); + openManagedLedgerAndHandleSegmentsFuture.completeExceptionally(exception); } }; - TopicName snapshotIndexTopicName = TopicName.get(TopicDomain.persistent.toString(), + TopicName snapshotSegmentTopicName = TopicName.get(TopicDomain.persistent.toString(), TopicName.get(topic.getName()).getNamespaceObject(), SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS); this.topic.getBrokerService().getPulsar().getManagedLedgerFactory() - .asyncOpenReadOnlyManagedLedger(snapshotIndexTopicName + .asyncOpenReadOnlyManagedLedger(snapshotSegmentTopicName .getPersistenceNamingEncoding(), callback, topic.getManagedLedger().getConfig(), null); @@ -342,7 +342,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob Wait the processor recover completely and then allow TB to recover the messages after the startReadCursorPosition. */ - return openManagedLedgerFuture + return openManagedLedgerAndHandleSegmentsFuture .thenCompose((ignore) -> FutureUtil.waitForAll(completableFutures)) .thenCompose((i) -> { /* From 5eec1116017040cedd65f17de2a2d8687a667016 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Wed, 8 Feb 2023 19:30:51 +0800 Subject: [PATCH 32/41] fix some comments --- ...napshotSegmentAbortedTxnProcessorImpl.java | 94 ++++++++++--------- 1 file changed, 51 insertions(+), 43 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 80720754aa657..5c33fe0c9613c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -213,7 +213,7 @@ public CompletableFuture takeAbortedTxnsSnapshot(PositionImpl maxReadPosit maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), convertTypeToTxnIDData(unsealedTxnIds)); return persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, - () -> persistentWorker.updateSnapshotIndex(metadata, indexes.values().stream().toList())); + () -> persistentWorker.updateSnapshotIndex(metadata)); } @Override @@ -349,10 +349,9 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob Update the snapshot segment index if there exist invalid indexes. */ if (hasInvalidIndex.get()) { - persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, () - -> persistentWorker - .updateSnapshotIndex(finalPersistentSnapshotIndexes.getSnapshot(), - indexes.values().stream().toList())); + persistentWorker.appendTask(PersistentWorker.OperationType.UpdateIndex, + () -> persistentWorker.updateSnapshotIndex( + finalPersistentSnapshotIndexes.getSnapshot())); } /* If there is no segment index, the persistent worker will write segment begin from 0. @@ -366,8 +365,7 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob So that we can trim the expired snapshot segment in aborts according to the latest transaction IDs in the segmentIndex. */ - convertTypeToTxnID(finalPersistentSnapshotIndexes.getSnapshot().getAborts()) - .forEach(txnID -> aborts.put(txnID, txnID)); + unsealedTxnIds.forEach(txnID -> aborts.put(txnID, txnID)); return CompletableFuture.completedFuture(finalStartReadCursorPosition); }).exceptionally(ex -> { log.error("[{}] Failed to recover snapshot segment", this.topic.getName(), ex); @@ -592,15 +590,17 @@ private void executeTask() { private CompletableFuture takeSnapshotSegmentAsync(LinkedList sealedAbortedTxnIdSegment, PositionImpl abortedMarkerPersistentPosition) { - return writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, abortedMarkerPersistentPosition).thenRun(() -> { - if (log.isDebugEnabled()) { - log.debug("Successes to take snapshot segment [{}] at maxReadPosition [{}] " - + "for the topic [{}], and the size of the segment is [{}]", - this.sequenceID, abortedMarkerPersistentPosition, topic.getName(), - sealedAbortedTxnIdSegment.size()); - } - this.sequenceID.getAndIncrement(); - }).exceptionally(e -> { + CompletableFuture res = writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, + abortedMarkerPersistentPosition).thenRun(() -> { + if (log.isDebugEnabled()) { + log.debug("Successes to take snapshot segment [{}] at maxReadPosition [{}] " + + "for the topic [{}], and the size of the segment is [{}]", + this.sequenceID, abortedMarkerPersistentPosition, topic.getName(), + sealedAbortedTxnIdSegment.size()); + } + this.sequenceID.getAndIncrement(); + }); + res.exceptionally(e -> { //Just log the error, and the processor will try to take snapshot again when the transactionBuffer //append aborted txn next time. log.error("Failed to take snapshot segment [{}] at maxReadPosition [{}] " @@ -609,6 +609,7 @@ private CompletableFuture takeSnapshotSegmentAsync(LinkedList seale sealedAbortedTxnIdSegment.size(), e); return null; }); + return res; } private CompletableFuture writeSnapshotSegmentAsync(LinkedList segment, @@ -638,8 +639,8 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm //If the index can not be written successfully, the snapshot segment wil be overwritten // when the processor writes snapshot segment next time. return updateSnapshotIndex(new TransactionBufferSnapshotIndexesMetadata( - maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), new LinkedList<>()), - indexes.values().stream().toList()); + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), + convertTypeToTxnIDData(unsealedTxnIds))); }); } @@ -648,9 +649,9 @@ private CompletableFuture deleteSnapshotSegment(List positio List> results = new ArrayList<>(); for (PositionImpl positionNeedToDelete : positionNeedToDeletes) { long sequenceIdNeedToDelete = indexes.get(positionNeedToDelete).getSequenceID(); - results.add(snapshotSegmentsWriterFuture + CompletableFuture res = snapshotSegmentsWriterFuture .thenCompose(writer -> writer.deleteAsync(buildKey(sequenceIdNeedToDelete), null)) - .thenRun(() -> { + .thenCompose(messageId -> { if (log.isDebugEnabled()) { log.debug("[{}] Successes to delete the snapshot segment, " + "whose sequenceId is [{}] and maxReadPosition is [{}]", @@ -659,44 +660,51 @@ private CompletableFuture deleteSnapshotSegment(List positio //The process will check whether the snapshot segment is null, // and update index when recovered. indexes.remove(positionNeedToDelete); - }).exceptionally(e -> { - log.warn("[{}] Failed to delete the snapshot segment, " - + "whose sequenceId is [{}] and maxReadPosition is [{}]", - this.topic.getName(), this.sequenceID, positionNeedToDelete, e); - return null; - })); + PositionImpl maxReadPosition = topic.getMaxReadPosition(); + return updateSnapshotIndex(new TransactionBufferSnapshotIndexesMetadata( + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), + convertTypeToTxnIDData(unsealedTxnIds))); + }); + res.exceptionally(e -> { + log.warn("[{}] Failed to delete the snapshot segment, " + + "whose sequenceId is [{}] and maxReadPosition is [{}]", + this.topic.getName(), this.sequenceID, positionNeedToDelete, e); + return null; + }); + results.add(res); } return FutureUtil.waitForAll(results); } - private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotIndexesMetadata snapshotSegment, - List indexList) { + private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotIndexesMetadata snapshotSegment) { TransactionBufferSnapshotIndexes snapshotIndexes = new TransactionBufferSnapshotIndexes(); - return snapshotIndexWriterFuture + CompletableFuture res = snapshotIndexWriterFuture .thenCompose((indexesWriter) -> { - snapshotIndexes.setIndexList(indexList); + snapshotIndexes.setIndexList(indexes.values().stream().toList()); snapshotIndexes.setSnapshot(snapshotSegment); - return indexesWriter.writeAsync(topic.getName(), snapshotIndexes); - }) - .thenRun(() -> lastSnapshotTimestamps = System.currentTimeMillis()) - .exceptionally(e -> { - log.error("[{}] Failed to update snapshot segment index", snapshotIndexes.getTopicName(), e); - return null; + return indexesWriter.writeAsync(topic.getName(), snapshotIndexes) + .thenCompose(messageId -> CompletableFuture.completedFuture(null)); }); + res.thenRun(() -> lastSnapshotTimestamps = System.currentTimeMillis()).exceptionally(e -> { + log.error("[{}] Failed to update snapshot segment index", snapshotIndexes.getTopicName(), e); + return null; + }); + return res; } private CompletableFuture clearSnapshotSegmentAndIndexes() { - return persistentWorker.clearAllSnapshotSegments() + CompletableFuture res = persistentWorker.clearAllSnapshotSegments() .thenCompose((ignore) -> snapshotIndexWriterFuture .thenCompose(indexesWriter -> indexesWriter.writeAsync(topic.getName(), null))) .thenRun(() -> log.debug("Successes to clear the snapshot segment and indexes for the topic [{}]", - topic.getName())) - .exceptionally(e -> { - log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", - topic.getName(), e); - return null; - }); + topic.getName())); + res.exceptionally(e -> { + log.error("Failed to clear the snapshot segment and indexes for the topic [{}]", + topic.getName(), e); + return null; + }); + return res; } /** From d095166813edd6200d718292d19a0d1001257aa5 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Wed, 8 Feb 2023 19:47:44 +0800 Subject: [PATCH 33/41] fix some comments --- .../impl/SnapshotSegmentAbortedTxnProcessorImpl.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 5c33fe0c9613c..bba3e1adbf72d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -499,7 +499,7 @@ public CompletableFuture appendTask(OperationType operationType, */ if (!taskQueue.isEmpty()) { executeTask(); - return CompletableFuture.completedFuture(null); + return cancelUpdateIndexTask(); } else if (STATE_UPDATER.compareAndSet(this, OperationState.None, OperationState.Operating)) { return task.get().whenComplete((ignore, throwable) -> { if (throwable != null && log.isDebugEnabled()) { @@ -508,7 +508,7 @@ public CompletableFuture appendTask(OperationType operationType, STATE_UPDATER.compareAndSet(this, OperationState.Operating, OperationState.None); }); } else { - return CompletableFuture.completedFuture(null); + return cancelUpdateIndexTask(); } } /* @@ -557,6 +557,14 @@ public CompletableFuture appendTask(OperationType operationType, } } + private CompletableFuture cancelUpdateIndexTask() { + if (log.isDebugEnabled()) { + log.debug("The operation of updating index is canceled due there is other operation executing"); + } + return FutureUtil.failedFuture(new BrokerServiceException + .ServiceUnitNotReadyException("The operation of updating index is canceled")); + } + private void executeTask() { if (taskQueue.isEmpty()) { return; From 34c9f3546438222459844c1eb92c0645dd9ff0f1 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Thu, 9 Feb 2023 19:29:07 +0800 Subject: [PATCH 34/41] fix some comments --- .../impl/SnapshotSegmentAbortedTxnProcessorImpl.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index bba3e1adbf72d..afabd2e03812a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -160,7 +160,8 @@ public void putAbortedTxnAndPosition(TxnID txnID, PositionImpl position) { LinkedList abortedSegment = unsealedTxnIds; segmentIndex.put(position, txnID); persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, - () -> persistentWorker.takeSnapshotSegmentAsync(abortedSegment, position)); + () -> persistentWorker.takeSnapshotSegmentAsync(abortedSegment, position, + topic.getMaxReadPosition())); this.unsealedTxnIds = new LinkedList<>(); } } @@ -597,9 +598,10 @@ private void executeTask() { } private CompletableFuture takeSnapshotSegmentAsync(LinkedList sealedAbortedTxnIdSegment, - PositionImpl abortedMarkerPersistentPosition) { + PositionImpl abortedMarkerPersistentPosition, + PositionImpl maxReadPosition) { CompletableFuture res = writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, - abortedMarkerPersistentPosition).thenRun(() -> { + abortedMarkerPersistentPosition, maxReadPosition).thenRun(() -> { if (log.isDebugEnabled()) { log.debug("Successes to take snapshot segment [{}] at maxReadPosition [{}] " + "for the topic [{}], and the size of the segment is [{}]", @@ -621,7 +623,8 @@ private CompletableFuture takeSnapshotSegmentAsync(LinkedList seale } private CompletableFuture writeSnapshotSegmentAsync(LinkedList segment, - PositionImpl abortedMarkerPersistentPosition) { + PositionImpl abortedMarkerPersistentPosition, + PositionImpl maxReadPosition) { TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = new TransactionBufferSnapshotSegment(); transactionBufferSnapshotSegment.setAborts(convertTypeToTxnIDData(segment)); transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); @@ -633,7 +636,6 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm transactionBufferSnapshotSegment.setSequenceId(this.sequenceID.get()); return segmentWriter.writeAsync(buildKey(this.sequenceID.get()), transactionBufferSnapshotSegment); }).thenCompose((messageId) -> { - PositionImpl maxReadPosition = topic.getMaxReadPosition(); //Build index for this segment TransactionBufferSnapshotIndex index = new TransactionBufferSnapshotIndex(); index.setSequenceID(transactionBufferSnapshotSegment.getSequenceId()); From 00eb60efc6fff555f366c31abc400a0b64464eaa Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Thu, 9 Feb 2023 19:50:07 +0800 Subject: [PATCH 35/41] Only update index when execute the latest task --- ...napshotSegmentAbortedTxnProcessorImpl.java | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index afabd2e03812a..6ef13748a02be 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -24,6 +24,7 @@ import java.util.LinkedList; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -160,8 +161,7 @@ public void putAbortedTxnAndPosition(TxnID txnID, PositionImpl position) { LinkedList abortedSegment = unsealedTxnIds; segmentIndex.put(position, txnID); persistentWorker.appendTask(PersistentWorker.OperationType.WriteSegment, - () -> persistentWorker.takeSnapshotSegmentAsync(abortedSegment, position, - topic.getMaxReadPosition())); + () -> persistentWorker.takeSnapshotSegmentAsync(abortedSegment, position)); this.unsealedTxnIds = new LinkedList<>(); } } @@ -598,10 +598,9 @@ private void executeTask() { } private CompletableFuture takeSnapshotSegmentAsync(LinkedList sealedAbortedTxnIdSegment, - PositionImpl abortedMarkerPersistentPosition, - PositionImpl maxReadPosition) { + PositionImpl abortedMarkerPersistentPosition) { CompletableFuture res = writeSnapshotSegmentAsync(sealedAbortedTxnIdSegment, - abortedMarkerPersistentPosition, maxReadPosition).thenRun(() -> { + abortedMarkerPersistentPosition).thenRun(() -> { if (log.isDebugEnabled()) { log.debug("Successes to take snapshot segment [{}] at maxReadPosition [{}] " + "for the topic [{}], and the size of the segment is [{}]", @@ -623,8 +622,7 @@ private CompletableFuture takeSnapshotSegmentAsync(LinkedList seale } private CompletableFuture writeSnapshotSegmentAsync(LinkedList segment, - PositionImpl abortedMarkerPersistentPosition, - PositionImpl maxReadPosition) { + PositionImpl abortedMarkerPersistentPosition) { TransactionBufferSnapshotSegment transactionBufferSnapshotSegment = new TransactionBufferSnapshotSegment(); transactionBufferSnapshotSegment.setAborts(convertTypeToTxnIDData(segment)); transactionBufferSnapshotSegment.setTopicName(this.topic.getName()); @@ -647,13 +645,23 @@ private CompletableFuture writeSnapshotSegmentAsync(LinkedList segm indexes.put(abortedMarkerPersistentPosition, index); //update snapshot segment index. //If the index can not be written successfully, the snapshot segment wil be overwritten - // when the processor writes snapshot segment next time. - return updateSnapshotIndex(new TransactionBufferSnapshotIndexesMetadata( - maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), - convertTypeToTxnIDData(unsealedTxnIds))); + //when the processor writes snapshot segment next time. + //And if the task is not the newest in the queue, it is no need to update the index. + return updateIndexWhenExecuteTheLatestTask(); }); } + private CompletionStage updateIndexWhenExecuteTheLatestTask() { + PositionImpl maxReadPosition = topic.getMaxReadPosition(); + List aborts = convertTypeToTxnIDData(unsealedTxnIds); + if (taskQueue.size() != 1) { + return CompletableFuture.completedFuture(null); + } else { + return updateSnapshotIndex(new TransactionBufferSnapshotIndexesMetadata( + maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), aborts)); + } + } + // update index after delete all segment. private CompletableFuture deleteSnapshotSegment(List positionNeedToDeletes) { List> results = new ArrayList<>(); @@ -667,13 +675,11 @@ private CompletableFuture deleteSnapshotSegment(List positio + "whose sequenceId is [{}] and maxReadPosition is [{}]", this.topic.getName(), this.sequenceID, positionNeedToDelete); } - //The process will check whether the snapshot segment is null, - // and update index when recovered. + //The index may fail to update but the processor will check + //whether the snapshot segment is null, and update the index when recovering. + //And if the task is not the newest in the queue, it is no need to update the index. indexes.remove(positionNeedToDelete); - PositionImpl maxReadPosition = topic.getMaxReadPosition(); - return updateSnapshotIndex(new TransactionBufferSnapshotIndexesMetadata( - maxReadPosition.getLedgerId(), maxReadPosition.getEntryId(), - convertTypeToTxnIDData(unsealedTxnIds))); + return updateIndexWhenExecuteTheLatestTask(); }); res.exceptionally(e -> { log.warn("[{}] Failed to delete the snapshot segment, " From f4bf25ff6279dbcf21363cf8ae39625d186e608e Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Fri, 10 Feb 2023 11:35:51 +0800 Subject: [PATCH 36/41] Do not retry the task immediately if the task happens exception. --- ...napshotSegmentAbortedTxnProcessorImpl.java | 8 ++++--- .../SegmentAbortedTxnProcessorTest.java | 24 +++++++++++++------ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 6ef13748a02be..5b10f772c2f34 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -583,16 +583,17 @@ private void executeTask() { log.debug("[{}] Failed to do operation do operation of [{}]", topic.getName(), firstTask.getKey().name(), throwable); } + //Do not execute the tasks in the task queue until the next task is appended to the task queue. firstTask.getRight().getKey().completeExceptionally(throwable); } else { firstTask.getRight().getKey().complete(null); taskQueue.removeFirst(); + //Execute the next task in the other thread. + topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this).submit(this::executeTask); } STATE_UPDATER.compareAndSet(this, OperationState.Operating, OperationState.None); - //Execute the next task in the other thread. - topic.getBrokerService().getPulsar().getTransactionExecutorProvider() - .getExecutor(this).submit(this::executeTask); }); } } @@ -698,6 +699,7 @@ private CompletableFuture updateSnapshotIndex(TransactionBufferSnapshotInd .thenCompose((indexesWriter) -> { snapshotIndexes.setIndexList(indexes.values().stream().toList()); snapshotIndexes.setSnapshot(snapshotSegment); + snapshotIndexes.setTopicName(topic.getName()); return indexesWriter.writeAsync(topic.getName(), snapshotIndexes) .thenCompose(messageId -> CompletableFuture.completedFuture(null)); }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java index 7d67ce87920fc..ba9883fcfe4e2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java @@ -188,6 +188,7 @@ public void testClearSnapshotSegments() throws Exception { PositionImpl position = new PositionImpl(0, j); processor.putAbortedTxnAndPosition(txnID, position); } + Awaitility.await().untilAsserted(() -> verifySnapshotSegmentsSize(PROCESSOR_TOPIC, 2)); //2. Close index writer, making the index can not be updated. Field field = SnapshotSegmentAbortedTxnProcessorImpl.class.getDeclaredField("persistentWorker"); field.setAccessible(true); @@ -200,7 +201,7 @@ public void testClearSnapshotSegments() throws Exception { (CompletableFuture>) indexWriteFutureField.get(worker); snapshotIndexWriterFuture.get().close(); - //3. Try to write a snapshot segment. + //3. Try to write a snapshot segment that will fail to update indexes. for (int j = 0; j < SEGMENT_SIZE; j++) { TxnID txnID = new TxnID(0, j); PositionImpl position = new PositionImpl(0, j); @@ -209,7 +210,12 @@ public void testClearSnapshotSegments() throws Exception { //4. Wait writing segment completed. Awaitility.await().untilAsserted(() -> verifySnapshotSegmentsSize(PROCESSOR_TOPIC, 3)); //5. Clear all the snapshot segments and indexes. - processor.clearAbortedTxnSnapshot().get(); + try { + processor.clearAbortedTxnSnapshot().get(); + //Failed to clear index due to the index writer is closed. + Assert.fail(); + } catch (Exception ignored) { + } //6. Do compaction and wait it completed. TopicName segmentTopicName = NamespaceEventsSystemTopicFactory.getSystemTopicName( TopicName.get(PROCESSOR_TOPIC).getNamespaceObject(), @@ -219,10 +225,9 @@ public void testClearSnapshotSegments() throws Exception { EventType.TRANSACTION_BUFFER_SNAPSHOT_INDEXES); doCompaction(segmentTopicName); doCompaction(indexTopicName); - - //6. Verify the snapshot segments and index after clearing. + //7. Verify the snapshot segments and index after clearing. verifySnapshotSegmentsSize(PROCESSOR_TOPIC, 0); - verifySnapshotSegmentsIndexSize(PROCESSOR_TOPIC, 0); + verifySnapshotSegmentsIndexSize(PROCESSOR_TOPIC, 1); } private void verifySnapshotSegmentsSize(String topic, int size) throws Exception { @@ -230,13 +235,15 @@ private void verifySnapshotSegmentsSize(String topic, int size) throws Exception pulsarService.getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotSegmentService() .createReader(TopicName.get(topic)).get(); + int segmentCount = 0; while (reader.hasMoreEvents()) { Message message = reader.readNextAsync() .get(5, TimeUnit.SECONDS); if (topic.equals(message.getValue().getTopicName())) { - Assert.assertFalse(size-- < 0); + segmentCount++; } } + Assert.assertEquals(segmentCount, size); } private void verifySnapshotSegmentsIndexSize(String topic, int size) throws Exception { @@ -244,13 +251,16 @@ private void verifySnapshotSegmentsIndexSize(String topic, int size) throws Exce pulsarService.getTransactionBufferSnapshotServiceFactory() .getTxnBufferSnapshotIndexService() .createReader(TopicName.get(topic)).get(); + int indexCount = 0; while (reader.hasMoreEvents()) { Message message = reader.readNextAsync() .get(5, TimeUnit.SECONDS); if (topic.equals(message.getValue().getTopicName())) { - Assert.assertFalse(size-- < 0); + indexCount++; } + System.out.printf("message.getValue().getTopicName() :" + message.getValue().getTopicName()); } + Assert.assertEquals(indexCount, size); } private void doCompaction(TopicName topic) throws Exception { From 4ad3c7a67a26c7d1be11e9b1962d07d5f27c1e61 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Fri, 10 Feb 2023 11:44:01 +0800 Subject: [PATCH 37/41] fix test --- .../transaction/SegmentAbortedTxnProcessorTest.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java index ba9883fcfe4e2..ffc059de8e656 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java @@ -33,6 +33,7 @@ import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.systopic.SystemTopicClient; @@ -159,7 +160,7 @@ private void verifyAbortedTxnIDAndSegmentIndex(AbortedTxnProcessor processor, in // Verify the update index future can be completed when the queue has other tasks. @Test - public void testFuturesCanCompleteWithException() throws Exception { + public void testFuturesCanCompleteWhenItIsCanceled() throws Exception { PersistentTopic persistentTopic = (PersistentTopic) pulsarService.getBrokerService() .getTopic(PROCESSOR_TOPIC, false).get().get(); AbortedTxnProcessor processor = new SnapshotSegmentAbortedTxnProcessorImpl(persistentTopic); @@ -174,7 +175,11 @@ public void testFuturesCanCompleteWithException() throws Exception { Queue queue = (Queue) taskQueueField.get(persistentWorker); queue.add(new MutablePair<>(SnapshotSegmentAbortedTxnProcessorImpl.PersistentWorker.OperationType.WriteSegment, new MutablePair<>(new CompletableFuture<>(), task))); - processor.takeAbortedTxnsSnapshot(new PositionImpl(1, 10)).get(2, TimeUnit.SECONDS); + try { + processor.takeAbortedTxnsSnapshot(new PositionImpl(1, 10)).get(2, TimeUnit.SECONDS); + } catch (Exception e) { + Assert.assertTrue(e.getCause() instanceof BrokerServiceException.ServiceUnitNotReadyException); + } } @Test From 20b9681175d4827c68847d4686a4f22fad98ef47 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Fri, 10 Feb 2023 12:51:00 +0800 Subject: [PATCH 38/41] separate catch exception --- .../impl/SingleSnapshotAbortedTxnProcessorImpl.java | 4 ++-- .../impl/SnapshotSegmentAbortedTxnProcessorImpl.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 71d22a185aace..87161e97512b9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -55,8 +55,8 @@ public SingleSnapshotAbortedTxnProcessorImpl(PersistentTopic topic) { this.topic = topic; this.takeSnapshotWriter = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName())) - .exceptionally((ex) -> { + .getTxnBufferSnapshotService().createWriter(TopicName.get(topic.getName())); + this.takeSnapshotWriter.exceptionally((ex) -> { log.error("{} Failed to create snapshot writer", topic.getName()); topic.close(); return null; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 5b10f772c2f34..7a9e0e1abedd9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -472,16 +472,16 @@ public PersistentWorker(PersistentTopic topic) { this.topic = topic; this.snapshotSegmentsWriterFuture = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName())) - .exceptionally(ex -> { + .getTxnBufferSnapshotSegmentService().createWriter(TopicName.get(topic.getName())); + this.snapshotSegmentsWriterFuture.exceptionally(ex -> { log.error("{} Failed to create snapshot index writer", topic.getName()); topic.close(); return null; }); this.snapshotIndexWriterFuture = this.topic.getBrokerService().getPulsar() .getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName())) - .exceptionally((ex) -> { + .getTxnBufferSnapshotIndexService().createWriter(TopicName.get(topic.getName())); + this.snapshotIndexWriterFuture.exceptionally((ex) -> { log.error("{} Failed to create snapshot writer", topic.getName()); topic.close(); return null; From 1c06fcfeff6674e17f74d1c0944518ff897e3987 Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Fri, 10 Feb 2023 13:51:57 +0800 Subject: [PATCH 39/41] update maxReadPosition after putting aborted txn ID --- .../broker/transaction/buffer/impl/TopicTransactionBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 7429c784d8b67..89a8e95afba1f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -353,8 +353,8 @@ public CompletableFuture abortTxn(TxnID txnID, long lowWaterMark) { @Override public void addComplete(Position position, ByteBuf entryData, Object ctx) { synchronized (TopicTransactionBuffer.this) { - updateMaxReadPosition(txnID); snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, (PositionImpl) position); + updateMaxReadPosition(txnID); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); takeSnapshotByChangeTimes(); } From af2ccab0cb95b6985da1b63ffa9cfb96da8ed6ae Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Fri, 10 Feb 2023 14:29:49 +0800 Subject: [PATCH 40/41] add notes --- .../buffer/metadata/v2/TransactionBufferSnapshotIndex.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java index 3d761af8ebd06..42687d51ca386 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java @@ -29,8 +29,12 @@ @NoArgsConstructor public class TransactionBufferSnapshotIndex { public long sequenceID; + /** location(ledger id of position) of a transaction marker in the origin topic **/ public long abortedMarkLedgerID; + /** location(entry id of position) of a transaction marker in the origin topic **/ public long abortedMarkEntryID; + /** location(ledger id of position) of a segment data in the system topic __transaction_buffer_snapshot_segments **/ public long segmentLedgerID; + /** location(entry id of position) of a segment data in the system topic __transaction_buffer_snapshot_segments **/ public long segmentEntryID; } From c7c1294f7acb4d930dc04de80dbb3a2e29b0d1bb Mon Sep 17 00:00:00 2001 From: xiangying <1984997880@qq.com> Date: Fri, 10 Feb 2023 14:41:47 +0800 Subject: [PATCH 41/41] add notes --- .../v2/TransactionBufferSnapshotIndex.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java index 42687d51ca386..b86edc845c1e5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/metadata/v2/TransactionBufferSnapshotIndex.java @@ -29,12 +29,21 @@ @NoArgsConstructor public class TransactionBufferSnapshotIndex { public long sequenceID; - /** location(ledger id of position) of a transaction marker in the origin topic **/ + /** + * Location(ledger id of position) of a transaction marker in the origin topic. + */ public long abortedMarkLedgerID; - /** location(entry id of position) of a transaction marker in the origin topic **/ + + /** + * Location(entry id of position) of a transaction marker in the origin topic. + */ public long abortedMarkEntryID; - /** location(ledger id of position) of a segment data in the system topic __transaction_buffer_snapshot_segments **/ + /** + * Location(ledger id of position) of a segment data in the system topic __transaction_buffer_snapshot_segments. + */ public long segmentLedgerID; - /** location(entry id of position) of a segment data in the system topic __transaction_buffer_snapshot_segments **/ + /** + * Location(entry id of position) of a segment data in the system topic __transaction_buffer_snapshot_segments. + */ public long segmentEntryID; }