From faed541bc64ed112eb3c61abd9cdda10ecaeed1e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 16 Sep 2024 16:07:03 +0300 Subject: [PATCH 1/5] [improve][broker] Decouple ManagedLedger interfaces from the current implementation Co-authored-by: Yan Zhao Co-authored-by: Yong Zhang --- .../bookkeeper/mledger/AsyncCallbacks.java | 3 +- .../bookkeeper/mledger/ManagedCursor.java | 15 + .../bookkeeper/mledger/ManagedLedger.java | 30 ++ .../mledger/ManagedLedgerFactory.java | 12 + .../bookkeeper/mledger/PositionBound.java | 24 + .../mledger/ReadOnlyManagedLedger.java | 33 ++ .../ReadOnlyManagedLedgerImplWrapper.java | 63 +++ .../mledger/impl/ManagedCursorImpl.java | 33 +- .../impl/ManagedLedgerFactoryImpl.java | 427 +++++++++++++++++- .../mledger/impl/ManagedLedgerImpl.java | 98 +++- .../impl/ManagedLedgerOfflineBacklog.java | 380 +--------------- .../bookkeeper/mledger/impl/OpFindNewest.java | 4 +- .../bookkeeper/mledger/impl/OpScan.java | 3 +- .../impl/OpenTelemetryManagedCursorStats.java | 3 +- .../impl/OpenTelemetryManagedLedgerStats.java | 5 +- .../mledger/impl/ReadOnlyCursorImpl.java | 2 +- .../impl/ReadOnlyManagedLedgerImpl.java | 8 +- .../impl/ManagedCursorContainerTest.java | 32 ++ .../impl/ManagedCursorPropertiesTest.java | 2 +- .../mledger/impl/ManagedLedgerTest.java | 17 +- .../impl/ReadOnlyManagedLedgerImplTest.java | 5 +- .../apache/pulsar/broker/PulsarService.java | 3 + .../admin/impl/PersistentTopicsBase.java | 12 +- .../bucket/BucketDelayedDeliveryTracker.java | 2 +- .../pulsar/broker/service/AbstractTopic.java | 3 + .../broker/service/BacklogQuotaManager.java | 29 +- .../pulsar/broker/service/BrokerService.java | 3 + .../pulsar/broker/service/ServerCnx.java | 8 +- .../PersistentMessageExpiryMonitor.java | 46 +- ...tStickyKeyDispatcherMultipleConsumers.java | 6 +- .../persistent/PersistentSubscription.java | 55 ++- .../service/persistent/PersistentTopic.java | 395 +++++++--------- .../broker/stats/metrics/AbstractMetrics.java | 13 +- .../stats/metrics/ManagedCursorMetrics.java | 13 +- .../stats/metrics/ManagedLedgerMetrics.java | 21 +- ...SingleSnapshotAbortedTxnProcessorImpl.java | 5 +- ...napshotSegmentAbortedTxnProcessorImpl.java | 19 +- .../buffer/impl/TopicTransactionBuffer.java | 5 +- .../pendingack/impl/MLPendingAckStore.java | 6 +- .../impl/MLPendingAckStoreProvider.java | 7 +- .../pendingack/impl/PendingAckHandleImpl.java | 13 +- .../pulsar/compaction/CompactedTopicImpl.java | 6 +- .../compaction/CompactedTopicUtils.java | 6 +- .../broker/delayed/MockManagedCursor.java | 31 ++ .../CurrentLedgerRolloverIfFullTest.java | 4 +- .../persistent/BucketDelayedDeliveryTest.java | 19 +- .../stats/ManagedLedgerMetricsTest.java | 6 +- .../TopicTransactionBufferRecoverTest.java | 4 +- .../broker/transaction/TransactionTest.java | 3 +- .../pendingack/PendingAckMetadataTest.java | 24 +- .../data/ManagedLedgerInternalStats.java | 5 + .../jcloud/impl/MockManagedLedger.java | 53 +++ 52 files changed, 1225 insertions(+), 799 deletions(-) create mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java create mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedger.java create mode 100644 managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedgerImplWrapper.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java index dcf2c225e8b35..70db427afce4f 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java @@ -24,7 +24,6 @@ import java.util.Optional; import org.apache.bookkeeper.common.annotation.InterfaceAudience; import org.apache.bookkeeper.common.annotation.InterfaceStability; -import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; /** * Definition of all the callbacks used for the ManagedLedger asynchronous API. @@ -48,7 +47,7 @@ interface OpenReadOnlyCursorCallback { } interface OpenReadOnlyManagedLedgerCallback { - void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl managedLedger, Object ctx); + void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger managedLedger, Object ctx); void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java index f6345e7b9ec5b..042e03998696c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java @@ -34,6 +34,7 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.SkipEntriesCallback; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; /** * A ManagedCursor is a persisted cursor inside a ManagedLedger. @@ -45,6 +46,8 @@ @InterfaceStability.Stable public interface ManagedCursor { + String CURSOR_INTERNAL_PROPERTY_PREFIX = "#pulsar.internal."; + @SuppressWarnings("checkstyle:javadoctype") enum FindPositionConstraint { SearchActiveEntries, SearchAllAvailableEntries @@ -885,4 +888,16 @@ default boolean periodicRollover() { default ManagedCursorAttributes getManagedCursorAttributes() { return new ManagedCursorAttributes(this); } + + ManagedLedgerInternalStats.CursorStats getCursorStats(); + + boolean isMessageDeleted(Position position); + + ManagedCursor duplicateNonDurableCursor(String nonDurableCursorName) throws ManagedLedgerException; + + long[] getBatchPositionAckSet(Position position); + + int applyMaxSizeCap(int maxEntries, long maxSizeBytes); + + void updateReadStats(int readEntriesCount, long readEntriesSize); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java index a9242d5cc65b4..de69d97bb79aa 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java @@ -18,8 +18,10 @@ */ package org.apache.bookkeeper.mledger; +import com.google.common.collect.Range; import io.netty.buffer.ByteBuf; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Predicate; @@ -374,6 +376,8 @@ void asyncOpenCursor(String name, InitialPosition initialPosition, Map range); + /** * Get the total number of active entries for this managed ledger. * @@ -703,4 +707,30 @@ default void skipNonRecoverableLedger(long ledgerId){} default ManagedLedgerAttributes getManagedLedgerAttributes() { return new ManagedLedgerAttributes(this); } + + void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx); + + /** + * Get all the managed ledgers. + */ + NavigableMap getLedgersInfo(); + + Position getNextValidPosition(Position position); + + Position getPreviousPosition(Position position); + + long getEstimatedBacklogSize(Position position); + + Position getPositionAfterN(Position startPosition, long n, PositionBound startRange); + + int getPendingAddEntriesCount(); + + long getCacheSize(); + + default CompletableFuture getLastDispatchablePosition(final Predicate predicate, + final Position startPosition) { + return CompletableFuture.completedFuture(PositionFactory.EARLIEST); + } + + Position getFirstPosition(); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java index e09fd84ea55f2..9854b5de34ecc 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java @@ -28,6 +28,8 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenReadOnlyCursorCallback; import org.apache.bookkeeper.mledger.impl.cache.EntryCacheManager; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats; /** * A factory to open/create managed ledgers and delete them. @@ -233,4 +235,14 @@ void asyncDelete(String name, CompletableFuture mlConfigFut * @return properties of this managedLedger. */ CompletableFuture> getManagedLedgerPropertiesAsync(String name); + + Map getManagedLedgers(); + + ManagedLedgerFactoryMXBean getCacheStats(); + + + void estimateUnloadedTopicBacklog(PersistentOfflineTopicStats offlineTopicStats, TopicName topicName, + boolean accurate, Object ctx) throws Exception; + + ManagedLedgerFactoryConfig getConfig(); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java new file mode 100644 index 0000000000000..f6f0cd3445cf0 --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger; + +public enum PositionBound { + // define boundaries for position based seeks and searches + startIncluded, startExcluded +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedger.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedger.java new file mode 100644 index 0000000000000..91b8f92eb637e --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedger.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger; + +import java.util.Map; + +public interface ReadOnlyManagedLedger { + + void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx); + + long getNumberOfEntries(); + + ReadOnlyCursor createReadOnlyCursor(Position position); + + Map getProperties(); + +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedgerImplWrapper.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedgerImplWrapper.java new file mode 100644 index 0000000000000..5bc94c04beefd --- /dev/null +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedgerImplWrapper.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.mledger; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.common.util.OrderedScheduler; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; +import org.apache.bookkeeper.mledger.impl.MetaStore; +import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; + +public class ReadOnlyManagedLedgerImplWrapper implements ReadOnlyManagedLedger { + + private final ReadOnlyManagedLedgerImpl readOnlyManagedLedger; + + public ReadOnlyManagedLedgerImplWrapper(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper, MetaStore store, + ManagedLedgerConfig config, OrderedScheduler scheduledExecutor, + String name) { + this.readOnlyManagedLedger = + new ReadOnlyManagedLedgerImpl(factory, bookKeeper, store, config, scheduledExecutor, name); + } + + public CompletableFuture initialize() { + return readOnlyManagedLedger.initialize(); + } + + @Override + public void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx) { + readOnlyManagedLedger.asyncReadEntry(position, callback, ctx); + } + + @Override + public long getNumberOfEntries() { + return readOnlyManagedLedger.getNumberOfEntries(); + } + + @Override + public ReadOnlyCursor createReadOnlyCursor(Position position) { + return readOnlyManagedLedger.createReadOnlyCursor(position); + } + + @Override + public Map getProperties() { + return readOnlyManagedLedger.getProperties(); + } +} diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index e808c31bc89f1..e27814eadd0b5 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -86,9 +86,9 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException; import org.apache.bookkeeper.mledger.ManagedLedgerException.NoMoreEntriesToReadException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.ScanOutcome; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.PositionBound; import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback; import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongProperty; @@ -98,6 +98,8 @@ import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.StringProperty; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; +import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.common.util.collections.LongPairRangeSet; @@ -124,8 +126,6 @@ public class ManagedCursorImpl implements ManagedCursor { protected final ManagedLedgerImpl ledger; private final String name; - public static final String CURSOR_INTERNAL_PROPERTY_PREFIX = "#pulsar.internal."; - private volatile Map cursorProperties; private final BookKeeper.DigestType digestType; @@ -1769,7 +1769,7 @@ public void asyncSkipEntries(int numEntriesToSkip, IndividualDeletedEntries dele } asyncMarkDelete(ledger.getPositionAfterN(markDeletePosition, numEntriesToSkip + numDeletedMessages, - ManagedLedgerImpl.PositionBound.startExcluded), new MarkDeleteCallback() { + PositionBound.startExcluded), new MarkDeleteCallback() { @Override public void markDeleteComplete(Object ctx) { callback.skipEntriesComplete(ctx); @@ -3485,6 +3485,7 @@ public Position processIndividuallyDeletedMessagesAndGetMarkDeletedPosition( return mdp; } + @Override public boolean isMessageDeleted(Position position) { lock.readLock().lock(); try { @@ -3496,6 +3497,7 @@ public boolean isMessageDeleted(Position position) { } //this method will return a copy of the position's ack set + @Override public long[] getBatchPositionAckSet(Position position) { if (batchDeletedIndexes != null) { BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.get(position); @@ -3618,6 +3620,7 @@ public ManagedCursorMXBean getStats() { return this.mbean; } + @Override public void updateReadStats(int readEntriesCount, long readEntriesSize) { this.entriesReadCount += readEntriesCount; this.entriesReadSize += readEntriesSize; @@ -3650,6 +3653,7 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { }, null); } + @Override public int applyMaxSizeCap(int maxEntries, long maxSizeBytes) { if (maxSizeBytes == NO_MAX_SIZE_LIMIT) { return maxEntries; @@ -3712,6 +3716,7 @@ public ManagedLedgerConfig getConfig() { /*** * Create a non-durable cursor and copy the ack stats. */ + @Override public ManagedCursor duplicateNonDurableCursor(String nonDurableCursorName) throws ManagedLedgerException { NonDurableCursorImpl newNonDurableCursor = (NonDurableCursorImpl) ledger.newNonDurableCursor(getMarkDeletedPosition(), nonDurableCursorName); @@ -3746,4 +3751,24 @@ public ManagedCursorAttributes getManagedCursorAttributes() { } return ATTRIBUTES_UPDATER.updateAndGet(this, old -> old != null ? old : new ManagedCursorAttributes(this)); } + + @Override + public ManagedLedgerInternalStats.CursorStats getCursorStats() { + ManagedLedgerInternalStats.CursorStats cs = new ManagedLedgerInternalStats.CursorStats(); + cs.markDeletePosition = getMarkDeletedPosition().toString(); + cs.readPosition = getReadPosition().toString(); + cs.waitingReadOp = hasPendingReadRequest(); + cs.pendingReadOps = getPendingReadOpsCount(); + cs.messagesConsumedCounter = getMessagesConsumedCounter(); + cs.cursorLedger = getCursorLedger(); + cs.cursorLedgerLastEntry = getCursorLedgerLastEntry(); + cs.individuallyDeletedMessages = getIndividuallyDeletedMessages(); + cs.lastLedgerSwitchTimestamp = DateFormatter.format(getLastLedgerSwitchTimestamp()); + cs.state = getState(); + cs.active = isActive(); + cs.numberOfEntriesSinceFirstNotAckedMessage = getNumberOfEntriesSinceFirstNotAckedMessage(); + cs.totalNonContiguousDeletedMessagesRange = getTotalNonContiguousDeletedMessagesRange(); + cs.properties = getProperties(); + return cs; + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java index 8ba800ff88130..586beb412d297 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java @@ -22,14 +22,19 @@ import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.NULL_OFFLOAD_PROMISE; import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables; import com.google.common.base.Predicates; +import com.google.common.collect.BoundType; import com.google.common.collect.Maps; +import com.google.common.collect.Range; +import com.google.protobuf.InvalidProtocolBufferException; import io.netty.util.concurrent.DefaultThreadFactory; import io.opentelemetry.api.OpenTelemetry; import java.io.IOException; import java.util.ArrayList; +import java.util.Enumeration; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -45,8 +50,12 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.Getter; +import org.apache.bookkeeper.client.AsyncCallback; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.LedgerEntry; +import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -70,7 +79,10 @@ import org.apache.bookkeeper.mledger.MetadataCompressionConfig; import org.apache.bookkeeper.mledger.OpenTelemetryManagedLedgerCacheStats; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.ReadOnlyCursor; +import org.apache.bookkeeper.mledger.ReadOnlyManagedLedger; +import org.apache.bookkeeper.mledger.ReadOnlyManagedLedgerImplWrapper; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.ManagedLedgerInitializeLedgerCallback; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.State; import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback; @@ -81,14 +93,18 @@ import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongProperty; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.MessageRange; +import org.apache.bookkeeper.mledger.util.Errors; import org.apache.bookkeeper.mledger.util.Futures; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig; +import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.Runnables; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.Stat; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; @@ -301,7 +317,8 @@ private synchronized void doCacheEviction() { * * @return */ - public Map getManagedLedgers() { + @Override + public Map getManagedLedgers() { // Return a view of already created ledger by filtering futures not yet completed return Maps.filterValues(Maps.transformValues(ledgers, future -> future.getNow(null)), Predicates.notNull()); } @@ -468,7 +485,7 @@ public void asyncOpenReadOnlyManagedLedger(String managedLedgerName, .get(new EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(), config.getBookKeeperEnsemblePlacementPolicyProperties())) .thenCompose(bk -> { - ReadOnlyManagedLedgerImpl roManagedLedger = new ReadOnlyManagedLedgerImpl(this, bk, + ReadOnlyManagedLedgerImplWrapper roManagedLedger = new ReadOnlyManagedLedgerImplWrapper(this, bk, store, config, scheduledExecutor, managedLedgerName); return roManagedLedger.initialize().thenApply(v -> roManagedLedger); }).thenAccept(roManagedLedger -> { @@ -524,7 +541,7 @@ public void asyncOpenReadOnlyCursor(String managedLedgerName, Position startPosi AsyncCallbacks.OpenReadOnlyManagedLedgerCallback openReadOnlyManagedLedgerCallback = new AsyncCallbacks.OpenReadOnlyManagedLedgerCallback() { @Override - public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger readOnlyManagedLedger, Object ctx) { callback.openReadOnlyCursorComplete(readOnlyManagedLedger. createReadOnlyCursor(startPosition), ctx); } @@ -1041,6 +1058,7 @@ public MetaStore getMetaStore() { return store; } + @Override public ManagedLedgerFactoryConfig getConfig() { return config; } @@ -1060,6 +1078,7 @@ public long getCacheEvictionTimeThreshold(){ return cacheEvictionTimeThresholdNanos; } + @Override public ManagedLedgerFactoryMXBean getCacheStats() { return this.mbean; } @@ -1068,6 +1087,408 @@ public CompletableFuture getBookKeeper() { return bookkeeperFactory.get(); } + @Override + public void estimateUnloadedTopicBacklog(PersistentOfflineTopicStats offlineTopicStats, + TopicName topicName, boolean accurate, Object ctx) + throws Exception { + String managedLedgerName = topicName.getPersistenceNamingEncoding(); + long numberOfEntries = 0; + long totalSize = 0; + BookKeeper.DigestType digestType = (BookKeeper.DigestType) ((List) ctx).get(0); + byte[] password = (byte[]) ((List) ctx).get(1); + NavigableMap ledgers = + getManagedLedgersInfo(topicName, accurate, digestType, password); + for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : ledgers.values()) { + numberOfEntries += ls.getEntries(); + totalSize += ls.getSize(); + if (accurate) { + offlineTopicStats.addLedgerDetails(ls.getEntries(), ls.getTimestamp(), ls.getSize(), ls.getLedgerId()); + } + } + offlineTopicStats.totalMessages = numberOfEntries; + offlineTopicStats.storageSize = totalSize; + if (log.isDebugEnabled()) { + log.debug("[{}] Total number of entries - {} and size - {}", managedLedgerName, numberOfEntries, totalSize); + } + + // calculate per cursor message backlog + calculateCursorBacklogs(topicName, ledgers, offlineTopicStats, accurate, digestType, password); + offlineTopicStats.statGeneratedAt.setTime(System.currentTimeMillis()); + } + + private NavigableMap getManagedLedgersInfo( + final TopicName topicName, boolean accurate, BookKeeper.DigestType digestType, byte[] password) + throws Exception { + final NavigableMap ledgers = new ConcurrentSkipListMap<>(); + + String managedLedgerName = topicName.getPersistenceNamingEncoding(); + MetaStore store = getMetaStore(); + + final CountDownLatch mlMetaCounter = new CountDownLatch(1); + store.getManagedLedgerInfo(managedLedgerName, false /* createIfMissing */, + new MetaStore.MetaStoreCallback() { + @Override + public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat) { + for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : mlInfo.getLedgerInfoList()) { + ledgers.put(ls.getLedgerId(), ls); + } + + // find no of entries in last ledger + if (!ledgers.isEmpty()) { + final long id = ledgers.lastKey(); + AsyncCallback.OpenCallback opencb = (rc, lh, ctx1) -> { + if (log.isDebugEnabled()) { + log.debug("[{}] Opened ledger {}: {}", managedLedgerName, id, + BKException.getMessage(rc)); + } + if (rc == BKException.Code.OK) { + MLDataFormats.ManagedLedgerInfo.LedgerInfo info = + MLDataFormats.ManagedLedgerInfo.LedgerInfo + .newBuilder().setLedgerId(id) + .setEntries(lh.getLastAddConfirmed() + 1) + .setSize(lh.getLength()).setTimestamp(System.currentTimeMillis()) + .build(); + ledgers.put(id, info); + mlMetaCounter.countDown(); + } else if (Errors.isNoSuchLedgerExistsException(rc)) { + log.warn("[{}] Ledger not found: {}", managedLedgerName, ledgers.lastKey()); + ledgers.remove(ledgers.lastKey()); + mlMetaCounter.countDown(); + } else { + log.error("[{}] Failed to open ledger {}: {}", managedLedgerName, id, + BKException.getMessage(rc)); + mlMetaCounter.countDown(); + } + }; + + if (log.isDebugEnabled()) { + log.debug("[{}] Opening ledger {}", managedLedgerName, id); + } + getBookKeeper() + .thenAccept(bk -> { + bk.asyncOpenLedgerNoRecovery(id, digestType, password, opencb, null); + }).exceptionally(ex -> { + log.warn("[{}] Failed to open ledger {}: {}", managedLedgerName, id, ex); + opencb.openComplete(-1, null, null); + mlMetaCounter.countDown(); + return null; + }); + } else { + log.warn("[{}] Ledger list empty", managedLedgerName); + mlMetaCounter.countDown(); + } + } + + @Override + public void operationFailed(ManagedLedgerException.MetaStoreException e) { + log.warn("[{}] Unable to obtain managed ledger metadata - {}", managedLedgerName, e); + mlMetaCounter.countDown(); + } + }); + + if (accurate) { + // block until however long it takes for operation to complete + mlMetaCounter.await(); + } else { + mlMetaCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + return ledgers; + } + + public void calculateCursorBacklogs(final TopicName topicName, + final NavigableMap ledgers, + final PersistentOfflineTopicStats offlineTopicStats, boolean accurate, + BookKeeper.DigestType digestType, byte[] password) throws Exception { + if (ledgers.isEmpty()) { + return; + } + String managedLedgerName = topicName.getPersistenceNamingEncoding(); + MetaStore store = getMetaStore(); + BookKeeper bk = getBookKeeper().get(); + final CountDownLatch allCursorsCounter = new CountDownLatch(1); + final long errorInReadingCursor = -1; + ConcurrentOpenHashMap ledgerRetryMap = + ConcurrentOpenHashMap.newBuilder().build(); + + final MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = ledgers.lastEntry().getValue(); + final Position lastLedgerPosition = + PositionFactory.create(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1); + if (log.isDebugEnabled()) { + log.debug("[{}] Last ledger position {}", managedLedgerName, lastLedgerPosition); + } + + store.getCursors(managedLedgerName, new MetaStore.MetaStoreCallback>() { + @Override + public void operationComplete(List cursors, Stat v) { + // Load existing cursors + if (log.isDebugEnabled()) { + log.debug("[{}] Found {} cursors", managedLedgerName, cursors.size()); + } + + if (cursors.isEmpty()) { + allCursorsCounter.countDown(); + return; + } + + final CountDownLatch cursorCounter = new CountDownLatch(cursors.size()); + + for (final String cursorName : cursors) { + // determine subscription position from cursor ledger + if (log.isDebugEnabled()) { + log.debug("[{}] Loading cursor {}", managedLedgerName, cursorName); + } + + AsyncCallback.OpenCallback cursorLedgerOpenCb = (rc, lh, ctx1) -> { + long ledgerId = lh.getId(); + if (log.isDebugEnabled()) { + log.debug("[{}] Opened cursor ledger {} for cursor {}. rc={}", managedLedgerName, ledgerId, + cursorName, rc); + } + if (rc != BKException.Code.OK) { + log.warn("[{}] Error opening metadata ledger {} for cursor {}: {}", managedLedgerName, + ledgerId, cursorName, BKException.getMessage(rc)); + cursorCounter.countDown(); + return; + } + long lac = lh.getLastAddConfirmed(); + if (log.isDebugEnabled()) { + log.debug("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, + ledgerId); + } + + if (lac == LedgerHandle.INVALID_ENTRY_ID) { + // save the ledger id and cursor to retry outside of this call back + // since we are trying to read the same cursor ledger, we will block until + // this current callback completes, since an attempt to read the entry + // will block behind this current operation to complete + ledgerRetryMap.put(cursorName, ledgerId); + log.info("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, + ledgerId); + cursorCounter.countDown(); + return; + } + final long entryId = lac; + // read last acked message position for subscription + lh.asyncReadEntries(entryId, entryId, new AsyncCallback.ReadCallback() { + @Override + public void readComplete(int rc, LedgerHandle lh, Enumeration seq, + Object ctx) { + try { + if (log.isDebugEnabled()) { + log.debug("readComplete rc={} entryId={}", rc, entryId); + } + if (rc != BKException.Code.OK) { + log.warn("[{}] Error reading from metadata ledger {} for cursor {}: {}", + managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc)); + // indicate that this cursor should be excluded + offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, + lh.getId()); + } else { + LedgerEntry entry = seq.nextElement(); + MLDataFormats.PositionInfo positionInfo; + try { + positionInfo = MLDataFormats.PositionInfo.parseFrom(entry.getEntry()); + } catch (InvalidProtocolBufferException e) { + log.warn( + "[{}] Error reading position from metadata ledger {} for cursor " + + "{}: {}", managedLedgerName, ledgerId, cursorName, e); + offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, + lh.getId()); + return; + } + final Position lastAckedMessagePosition = + PositionFactory.create(positionInfo.getLedgerId(), + positionInfo.getEntryId()); + if (log.isDebugEnabled()) { + log.debug("[{}] Cursor {} MD {} read last ledger position {}", + managedLedgerName, cursorName, lastAckedMessagePosition, + lastLedgerPosition); + } + // calculate cursor backlog + Range range = Range.openClosed(lastAckedMessagePosition, + lastLedgerPosition); + if (log.isDebugEnabled()) { + log.debug("[{}] Calculating backlog for cursor {} using range {}", + managedLedgerName, cursorName, range); + } + long cursorBacklog = getNumberOfEntries(range, ledgers); + offlineTopicStats.messageBacklog += cursorBacklog; + offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, lh.getId()); + } + } finally { + cursorCounter.countDown(); + } + } + }, null); + + }; // end of cursor meta read callback + + store.asyncGetCursorInfo(managedLedgerName, cursorName, + new MetaStore.MetaStoreCallback() { + @Override + public void operationComplete(MLDataFormats.ManagedCursorInfo info, + Stat stat) { + long cursorLedgerId = info.getCursorsLedgerId(); + if (log.isDebugEnabled()) { + log.debug("[{}] Cursor {} meta-data read ledger id {}", managedLedgerName, + cursorName, cursorLedgerId); + } + if (cursorLedgerId != -1) { + bk.asyncOpenLedgerNoRecovery(cursorLedgerId, digestType, password, + cursorLedgerOpenCb, null); + } else { + Position lastAckedMessagePosition = PositionFactory.create( + info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId()); + Range range = Range.openClosed(lastAckedMessagePosition, + lastLedgerPosition); + if (log.isDebugEnabled()) { + log.debug("[{}] Calculating backlog for cursor {} using range {}", + managedLedgerName, cursorName, range); + } + long cursorBacklog = getNumberOfEntries(range, ledgers); + offlineTopicStats.messageBacklog += cursorBacklog; + offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, cursorLedgerId); + cursorCounter.countDown(); + } + + } + + @Override + public void operationFailed(ManagedLedgerException.MetaStoreException e) { + log.warn("[{}] Unable to obtain cursor ledger for cursor {}: {}", managedLedgerName, + cursorName, e); + cursorCounter.countDown(); + } + }); + } // for every cursor find backlog + try { + if (accurate) { + cursorCounter.await(); + } else { + cursorCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + } catch (Exception e) { + log.warn("[{}] Error reading subscription positions{}", managedLedgerName, e); + } finally { + allCursorsCounter.countDown(); + } + } + + @Override + public void operationFailed(ManagedLedgerException.MetaStoreException e) { + log.warn("[{}] Failed to get the cursors list", managedLedgerName, e); + allCursorsCounter.countDown(); + } + }); + if (accurate) { + allCursorsCounter.await(); + } else { + allCursorsCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + // go through ledgers where LAC was -1 + if (accurate && ledgerRetryMap.size() > 0) { + ledgerRetryMap.forEach((cursorName, ledgerId) -> { + if (log.isDebugEnabled()) { + log.debug("Cursor {} Ledger {} Trying to obtain MD from BkAdmin", cursorName, ledgerId); + } + Position lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName); + if (lastAckedMessagePosition == null) { + log.warn("[{}] Cursor {} read from ledger {}. Unable to determine cursor position", + managedLedgerName, cursorName, ledgerId); + } else { + if (log.isDebugEnabled()) { + log.debug("[{}] Cursor {} read from ledger using bk admin {}. position {}", managedLedgerName, + cursorName, ledgerId, lastAckedMessagePosition); + } + // calculate cursor backlog + Range range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition); + if (log.isDebugEnabled()) { + log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, + cursorName, range); + } + long cursorBacklog = getNumberOfEntries(range, ledgers); + offlineTopicStats.messageBacklog += cursorBacklog; + offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId); + } + }); + } + } + + // need a better way than to duplicate the functionality below from ML + private long getNumberOfEntries(Range range, + NavigableMap ledgers) { + Position fromPosition = range.lowerEndpoint(); + boolean fromIncluded = range.lowerBoundType() == BoundType.CLOSED; + Position toPosition = range.upperEndpoint(); + boolean toIncluded = range.upperBoundType() == BoundType.CLOSED; + + if (fromPosition.getLedgerId() == toPosition.getLedgerId()) { + // If the 2 positions are in the same ledger + long count = toPosition.getEntryId() - fromPosition.getEntryId() - 1; + count += fromIncluded ? 1 : 0; + count += toIncluded ? 1 : 0; + return count; + } else { + long count = 0; + // If the from & to are pointing to different ledgers, then we need to : + // 1. Add the entries in the ledger pointed by toPosition + count += toPosition.getEntryId(); + count += toIncluded ? 1 : 0; + + // 2. Add the entries in the ledger pointed by fromPosition + MLDataFormats.ManagedLedgerInfo.LedgerInfo li = ledgers.get(fromPosition.getLedgerId()); + if (li != null) { + count += li.getEntries() - (fromPosition.getEntryId() + 1); + count += fromIncluded ? 1 : 0; + } + + // 3. Add the whole ledgers entries in between + for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : ledgers + .subMap(fromPosition.getLedgerId(), false, toPosition.getLedgerId(), false).values()) { + count += ls.getEntries(); + } + + return count; + } + } + + + private Position tryGetMDPosition(BookKeeper bookKeeper, long ledgerId, String cursorName) { + BookKeeperAdmin bookKeeperAdmin = null; + long lastEntry = LedgerHandle.INVALID_ENTRY_ID; + Position lastAckedMessagePosition = null; + try { + bookKeeperAdmin = new BookKeeperAdmin(bookKeeper); + for (LedgerEntry ledgerEntry : bookKeeperAdmin.readEntries(ledgerId, 0, lastEntry)) { + lastEntry = ledgerEntry.getEntryId(); + if (log.isDebugEnabled()) { + log.debug(" Read entry {} from ledger {} for cursor {}", lastEntry, ledgerId, cursorName); + } + MLDataFormats.PositionInfo positionInfo = MLDataFormats.PositionInfo.parseFrom(ledgerEntry.getEntry()); + lastAckedMessagePosition = + PositionFactory.create(positionInfo.getLedgerId(), positionInfo.getEntryId()); + if (log.isDebugEnabled()) { + log.debug("Cursor {} read position {}", cursorName, lastAckedMessagePosition); + } + } + } catch (Exception e) { + log.warn("Unable to determine LAC for ledgerId {} for cursor {}: {}", ledgerId, cursorName, e); + } finally { + if (bookKeeperAdmin != null) { + try { + bookKeeperAdmin.close(); + } catch (Exception e) { + log.warn("Unable to close bk admin for ledgerId {} for cursor {}", ledgerId, cursorName, e); + } + } + + } + return lastAckedMessagePosition; + } + + private static final int META_READ_TIMEOUT_SECONDS = 60; + /** * Factory to create Bookkeeper-client for a given ensemblePlacementPolicy. * diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 8cb5a3ee6acec..7ca15d835aed2 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.time.Clock; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -77,10 +78,12 @@ import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.client.api.ReadHandle; import org.apache.bookkeeper.common.util.Backoff; import org.apache.bookkeeper.common.util.OrderedScheduler; import org.apache.bookkeeper.common.util.Retries; +import org.apache.bookkeeper.discover.RegistrationClient; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback; @@ -115,6 +118,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException; import org.apache.bookkeeper.mledger.ManagedLedgerMXBean; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.WaitingEntryCallBack; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback; @@ -129,6 +133,7 @@ import org.apache.bookkeeper.mledger.proto.MLDataFormats.OffloadContext; import org.apache.bookkeeper.mledger.util.CallbackMutex; import org.apache.bookkeeper.mledger.util.Futures; +import org.apache.bookkeeper.mledger.util.ManagedLedgerImplUtils; import org.apache.bookkeeper.net.BookieId; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition; @@ -286,10 +291,7 @@ public boolean isFenced() { } } - // define boundaries for position based seeks and searches - public enum PositionBound { - startIncluded, startExcluded - } + protected static final AtomicReferenceFieldUpdater STATE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(ManagedLedgerImpl.class, State.class, "state"); @@ -1207,7 +1209,7 @@ public CompletableFuture getEarliestMessagePublishTimeInBacklog() { return getEarliestMessagePublishTimeOfPos(pos); } - public CompletableFuture getEarliestMessagePublishTimeOfPos(Position pos) { + private CompletableFuture getEarliestMessagePublishTimeOfPos(Position pos) { CompletableFuture future = new CompletableFuture<>(); if (pos == null) { future.complete(0L); @@ -1881,12 +1883,12 @@ void asyncReadEntries(OpReadEntry opReadEntry) { } } - public CompletableFuture getLedgerMetadata(long ledgerId) { + public CompletableFuture getLedgerMetadata(long ledgerId) { LedgerHandle currentLedger = this.currentLedger; if (currentLedger != null && ledgerId == currentLedger.getId()) { - return CompletableFuture.completedFuture(currentLedger.getLedgerMetadata().toSafeString()); + return CompletableFuture.completedFuture(currentLedger.getLedgerMetadata()); } else { - return getLedgerHandle(ledgerId).thenApply(rh -> rh.getLedgerMetadata().toSafeString()); + return getLedgerHandle(ledgerId).thenApply(rh -> rh.getLedgerMetadata()); } } @@ -1992,6 +1994,7 @@ public void invalidateLedgerHandle(ReadHandle ledgerHandle) { } } + @Override public void asyncReadEntry(Position position, ReadEntryCallback callback, Object ctx) { LedgerHandle currentLedger = this.currentLedger; if (log.isDebugEnabled()) { @@ -3511,6 +3514,7 @@ public long getNumberOfEntries(Range range) { * specifies whether to include the start position in calculating the distance * @return the new position that is n entries ahead */ + @Override public Position getPositionAfterN(final Position startPosition, long n, PositionBound startRange) { long entriesToSkip = n; long currentLedgerId; @@ -3609,6 +3613,7 @@ private boolean isNoMessagesAfterPosForSpecifiedLac(Position specifiedLac, Posit * the current position * @return the previous position */ + @Override public Position getPreviousPosition(Position position) { if (position.getEntryId() > 0) { return PositionFactory.create(position.getLedgerId(), position.getEntryId() - 1); @@ -3687,9 +3692,11 @@ public Long getNextValidLedger(long ledgerId) { return ledgers.ceilingKey(ledgerId + 1); } + @Override public Position getNextValidPosition(final Position position) { return getValidPositionAfterSkippedEntries(position, 1); } + public Position getValidPositionAfterSkippedEntries(final Position position, int skippedEntryNum) { Position skippedPosition = position.getPositionAfterEntries(skippedEntryNum); while (!isValidPosition(skippedPosition)) { @@ -3839,6 +3846,7 @@ public List getLedgersInfoAsList() { return Lists.newArrayList(ledgers.values()); } + @Override public NavigableMap getLedgersInfo() { return ledgers; } @@ -3952,6 +3960,7 @@ public int getWaitingCursorsCount() { return waitingCursors.size(); } + @Override public int getPendingAddEntriesCount() { return pendingAddEntries.size(); } @@ -3965,6 +3974,7 @@ public State getState() { return STATE_UPDATER.get(this); } + @Override public long getCacheSize() { return entryCache.getSize(); } @@ -4382,7 +4392,7 @@ public CompletableFuture getManagedLedgerInternalSta List ledgersInfos = new ArrayList<>(this.getLedgersInfo().values()); // add asynchronous metadata retrieval operations to a hashmap - Map> ledgerMetadataFutures = new HashMap(); + Map> ledgerMetadataFutures = new HashMap(); if (includeLedgerMetadata) { ledgersInfos.forEach(li -> { long ledgerId = li.getLedgerId(); @@ -4393,24 +4403,55 @@ public CompletableFuture getManagedLedgerInternalSta }); } + + CompletableFuture> bookiesFuture; + if (includeLedgerMetadata) { + RegistrationClient registrationClient = bookKeeper.getMetadataClientDriver().getRegistrationClient(); + bookiesFuture = registrationClient.getReadOnlyBookies() + .thenCombine(registrationClient.getWritableBookies(), (readOnlyBookies, writableBookies) -> { + Set bookies = new HashSet<>(); + bookies.addAll(readOnlyBookies.getValue()); + bookies.addAll(writableBookies.getValue()); + return bookies; + }); + } else { + bookiesFuture = CompletableFuture.completedFuture(null); + } + // wait until metadata has been retrieved - FutureUtil.waitForAll(ledgerMetadataFutures.values()).thenAccept(__ -> { - stats.ledgers = new ArrayList(); - ledgersInfos.forEach(li -> { - ManagedLedgerInternalStats.LedgerInfo info = new ManagedLedgerInternalStats.LedgerInfo(); - info.ledgerId = li.getLedgerId(); - info.entries = li.getEntries(); - info.size = li.getSize(); - info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete(); - if (includeLedgerMetadata) { - // lookup metadata from the hashmap which contains completed async operations - info.metadata = ledgerMetadataFutures.get(li.getLedgerId()).getNow(null); - } - stats.ledgers.add(info); + bookiesFuture.thenCompose(bookies -> + FutureUtil.waitForAll(ledgerMetadataFutures.values()).thenAccept(__ -> { + stats.ledgers = new ArrayList<>(); + ledgersInfos.forEach(li -> { + ManagedLedgerInternalStats.LedgerInfo info = new ManagedLedgerInternalStats.LedgerInfo(); + info.ledgerId = li.getLedgerId(); + info.entries = li.getEntries(); + info.size = li.getSize(); + info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete(); + if (includeLedgerMetadata) { + // lookup metadata from the hashmap which contains completed async operations + LedgerMetadata lm = ledgerMetadataFutures.get(li.getLedgerId()).getNow(null); + if (lm == null) { + info.metadata = null; + info.underReplicated = false; + } else { + info.metadata = lm.toSafeString(); + Set ensemble = lm.getAllEnsembles().values().stream() + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + if (bookies != null) { + info.underReplicated = !bookies.contains(ensemble); + } + } + } + stats.ledgers.add(info); + }); + statFuture.complete(stats); + })) + .exceptionally(e -> { + statFuture.completeExceptionally(e); + return null; }); - statFuture.complete(stats); - }); - return statFuture; } @@ -4540,4 +4581,11 @@ public Position getTheSlowestNonDurationReadPosition() { } return theSlowestNonDurableReadPosition; } + + @Override + public CompletableFuture getLastDispatchablePosition(final Predicate predicate, + final Position startPosition) { + return ManagedLedgerImplUtils + .asyncGetLastValidPosition(this, predicate, startPosition); + } } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java index 60e24e8df0498..9a1753c715eff 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java @@ -20,29 +20,16 @@ import com.google.common.collect.BoundType; import com.google.common.collect.Range; -import com.google.protobuf.InvalidProtocolBufferException; -import java.util.Enumeration; +import java.util.ArrayList; import java.util.List; import java.util.NavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import org.apache.bookkeeper.client.AsyncCallback; -import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; -import org.apache.bookkeeper.client.BookKeeperAdmin; -import org.apache.bookkeeper.client.LedgerEntry; -import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.api.DigestType; -import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.proto.MLDataFormats; -import org.apache.bookkeeper.mledger.util.Errors; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.metadata.api.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,369 +89,28 @@ private long getNumberOfEntries(Range range, } } - public PersistentOfflineTopicStats getEstimatedUnloadedTopicBacklog(ManagedLedgerFactoryImpl factory, + public PersistentOfflineTopicStats getEstimatedUnloadedTopicBacklog(ManagedLedgerFactory factory, String managedLedgerName) throws Exception { return estimateUnloadedTopicBacklog(factory, TopicName.get("persistent://" + managedLedgerName)); } - public PersistentOfflineTopicStats estimateUnloadedTopicBacklog(ManagedLedgerFactoryImpl factory, - TopicName topicName) throws Exception { + public PersistentOfflineTopicStats estimateUnloadedTopicBacklog(ManagedLedgerFactory factory, + TopicName topicName) throws Exception { String managedLedgerName = topicName.getPersistenceNamingEncoding(); - long numberOfEntries = 0; - long totalSize = 0; - final NavigableMap ledgers = new ConcurrentSkipListMap<>(); final PersistentOfflineTopicStats offlineTopicStats = new PersistentOfflineTopicStats(managedLedgerName, brokerName); - - // calculate total managed ledger size and number of entries without loading the topic - readLedgerMeta(factory, topicName, ledgers); - for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : ledgers.values()) { - numberOfEntries += ls.getEntries(); - totalSize += ls.getSize(); - if (accurate) { - offlineTopicStats.addLedgerDetails(ls.getEntries(), ls.getTimestamp(), ls.getSize(), ls.getLedgerId()); - } - } - offlineTopicStats.totalMessages = numberOfEntries; - offlineTopicStats.storageSize = totalSize; - if (log.isDebugEnabled()) { - log.debug("[{}] Total number of entries - {} and size - {}", managedLedgerName, numberOfEntries, totalSize); - } - - // calculate per cursor message backlog - calculateCursorBacklogs(factory, topicName, ledgers, offlineTopicStats); - offlineTopicStats.statGeneratedAt.setTime(System.currentTimeMillis()); - - return offlineTopicStats; - } - - private void readLedgerMeta(final ManagedLedgerFactoryImpl factory, final TopicName topicName, - final NavigableMap ledgers) throws Exception { - String managedLedgerName = topicName.getPersistenceNamingEncoding(); - MetaStore store = factory.getMetaStore(); - - final CountDownLatch mlMetaCounter = new CountDownLatch(1); - - store.getManagedLedgerInfo(managedLedgerName, false /* createIfMissing */, - new MetaStore.MetaStoreCallback() { - @Override - public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat) { - for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : mlInfo.getLedgerInfoList()) { - ledgers.put(ls.getLedgerId(), ls); - } - - // find no of entries in last ledger - if (!ledgers.isEmpty()) { - final long id = ledgers.lastKey(); - AsyncCallback.OpenCallback opencb = (rc, lh, ctx1) -> { - if (log.isDebugEnabled()) { - log.debug("[{}] Opened ledger {}: {}", managedLedgerName, id, - BKException.getMessage(rc)); - } - if (rc == BKException.Code.OK) { - MLDataFormats.ManagedLedgerInfo.LedgerInfo info = - MLDataFormats.ManagedLedgerInfo.LedgerInfo - .newBuilder().setLedgerId(id).setEntries(lh.getLastAddConfirmed() + 1) - .setSize(lh.getLength()).setTimestamp(System.currentTimeMillis()).build(); - ledgers.put(id, info); - mlMetaCounter.countDown(); - } else if (Errors.isNoSuchLedgerExistsException(rc)) { - log.warn("[{}] Ledger not found: {}", managedLedgerName, ledgers.lastKey()); - ledgers.remove(ledgers.lastKey()); - mlMetaCounter.countDown(); - } else { - log.error("[{}] Failed to open ledger {}: {}", managedLedgerName, id, - BKException.getMessage(rc)); - mlMetaCounter.countDown(); - } - }; - - if (log.isDebugEnabled()) { - log.debug("[{}] Opening ledger {}", managedLedgerName, id); - } - - factory.getBookKeeper() - .thenAccept(bk -> { - bk.asyncOpenLedgerNoRecovery(id, digestType, password, opencb, null); - }).exceptionally(ex -> { - log.warn("[{}] Failed to open ledger {}: {}", managedLedgerName, id, ex); - opencb.openComplete(-1, null, null); - mlMetaCounter.countDown(); - return null; - }); - } else { - log.warn("[{}] Ledger list empty", managedLedgerName); - mlMetaCounter.countDown(); - } - } - - @Override - public void operationFailed(ManagedLedgerException.MetaStoreException e) { - log.warn("[{}] Unable to obtain managed ledger metadata - {}", managedLedgerName, e); - mlMetaCounter.countDown(); - } - }); - - if (accurate) { - // block until however long it takes for operation to complete - mlMetaCounter.await(); + if (factory instanceof ManagedLedgerFactoryImpl) { + List ctx = new ArrayList<>(); + ctx.add(digestType); + ctx.add(password); + factory.estimateUnloadedTopicBacklog(offlineTopicStats, topicName, accurate, ctx); } else { - mlMetaCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); - + Object ctx = null; + factory.estimateUnloadedTopicBacklog(offlineTopicStats, topicName, accurate, ctx); } - } - - private void calculateCursorBacklogs(final ManagedLedgerFactoryImpl factory, final TopicName topicName, - final NavigableMap ledgers, - final PersistentOfflineTopicStats offlineTopicStats) throws Exception { - - if (ledgers.isEmpty()) { - return; - } - String managedLedgerName = topicName.getPersistenceNamingEncoding(); - MetaStore store = factory.getMetaStore(); - BookKeeper bk = factory.getBookKeeper().get(); - final CountDownLatch allCursorsCounter = new CountDownLatch(1); - final long errorInReadingCursor = -1; - ConcurrentOpenHashMap ledgerRetryMap = - ConcurrentOpenHashMap.newBuilder().build(); - final MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = ledgers.lastEntry().getValue(); - final Position lastLedgerPosition = - PositionFactory.create(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1); - if (log.isDebugEnabled()) { - log.debug("[{}] Last ledger position {}", managedLedgerName, lastLedgerPosition); - } - store.getCursors(managedLedgerName, new MetaStore.MetaStoreCallback>() { - @Override - public void operationComplete(List cursors, Stat v) { - // Load existing cursors - if (log.isDebugEnabled()) { - log.debug("[{}] Found {} cursors", managedLedgerName, cursors.size()); - } - - if (cursors.isEmpty()) { - allCursorsCounter.countDown(); - return; - } - - final CountDownLatch cursorCounter = new CountDownLatch(cursors.size()); - - for (final String cursorName : cursors) { - // determine subscription position from cursor ledger - if (log.isDebugEnabled()) { - log.debug("[{}] Loading cursor {}", managedLedgerName, cursorName); - } - - AsyncCallback.OpenCallback cursorLedgerOpenCb = (rc, lh, ctx1) -> { - long ledgerId = lh.getId(); - if (log.isDebugEnabled()) { - log.debug("[{}] Opened cursor ledger {} for cursor {}. rc={}", managedLedgerName, ledgerId, - cursorName, rc); - } - if (rc != BKException.Code.OK) { - log.warn("[{}] Error opening metadata ledger {} for cursor {}: {}", managedLedgerName, - ledgerId, cursorName, BKException.getMessage(rc)); - cursorCounter.countDown(); - return; - } - long lac = lh.getLastAddConfirmed(); - if (log.isDebugEnabled()) { - log.debug("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, - ledgerId); - } - - if (lac == LedgerHandle.INVALID_ENTRY_ID) { - // save the ledger id and cursor to retry outside of this call back - // since we are trying to read the same cursor ledger, we will block until - // this current callback completes, since an attempt to read the entry - // will block behind this current operation to complete - ledgerRetryMap.put(cursorName, ledgerId); - log.info("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, - ledgerId); - cursorCounter.countDown(); - return; - } - final long entryId = lac; - // read last acked message position for subscription - lh.asyncReadEntries(entryId, entryId, new AsyncCallback.ReadCallback() { - @Override - public void readComplete(int rc, LedgerHandle lh, Enumeration seq, - Object ctx) { - try { - if (log.isDebugEnabled()) { - log.debug("readComplete rc={} entryId={}", rc, entryId); - } - if (rc != BKException.Code.OK) { - log.warn("[{}] Error reading from metadata ledger {} for cursor {}: {}", - managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc)); - // indicate that this cursor should be excluded - offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, - lh.getId()); - } else { - LedgerEntry entry = seq.nextElement(); - MLDataFormats.PositionInfo positionInfo; - try { - positionInfo = MLDataFormats.PositionInfo.parseFrom(entry.getEntry()); - } catch (InvalidProtocolBufferException e) { - log.warn( - "[{}] Error reading position from metadata ledger {} for cursor {}: {}", - managedLedgerName, ledgerId, cursorName, e); - offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, - lh.getId()); - return; - } - final Position lastAckedMessagePosition = - PositionFactory.create(positionInfo.getLedgerId(), - positionInfo.getEntryId()); - if (log.isDebugEnabled()) { - log.debug("[{}] Cursor {} MD {} read last ledger position {}", - managedLedgerName, cursorName, lastAckedMessagePosition, - lastLedgerPosition); - } - // calculate cursor backlog - Range range = Range.openClosed(lastAckedMessagePosition, - lastLedgerPosition); - if (log.isDebugEnabled()) { - log.debug("[{}] Calculating backlog for cursor {} using range {}", - managedLedgerName, cursorName, range); - } - long cursorBacklog = getNumberOfEntries(range, ledgers); - offlineTopicStats.messageBacklog += cursorBacklog; - offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, lh.getId()); - } - } finally { - cursorCounter.countDown(); - } - } - }, null); - - }; // end of cursor meta read callback - - store.asyncGetCursorInfo(managedLedgerName, cursorName, - new MetaStore.MetaStoreCallback() { - @Override - public void operationComplete(MLDataFormats.ManagedCursorInfo info, - Stat stat) { - long cursorLedgerId = info.getCursorsLedgerId(); - if (log.isDebugEnabled()) { - log.debug("[{}] Cursor {} meta-data read ledger id {}", managedLedgerName, - cursorName, cursorLedgerId); - } - if (cursorLedgerId != -1) { - bk.asyncOpenLedgerNoRecovery(cursorLedgerId, digestType, password, - cursorLedgerOpenCb, null); - } else { - Position lastAckedMessagePosition = PositionFactory.create( - info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId()); - Range range = Range.openClosed(lastAckedMessagePosition, - lastLedgerPosition); - if (log.isDebugEnabled()) { - log.debug("[{}] Calculating backlog for cursor {} using range {}", - managedLedgerName, cursorName, range); - } - long cursorBacklog = getNumberOfEntries(range, ledgers); - offlineTopicStats.messageBacklog += cursorBacklog; - offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, cursorLedgerId); - cursorCounter.countDown(); - } - - } - - @Override - public void operationFailed(ManagedLedgerException.MetaStoreException e) { - log.warn("[{}] Unable to obtain cursor ledger for cursor {}: {}", managedLedgerName, - cursorName, e); - cursorCounter.countDown(); - } - }); - } // for every cursor find backlog - try { - if (accurate) { - cursorCounter.await(); - } else { - cursorCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } - } catch (Exception e) { - log.warn("[{}] Error reading subscription positions{}", managedLedgerName, e); - } finally { - allCursorsCounter.countDown(); - } - } - - @Override - public void operationFailed(ManagedLedgerException.MetaStoreException e) { - log.warn("[{}] Failed to get the cursors list", managedLedgerName, e); - allCursorsCounter.countDown(); - } - }); - if (accurate) { - allCursorsCounter.await(); - } else { - allCursorsCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } - - // go through ledgers where LAC was -1 - if (accurate && ledgerRetryMap.size() > 0) { - ledgerRetryMap.forEach((cursorName, ledgerId) -> { - if (log.isDebugEnabled()) { - log.debug("Cursor {} Ledger {} Trying to obtain MD from BkAdmin", cursorName, ledgerId); - } - Position lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName); - if (lastAckedMessagePosition == null) { - log.warn("[{}] Cursor {} read from ledger {}. Unable to determine cursor position", - managedLedgerName, cursorName, ledgerId); - } else { - if (log.isDebugEnabled()) { - log.debug("[{}] Cursor {} read from ledger using bk admin {}. position {}", managedLedgerName, - cursorName, ledgerId, lastAckedMessagePosition); - } - // calculate cursor backlog - Range range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition); - if (log.isDebugEnabled()) { - log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, - cursorName, range); - } - long cursorBacklog = getNumberOfEntries(range, ledgers); - offlineTopicStats.messageBacklog += cursorBacklog; - offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId); - } - }); - } - } - - private Position tryGetMDPosition(BookKeeper bookKeeper, long ledgerId, String cursorName) { - BookKeeperAdmin bookKeeperAdmin = null; - long lastEntry = LedgerHandle.INVALID_ENTRY_ID; - Position lastAckedMessagePosition = null; - try { - bookKeeperAdmin = new BookKeeperAdmin(bookKeeper); - for (LedgerEntry ledgerEntry : bookKeeperAdmin.readEntries(ledgerId, 0, lastEntry)) { - lastEntry = ledgerEntry.getEntryId(); - if (log.isDebugEnabled()) { - log.debug(" Read entry {} from ledger {} for cursor {}", lastEntry, ledgerId, cursorName); - } - MLDataFormats.PositionInfo positionInfo = MLDataFormats.PositionInfo.parseFrom(ledgerEntry.getEntry()); - lastAckedMessagePosition = - PositionFactory.create(positionInfo.getLedgerId(), positionInfo.getEntryId()); - if (log.isDebugEnabled()) { - log.debug("Cursor {} read position {}", cursorName, lastAckedMessagePosition); - } - } - } catch (Exception e) { - log.warn("Unable to determine LAC for ledgerId {} for cursor {}: {}", ledgerId, cursorName, e); - } finally { - if (bookKeeperAdmin != null) { - try { - bookKeeperAdmin.close(); - } catch (Exception e) { - log.warn("Unable to close bk admin for ledgerId {} for cursor {}", ledgerId, cursorName, e); - } - } - - } - return lastAckedMessagePosition; + return offlineTopicStats; } private static final Logger log = LoggerFactory.getLogger(ManagedLedgerOfflineBacklog.class); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpFindNewest.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpFindNewest.java index 707cb389eba1a..26d5e8d3f661d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpFindNewest.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpFindNewest.java @@ -26,7 +26,7 @@ import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.PositionBound; +import org.apache.bookkeeper.mledger.PositionBound; @Slf4j class OpFindNewest implements ReadEntryCallback { @@ -97,7 +97,7 @@ public void readEntryComplete(Entry entry, Object ctx) { searchPosition = ledger.getPositionAfterN(searchPosition, max, PositionBound.startExcluded); Position lastPosition = ledger.getLastPosition(); searchPosition = - ledger.getPositionAfterN(searchPosition, max, ManagedLedgerImpl.PositionBound.startExcluded); + ledger.getPositionAfterN(searchPosition, max, PositionBound.startExcluded); if (lastPosition.compareTo(searchPosition) < 0) { if (log.isDebugEnabled()) { log.debug("first position {} matches, last should be {}, but moving to lastPos {}", position, diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpScan.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpScan.java index e4f6fd04ff4da..72d05ede3a0f5 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpScan.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpScan.java @@ -29,6 +29,7 @@ import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; import org.apache.bookkeeper.mledger.ScanOutcome; @Slf4j @@ -88,7 +89,7 @@ public void readEntriesComplete(List entries, Object ctx) { } } searchPosition = ledger.getPositionAfterN(lastPositionForBatch, 1, - ManagedLedgerImpl.PositionBound.startExcluded); + PositionBound.startExcluded); if (log.isDebugEnabled()) { log.debug("readEntryComplete {} at {} next is {}", lastPositionForBatch, searchPosition); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java index 93a749d4aef51..ec73c9d5e5eb2 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedCursorStats.java @@ -23,6 +23,7 @@ import io.opentelemetry.api.metrics.BatchCallback; import io.opentelemetry.api.metrics.ObservableLongMeasurement; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.pulsar.opentelemetry.Constants; public class OpenTelemetryManagedCursorStats implements AutoCloseable { @@ -98,7 +99,7 @@ public OpenTelemetryManagedCursorStats(OpenTelemetry openTelemetry, ManagedLedge batchCallback = meter.batchCallback(() -> factory.getManagedLedgers() .values() .stream() - .map(ManagedLedgerImpl::getCursors) + .map(ManagedLedger::getCursors) .flatMap(Streams::stream) .forEach(this::recordMetrics), persistOperationCounter, diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedLedgerStats.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedLedgerStats.java index f7b9d91dff6ad..26c4b62cf7694 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedLedgerStats.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpenTelemetryManagedLedgerStats.java @@ -21,6 +21,7 @@ import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.metrics.BatchCallback; import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.pulsar.opentelemetry.Constants; public class OpenTelemetryManagedLedgerStats implements AutoCloseable { @@ -130,8 +131,8 @@ public void close() { batchCallback.close(); } - private void recordMetrics(ManagedLedgerImpl ml) { - var stats = ml.getMbean(); + private void recordMetrics(ManagedLedger ml) { + var stats = ml.getStats(); var ledgerAttributeSet = ml.getManagedLedgerAttributes(); var attributes = ledgerAttributeSet.getAttributes(); var attributesSucceed = ledgerAttributeSet.getAttributesOperationSucceed(); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java index bd3e461d94e5c..00ed5a0c5b9d9 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java @@ -23,9 +23,9 @@ import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.ReadOnlyCursor; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.PositionBound; import org.apache.bookkeeper.mledger.proto.MLDataFormats; @Slf4j diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImpl.java index e64941c3201cb..1fb2aa3629092 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImpl.java @@ -43,12 +43,12 @@ public class ReadOnlyManagedLedgerImpl extends ManagedLedgerImpl { public ReadOnlyManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper, MetaStore store, - ManagedLedgerConfig config, OrderedScheduler scheduledExecutor, - String name) { + ManagedLedgerConfig config, OrderedScheduler scheduledExecutor, + String name) { super(factory, bookKeeper, store, config, scheduledExecutor, name); } - CompletableFuture initialize() { + public CompletableFuture initialize() { CompletableFuture future = new CompletableFuture<>(); // Fetch the list of existing ledgers in the managed ledger @@ -128,7 +128,7 @@ public void operationFailed(MetaStoreException e) { return future; } - ReadOnlyCursor createReadOnlyCursor(Position startPosition) { + public ReadOnlyCursor createReadOnlyCursor(Position startPosition) { if (ledgers.isEmpty()) { lastConfirmedEntry = PositionFactory.EARLIEST; } else if (ledgers.lastEntry().getValue().getEntries() > 0) { diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java index a387331f3c047..2afbcef0926e7 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java @@ -47,8 +47,10 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedCursorMXBean; import org.apache.bookkeeper.mledger.ManagedLedger; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; import org.testng.annotations.Test; public class ManagedCursorContainerTest { @@ -409,6 +411,36 @@ public boolean checkAndUpdateReadPositionChanged() { public boolean isClosed() { return false; } + + @Override + public ManagedLedgerInternalStats.CursorStats getCursorStats() { + return null; + } + + @Override + public boolean isMessageDeleted(Position position) { + return false; + } + + @Override + public ManagedCursor duplicateNonDurableCursor(String nonDurableCursorName) throws ManagedLedgerException { + return null; + } + + @Override + public long[] getBatchPositionAckSet(Position position) { + return new long[0]; + } + + @Override + public int applyMaxSizeCap(int maxEntries, long maxSizeBytes) { + return 0; + } + + @Override + public void updateReadStats(int readEntriesCount, long readEntriesSize) { + + } } @Test diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java index 500de5dd13879..990c298604e59 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java @@ -18,7 +18,7 @@ */ package org.apache.bookkeeper.mledger.impl; -import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; +import static org.apache.bookkeeper.mledger.ManagedCursor.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index bb38114ef7117..83a6c771513a9 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -123,6 +123,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig; import org.apache.bookkeeper.mledger.ManagedLedgerFactoryMXBean; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback; import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback; @@ -2178,7 +2179,9 @@ public void testNoRolloverIfNoMetadataSession() throws Exception { ledger.addEntry("data".getBytes()); // After the re-establishment, we'll be creating new ledgers - assertEquals(ledger.getLedgersInfoAsList().size(), 3); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(ledger.getLedgersInfoAsList().size(), 4); + }); } @Test @@ -2595,11 +2598,11 @@ public void testGetPositionAfterN() throws Exception { Position startPosition = PositionFactory.create(firstLedger, 0); - Position targetPosition = managedLedger.getPositionAfterN(startPosition, 1, ManagedLedgerImpl.PositionBound.startExcluded); + Position targetPosition = managedLedger.getPositionAfterN(startPosition, 1, PositionBound.startExcluded); assertEquals(targetPosition.getLedgerId(), firstLedger); assertEquals(targetPosition.getEntryId(), 1); - targetPosition = managedLedger.getPositionAfterN(startPosition, 4, ManagedLedgerImpl.PositionBound.startExcluded); + targetPosition = managedLedger.getPositionAfterN(startPosition, 4, PositionBound.startExcluded); assertEquals(targetPosition.getLedgerId(), firstLedger); assertEquals(targetPosition.getEntryId(), 4); @@ -2607,25 +2610,25 @@ public void testGetPositionAfterN() throws Exception { Position searchPosition = managedLedger.getNextValidPosition(managedCursor.getMarkDeletedPosition()); long length = managedCursor.getNumberOfEntriesInStorage(); // return the last confirm entry position if searchPosition is exceed the last confirm entry - targetPosition = managedLedger.getPositionAfterN(searchPosition, length, ManagedLedgerImpl.PositionBound.startExcluded); + targetPosition = managedLedger.getPositionAfterN(searchPosition, length, PositionBound.startExcluded); log.info("Target position is {}", targetPosition); assertEquals(targetPosition.getLedgerId(), secondLedger); assertEquals(targetPosition.getEntryId(), 4); // test for n > NumberOfEntriesInStorage searchPosition = PositionFactory.create(secondLedger, 0); - targetPosition = managedLedger.getPositionAfterN(searchPosition, 100, ManagedLedgerImpl.PositionBound.startIncluded); + targetPosition = managedLedger.getPositionAfterN(searchPosition, 100, PositionBound.startIncluded); assertEquals(targetPosition.getLedgerId(), secondLedger); assertEquals(targetPosition.getEntryId(), 4); // test for startPosition > current ledger searchPosition = PositionFactory.create(999, 0); - targetPosition = managedLedger.getPositionAfterN(searchPosition, 0, ManagedLedgerImpl.PositionBound.startIncluded); + targetPosition = managedLedger.getPositionAfterN(searchPosition, 0, PositionBound.startIncluded); assertEquals(targetPosition.getLedgerId(), secondLedger); assertEquals(targetPosition.getEntryId(), 4); searchPosition = PositionFactory.create(999, 0); - targetPosition = managedLedger.getPositionAfterN(searchPosition, 10, ManagedLedgerImpl.PositionBound.startExcluded); + targetPosition = managedLedger.getPositionAfterN(searchPosition, 10, PositionBound.startExcluded); assertEquals(targetPosition.getLedgerId(), secondLedger); assertEquals(targetPosition.getEntryId(), 4); } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImplTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImplTest.java index 028ecad407276..61056d0b4b602 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImplTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImplTest.java @@ -31,6 +31,7 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.ReadOnlyManagedLedger; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.testng.annotations.Test; @@ -56,7 +57,7 @@ public void testReadOnlyManagedLedgerImplAttachProperties() factory.asyncOpenReadOnlyManagedLedger(MANAGED_LEDGER_NAME_ATTACHED_PROPERTIES, new AsyncCallbacks.OpenReadOnlyManagedLedgerCallback() { @Override - public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl managedLedger, + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger managedLedger, Object ctx) { managedLedger.getProperties().forEach((key, value) -> { assertEquals(key, propertiesKey); @@ -85,7 +86,7 @@ public void testReadOnlyManagedLedgerImplNoProperties() factory.asyncOpenReadOnlyManagedLedger(MANAGED_LEDGER_NAME_NON_PROPERTIES, new AsyncCallbacks.OpenReadOnlyManagedLedgerCallback() { @Override - public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl managedLedger, + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger managedLedger, Object ctx) { assertEquals(managedLedger.getProperties().size(), 0); future.complete(null); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 425e7dafa1bf8..87196d3f3a9a6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -37,6 +37,7 @@ import java.lang.reflect.Constructor; import java.net.InetSocketAddress; import java.net.MalformedURLException; +import java.time.Clock; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; @@ -220,6 +221,7 @@ public class PulsarService implements AutoCloseable, ShutdownService { private StrategicTwoPhaseCompactor strategicCompactor; private ResourceUsageTransportManager resourceUsageTransportManager; private ResourceGroupService resourceGroupServiceManager; + private final Clock clock; private final ScheduledExecutorService executor; @@ -340,6 +342,7 @@ public PulsarService(ServiceConfiguration config, PulsarConfigurationLoader.isComplete(config); TransactionBatchedWriteValidator.validate(config); this.config = config; + this.clock = Clock.systemUTC(); this.openTelemetry = new PulsarBrokerOpenTelemetry(config, openTelemetrySdkBuilderCustomizer); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index b2d455f645daf..4d04dfeda7a74 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -66,8 +66,6 @@ import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.ScanOutcome; import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerOfflineBacklog; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; @@ -2622,7 +2620,7 @@ private void getEntryBatchSize(CompletableFuture batchSizeFuture, Persi MessageIdImpl messageId, int batchIndex) { if (batchIndex >= 0) { try { - ManagedLedgerImpl ledger = (ManagedLedgerImpl) topic.getManagedLedger(); + ManagedLedger ledger = topic.getManagedLedger(); ledger.asyncReadEntry(PositionFactory.create(messageId.getLedgerId(), messageId.getEntryId()), new AsyncCallbacks.ReadEntryCallback() { @Override @@ -2733,8 +2731,7 @@ protected CompletableFuture internalGetMessageById(long ledgerId, long .thenCompose(__ -> getTopicReferenceAsync(topicName)) .thenCompose(topic -> { CompletableFuture results = new CompletableFuture<>(); - ManagedLedgerImpl ledger = - (ManagedLedgerImpl) ((PersistentTopic) topic).getManagedLedger(); + ManagedLedger ledger = ((PersistentTopic) topic).getManagedLedger(); ledger.asyncReadEntry(PositionFactory.create(ledgerId, entryId), new AsyncCallbacks.ReadEntryCallback() { @Override @@ -3173,7 +3170,7 @@ protected CompletableFuture internalGetBacklogAsync try { PersistentOfflineTopicStats estimateOfflineTopicStats = offlineTopicBacklog.estimateUnloadedTopicBacklog( - (ManagedLedgerFactoryImpl) pulsar().getManagedLedgerFactory(), + pulsar().getManagedLedgerFactory(), topicName); pulsar().getBrokerService() .cacheOfflineTopicStats(topicName, estimateOfflineTopicStats); @@ -3248,8 +3245,7 @@ protected void internalGetBacklogSizeByMessageId(AsyncResponse asyncResponse, getTopicNotFoundErrorMessage(topicName.toString()))); return; } - ManagedLedgerImpl managedLedger = - (ManagedLedgerImpl) topic.getManagedLedger(); + ManagedLedger managedLedger = topic.getManagedLedger(); if (messageId.getLedgerId() == -1) { asyncResponse.resume(managedLedger.getTotalSize()); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 5a6df389aeddb..47c78fa9ee2ec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; +import static org.apache.bookkeeper.mledger.ManagedCursor.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELIMITER; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.HashBasedTable; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index f25dfef966bfd..9e5d6ef7191d1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -23,6 +23,7 @@ import static org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl.ENTRY_LATENCY_BUCKETS_USEC; import static org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION; import com.google.common.base.MoreObjects; +import java.time.Clock; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -164,11 +165,13 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener> entryFilters; protected volatile boolean transferring = false; private volatile List activeRateLimiters; + protected final Clock clock; protected Set additionalSystemCursorNames = new TreeSet<>(); public AbstractTopic(String topic, BrokerService brokerService) { this.topic = topic; + this.clock = brokerService.getClock(); this.brokerService = brokerService; this.producers = new ConcurrentHashMap<>(); this.isFenced = false; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java index 012cbcad1e26d..689e8514078c2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java @@ -27,10 +27,10 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo; +import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.resources.NamespaceResources; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -132,7 +132,7 @@ private void dropBacklogForSizeLimit(PersistentTopic persistentTopic, BacklogQuo // Get estimated unconsumed size for the managed ledger associated with this topic. Estimated size is more // useful than the actual storage size. Actual storage size gets updated only when managed ledger is trimmed. - ManagedLedgerImpl mLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + ManagedLedger mLedger = persistentTopic.getManagedLedger(); long backlogSize = mLedger.getEstimatedBacklogSize(); if (log.isDebugEnabled()) { @@ -214,29 +214,30 @@ private void dropBacklogForTimeLimit(PersistentTopic persistentTopic, BacklogQuo ); } else { // If disabled precise time based backlog quota check, will try to remove whole ledger from cursor's backlog - long currentMillis = ((ManagedLedgerImpl) persistentTopic.getManagedLedger()).getClock().millis(); - ManagedLedgerImpl mLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + long currentMillis = persistentTopic.getManagedLedger().getConfig().getClock().millis(); + ManagedLedger mLedger = persistentTopic.getManagedLedger(); try { for (; ; ) { ManagedCursor slowestConsumer = mLedger.getSlowestConsumer(); Position oldestPosition = slowestConsumer.getMarkDeletedPosition(); if (log.isDebugEnabled()) { log.debug("[{}] slowest consumer mark delete position is [{}], read position is [{}]", - slowestConsumer.getName(), oldestPosition, slowestConsumer.getReadPosition()); + slowestConsumer.getName(), oldestPosition, slowestConsumer.getReadPosition()); } - ManagedLedgerInfo.LedgerInfo ledgerInfo = mLedger.getLedgerInfo(oldestPosition.getLedgerId()).get(); + MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = + mLedger.getLedgerInfo(oldestPosition.getLedgerId()).get(); if (ledgerInfo == null) { - Position nextPosition = - PositionFactory.create(mLedger.getNextValidLedger(oldestPosition.getLedgerId()), -1); + long ledgerId = mLedger.getLedgersInfo().ceilingKey(oldestPosition.getLedgerId() + 1); + Position nextPosition = PositionFactory.create(ledgerId, -1); slowestConsumer.markDelete(nextPosition); continue; } // Timestamp only > 0 if ledger has been closed if (ledgerInfo.getTimestamp() > 0 - && currentMillis - ledgerInfo.getTimestamp() > SECONDS.toMillis(quota.getLimitTime())) { + && currentMillis - ledgerInfo.getTimestamp() > SECONDS.toMillis(quota.getLimitTime())) { // skip whole ledger for the slowest cursor - Position nextPosition = - PositionFactory.create(mLedger.getNextValidLedger(ledgerInfo.getLedgerId()), -1); + long ledgerId = mLedger.getLedgersInfo().ceilingKey(oldestPosition.getLedgerId() + 1); + Position nextPosition = PositionFactory.create(ledgerId, -1); if (!nextPosition.equals(oldestPosition)) { slowestConsumer.markDelete(nextPosition); continue; @@ -246,7 +247,7 @@ private void dropBacklogForTimeLimit(PersistentTopic persistentTopic, BacklogQuo } } catch (Exception e) { log.error("[{}] Error resetting cursor for slowest consumer [{}]", persistentTopic.getName(), - mLedger.getSlowestConsumer().getName(), e); + mLedger.getSlowestConsumer().getName(), e); } } } @@ -285,7 +286,7 @@ private void disconnectProducers(PersistentTopic persistentTopic) { */ private boolean advanceSlowestSystemCursor(PersistentTopic persistentTopic) { - ManagedLedgerImpl mLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + ManagedLedger mLedger = persistentTopic.getManagedLedger(); ManagedCursor slowestConsumer = mLedger.getSlowestConsumer(); if (slowestConsumer == null) { return false; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 17e5288b5f179..cb5e0853d53f3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -49,6 +49,7 @@ import java.lang.reflect.Field; import java.net.InetSocketAddress; import java.net.SocketAddress; +import java.time.Clock; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -288,6 +289,7 @@ public class BrokerService implements Closeable { private final int keepAliveIntervalSeconds; private final PulsarStats pulsarStats; private final AuthenticationService authenticationService; + private final Clock clock; public static final String MANAGED_LEDGER_PATH_ZNODE = "/managed-ledgers"; @@ -327,6 +329,7 @@ public class BrokerService implements Closeable { public BrokerService(PulsarService pulsar, EventLoopGroup eventLoopGroup) throws Exception { this.pulsar = pulsar; + this.clock = pulsar.getClock(); this.dynamicConfigurationMap = prepareDynamicConfigurationMap(); this.brokerPublishRateLimiter = new PublishRateLimiterImpl(pulsar.getMonotonicSnapshotClock()); this.preciseTopicPublishRateLimitingEnable = diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 0229b9c0f9788..7d196ad30235b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -65,11 +65,11 @@ import javax.net.ssl.SSLSession; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.pulsar.broker.PulsarServerException; @@ -2277,7 +2277,7 @@ private void getLargestBatchIndexWhenPossible( boolean readCompacted) { PersistentTopic persistentTopic = (PersistentTopic) topic; - ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + ManagedLedger ml = persistentTopic.getManagedLedger(); // If it's not pointing to a valid entry, respond messageId of the current position. // If the compaction cursor reach the end of the topic, respond messageId from compacted ledger @@ -2292,7 +2292,8 @@ private void getLargestBatchIndexWhenPossible( return; } - if (lastPosition.getEntryId() == -1 || !ml.ledgerExists(lastPosition.getLedgerId())) { + + if (lastPosition.getEntryId() == -1 || !ml.getLedgersInfo().containsKey(lastPosition.getLedgerId())) { // there is no entry in the original topic if (compactionHorizon != null) { // if readCompacted is true, we need to read the last entry from compacted topic @@ -2367,6 +2368,7 @@ public String toString() { }); }); } + private void handleLastMessageIdFromCompactionService(PersistentTopic persistentTopic, long requestId, int partitionIndex, Position markDeletePosition) { persistentTopic.getTopicCompactionService().readLastCompactedEntry().thenAccept(entry -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java index 9a8a39c8e9a12..3b4bc9d8bceb1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java @@ -28,13 +28,12 @@ import org.apache.bookkeeper.mledger.AsyncCallbacks.FindEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerException.LedgerNotExistException; import org.apache.bookkeeper.mledger.ManagedLedgerException.NonRecoverableLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.pulsar.broker.service.MessageExpirer; import org.apache.pulsar.client.impl.MessageImpl; @@ -113,27 +112,25 @@ private void checkExpiryByLedgerClosureTime(ManagedCursor cursor, int messageTTL if (messageTTLInSeconds <= 0) { return; } - if (cursor instanceof ManagedCursorImpl managedCursor) { - ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) managedCursor.getManagedLedger(); - Position deletedPosition = managedCursor.getMarkDeletedPosition(); - SortedMap ledgerInfoSortedMap = - managedLedger.getLedgersInfo().subMap(deletedPosition.getLedgerId(), true, - managedLedger.getLedgersInfo().lastKey(), true); - MLDataFormats.ManagedLedgerInfo.LedgerInfo info = null; - for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo : ledgerInfoSortedMap.values()) { - if (!ledgerInfo.hasTimestamp() || ledgerInfo.getTimestamp() == 0L - || !MessageImpl.isEntryExpired(messageTTLInSeconds, ledgerInfo.getTimestamp())) { - break; - } - info = ledgerInfo; + ManagedLedger managedLedger = cursor.getManagedLedger(); + Position deletedPosition = cursor.getMarkDeletedPosition(); + SortedMap ledgerInfoSortedMap = + managedLedger.getLedgersInfo().subMap(deletedPosition.getLedgerId(), true, + managedLedger.getLedgersInfo().lastKey(), true); + MLDataFormats.ManagedLedgerInfo.LedgerInfo info = null; + for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo : ledgerInfoSortedMap.values()) { + if (!ledgerInfo.hasTimestamp() || ledgerInfo.getTimestamp() == 0L + || !MessageImpl.isEntryExpired(messageTTLInSeconds, ledgerInfo.getTimestamp())) { + break; } - if (info != null && info.getLedgerId() > -1) { - Position position = PositionFactory.create(info.getLedgerId(), info.getEntries() - 1); - if (managedLedger.getLastConfirmedEntry().compareTo(position) < 0) { - findEntryComplete(managedLedger.getLastConfirmedEntry(), null); - } else { - findEntryComplete(position, null); - } + info = ledgerInfo; + } + if (info != null && info.getLedgerId() > -1) { + Position position = PositionFactory.create(info.getLedgerId(), info.getEntries() - 1); + if (managedLedger.getLastConfirmedEntry().compareTo(position) < 0) { + findEntryComplete(managedLedger.getLastConfirmedEntry(), null); + } else { + findEntryComplete(position, null); } } } @@ -240,11 +237,12 @@ public void findEntryFailed(ManagedLedgerException exception, Optional exception.getMessage()); if (exception instanceof LedgerNotExistException) { long failedLedgerId = failedReadPosition.get().getLedgerId(); - ManagedLedgerImpl ledger = ((ManagedLedgerImpl) cursor.getManagedLedger()); + ManagedLedger ledger = cursor.getManagedLedger(); Position lastPositionInLedger = ledger.getOptionalLedgerInfo(failedLedgerId) .map(ledgerInfo -> PositionFactory.create(failedLedgerId, ledgerInfo.getEntries() - 1)) .orElseGet(() -> { - Long nextExistingLedger = ledger.getNextValidLedger(failedReadPosition.get().getLedgerId()); + Long nextExistingLedger = + ledger.getLedgersInfo().ceilingKey(failedReadPosition.get().getLedgerId() + 1); if (nextExistingLedger == null) { log.info("[{}] [{}] Couldn't find next next valid ledger for expiry monitor when find " + "entry failed {}", ledger.getName(), ledger.getName(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 97e6c943b0baa..e8e4919a9be52 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -37,10 +37,10 @@ import javax.annotation.Nullable; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.collections4.MapUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerServiceException; @@ -323,7 +323,7 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis } if (messagesForC > 0) { - final ManagedLedgerImpl managedLedger = ((ManagedLedgerImpl) cursor.getManagedLedger()); + final ManagedLedger managedLedger = cursor.getManagedLedger(); for (int i = 0; i < messagesForC; i++) { final Entry entry = entriesWithSameKey.get(i); // remove positions first from replay list first : sendMessages recycles entries @@ -368,7 +368,7 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis // Update the last sent position and remove ranges from individuallySentPositions if necessary if (!allowOutOfOrderDelivery && lastSentPosition != null) { - final ManagedLedgerImpl managedLedger = ((ManagedLedgerImpl) cursor.getManagedLedger()); + final ManagedLedger managedLedger = cursor.getManagedLedger(); com.google.common.collect.Range range = individuallySentPositions.firstRange(); // If the upper bound is before the last sent position, we need to move ahead as these diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index ea1b7d7602be7..9a0545e6f0ab2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -23,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import io.netty.buffer.ByteBuf; +import java.io.IOException; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -51,8 +52,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.InvalidCursorPositionException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.ScanOutcome; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.ServiceConfiguration; @@ -405,7 +404,7 @@ public void acknowledgeMessage(List positions, AckType ackType, Map { - if (((ManagedCursorImpl) cursor).isMessageDeleted(position)) { + if ((cursor.isMessageDeleted(position))) { pendingAckHandle.clearIndividualPosition(position); } }); @@ -552,7 +551,7 @@ public CompletableFuture analyzeBacklog(Optional final String newNonDurableCursorName = "analyze-backlog-" + UUID.randomUUID(); ManagedCursor newNonDurableCursor; try { - newNonDurableCursor = ((ManagedCursorImpl) cursor).duplicateNonDurableCursor(newNonDurableCursorName); + newNonDurableCursor = cursor.duplicateNonDurableCursor(newNonDurableCursorName); } catch (ManagedLedgerException e) { return CompletableFuture.failedFuture(e); } @@ -1281,7 +1280,7 @@ public CompletableFuture getStatsAsync(GetStatsOptions ge } subStats.msgBacklog = getNumberOfEntriesInBacklog(getStatsOptions.isGetPreciseBacklog()); if (getStatsOptions.isSubscriptionBacklogSize()) { - subStats.backlogSize = ((ManagedLedgerImpl) topic.getManagedLedger()) + subStats.backlogSize = topic.getManagedLedger() .getEstimatedBacklogSize(cursor.getMarkDeletedPosition()); } else { subStats.backlogSize = -1; @@ -1331,9 +1330,9 @@ public CompletableFuture getStatsAsync(GetStatsOptions ge return CompletableFuture.completedFuture(subStats); } if (subStats.msgBacklog > 0) { - ManagedLedgerImpl managedLedger = ((ManagedLedgerImpl) cursor.getManagedLedger()); + ManagedLedger managedLedger = cursor.getManagedLedger(); Position markDeletedPosition = cursor.getMarkDeletedPosition(); - return managedLedger.getEarliestMessagePublishTimeOfPos(markDeletedPosition).thenApply(v -> { + return getEarliestMessagePublishTimeOfPos(managedLedger, markDeletedPosition).thenApply(v -> { subStats.earliestMsgPublishTimeInBacklog = v; return subStats; }); @@ -1343,6 +1342,48 @@ public CompletableFuture getStatsAsync(GetStatsOptions ge } } + private CompletableFuture getEarliestMessagePublishTimeOfPos(ManagedLedger ml, Position pos) { + CompletableFuture future = new CompletableFuture<>(); + if (pos == null) { + future.complete(0L); + return future; + } + Position nextPos = ml.getNextValidPosition(pos); + + if (nextPos.compareTo(ml.getLastConfirmedEntry()) > 0) { + return CompletableFuture.completedFuture(-1L); + } + + ml.asyncReadEntry(nextPos, new ReadEntryCallback() { + @Override + public void readEntryComplete(Entry entry, Object ctx) { + try { + long entryTimestamp = Commands.getEntryTimestamp(entry.getDataBuffer()); + future.complete(entryTimestamp); + } catch (IOException e) { + log.error("Error deserializing message for message position {}", nextPos, e); + future.completeExceptionally(e); + } finally { + entry.release(); + } + } + + @Override + public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + log.error("Error read entry for position {}", nextPos, exception); + future.completeExceptionally(exception); + } + + @Override + public String toString() { + return String.format("ML [%s] get earliest message publish time of pos", + ml.getName()); + } + }, null); + + return future; + } + @Override public void redeliverUnacknowledgedMessages(Consumer consumer, long consumerEpoch) { Dispatcher dispatcher = getDispatcher(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index f90e10ee64e65..fc47889c60aac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -54,7 +54,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.function.BiFunction; -import java.util.stream.Collectors; import javax.annotation.Nonnull; import lombok.Getter; import lombok.Value; @@ -80,14 +79,11 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException; import org.apache.bookkeeper.mledger.ManagedLedgerException.NonRecoverableLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer; import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer.CursorInfo; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.util.Futures; -import org.apache.bookkeeper.mledger.util.ManagedLedgerImplUtils; -import org.apache.bookkeeper.net.BookieId; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.mutable.MutableInt; @@ -189,7 +185,6 @@ import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.topics.TopicCompactionStrategy; import org.apache.pulsar.common.util.Codec; -import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.compaction.CompactedTopicContext; @@ -689,32 +684,15 @@ private void asyncAddEntry(ByteBuf headersAndPayload, PublishContext publishCont } public void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx) { - if (ledger instanceof ManagedLedgerImpl) { - ((ManagedLedgerImpl) ledger).asyncReadEntry(position, callback, ctx); - } else { - callback.readEntryFailed(new ManagedLedgerException( - "Unexpected managedledger implementation, doesn't support " - + "direct read entry operation."), ctx); - } + ledger.asyncReadEntry(position, callback, ctx); } public Position getPositionAfterN(Position startPosition, long n) throws ManagedLedgerException { - if (ledger instanceof ManagedLedgerImpl) { - return ((ManagedLedgerImpl) ledger).getPositionAfterN(startPosition, n, - ManagedLedgerImpl.PositionBound.startExcluded); - } else { - throw new ManagedLedgerException("Unexpected managedledger implementation, doesn't support " - + "getPositionAfterN operation."); - } + return ledger.getPositionAfterN(startPosition, n, PositionBound.startExcluded); } public Position getFirstPosition() throws ManagedLedgerException { - if (ledger instanceof ManagedLedgerImpl) { - return ((ManagedLedgerImpl) ledger).getFirstPosition(); - } else { - throw new ManagedLedgerException("Unexpected managedledger implementation, doesn't support " - + "getFirstPosition operation."); - } + return ledger.getFirstPosition(); } public long getNumberOfEntries() { @@ -2545,7 +2523,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats topicStatsStream.writePair("msgThroughputOut", topicStatsHelper.aggMsgThroughputOut); topicStatsStream.writePair("storageSize", ledger.getTotalSize()); topicStatsStream.writePair("backlogSize", ledger.getEstimatedBacklogSize()); - topicStatsStream.writePair("pendingAddEntriesCount", ((ManagedLedgerImpl) ledger).getPendingAddEntriesCount()); + topicStatsStream.writePair("pendingAddEntriesCount", ledger.getPendingAddEntriesCount()); topicStatsStream.writePair("filteredEntriesCount", getFilteredEntriesCount()); nsStats.msgRateIn += topicStatsHelper.aggMsgRateIn; @@ -2558,7 +2536,7 @@ public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats bundleStats.msgRateOut += topicStatsHelper.aggMsgRateOut; bundleStats.msgThroughputIn += topicStatsHelper.aggMsgThroughputIn; bundleStats.msgThroughputOut += topicStatsHelper.aggMsgThroughputOut; - bundleStats.cacheSize += ((ManagedLedgerImpl) ledger).getCacheSize(); + bundleStats.cacheSize += ledger.getCacheSize(); // Close topic object topicStatsStream.endObject(); @@ -2756,194 +2734,158 @@ public CompletableFuture deleteSchema() { public CompletableFuture getInternalStats(boolean includeLedgerMetadata) { CompletableFuture statFuture = new CompletableFuture<>(); - PersistentTopicInternalStats stats = new PersistentTopicInternalStats(); - - ManagedLedgerImpl ml = (ManagedLedgerImpl) ledger; - stats.entriesAddedCounter = ml.getEntriesAddedCounter(); - stats.numberOfEntries = ml.getNumberOfEntries(); - stats.totalSize = ml.getTotalSize(); - stats.currentLedgerEntries = ml.getCurrentLedgerEntries(); - stats.currentLedgerSize = ml.getCurrentLedgerSize(); - stats.lastLedgerCreatedTimestamp = DateFormatter.format(ml.getLastLedgerCreatedTimestamp()); - if (ml.getLastLedgerCreationFailureTimestamp() != 0) { - stats.lastLedgerCreationFailureTimestamp = DateFormatter.format(ml.getLastLedgerCreationFailureTimestamp()); - } - - stats.waitingCursorsCount = ml.getWaitingCursorsCount(); - stats.pendingAddEntriesCount = ml.getPendingAddEntriesCount(); - - stats.lastConfirmedEntry = ml.getLastConfirmedEntry().toString(); - stats.state = ml.getState().toString(); - - stats.ledgers = new ArrayList<>(); - Set> futures = Sets.newConcurrentHashSet(); - CompletableFuture> availableBookiesFuture = - brokerService.pulsar().getPulsarResources().getBookieResources().listAvailableBookiesAsync(); - futures.add( - availableBookiesFuture - .whenComplete((bookies, e) -> { - if (e != null) { - log.error("[{}] Failed to fetch available bookies.", topic, e); - statFuture.completeExceptionally(e); - } else { - ml.getLedgersInfo().forEach((id, li) -> { - LedgerInfo info = new LedgerInfo(); - info.ledgerId = li.getLedgerId(); - info.entries = li.getEntries(); - info.size = li.getSize(); - info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete(); - stats.ledgers.add(info); - if (includeLedgerMetadata) { - futures.add(ml.getLedgerMetadata(li.getLedgerId()).handle((lMetadata, ex) -> { - if (ex == null) { - info.metadata = lMetadata; - } - return null; - })); - futures.add(ml.getEnsemblesAsync(li.getLedgerId()).handle((ensembles, ex) -> { - if (ex == null) { - info.underReplicated = - !bookies.containsAll(ensembles.stream().map(BookieId::toString) - .collect(Collectors.toList())); - } - return null; - })); - } - }); + + ledger.getManagedLedgerInternalStats(includeLedgerMetadata) + .thenCombine(getCompactedTopicContextAsync(), (ledgerInternalStats, compactedTopicContext) -> { + PersistentTopicInternalStats stats = new PersistentTopicInternalStats(); + stats.entriesAddedCounter = ledgerInternalStats.getEntriesAddedCounter(); + stats.numberOfEntries = ledgerInternalStats.getNumberOfEntries(); + stats.totalSize = ledgerInternalStats.getTotalSize(); + stats.currentLedgerEntries = ledgerInternalStats.getCurrentLedgerEntries(); + stats.currentLedgerSize = ledgerInternalStats.getCurrentLedgerSize(); + stats.lastLedgerCreatedTimestamp = ledgerInternalStats.getLastLedgerCreatedTimestamp(); + stats.lastLedgerCreationFailureTimestamp = ledgerInternalStats.getLastLedgerCreationFailureTimestamp(); + stats.waitingCursorsCount = ledgerInternalStats.getWaitingCursorsCount(); + stats.pendingAddEntriesCount = ledgerInternalStats.getPendingAddEntriesCount(); + stats.lastConfirmedEntry = ledgerInternalStats.getLastConfirmedEntry(); + stats.state = ledgerInternalStats.getState(); + stats.ledgers = ledgerInternalStats.ledgers; + + // Add ledger info for compacted topic ledger if exist. + LedgerInfo info = new LedgerInfo(); + info.ledgerId = -1; + info.entries = -1; + info.size = -1; + if (compactedTopicContext != null) { + info.ledgerId = compactedTopicContext.getLedger().getId(); + info.entries = compactedTopicContext.getLedger().getLastAddConfirmed() + 1; + info.size = compactedTopicContext.getLedger().getLength(); + } + + stats.compactedLedger = info; + + stats.cursors = new HashMap<>(); + ledger.getCursors().forEach(c -> { + CursorStats cs = new CursorStats(); + + CursorStats cursorInternalStats = c.getCursorStats(); + cs.markDeletePosition = cursorInternalStats.getMarkDeletePosition(); + cs.readPosition = cursorInternalStats.getReadPosition(); + cs.waitingReadOp = cursorInternalStats.isWaitingReadOp(); + cs.pendingReadOps = cursorInternalStats.getPendingReadOps(); + cs.messagesConsumedCounter = cursorInternalStats.getMessagesConsumedCounter(); + cs.cursorLedger = cursorInternalStats.getCursorLedger(); + cs.cursorLedgerLastEntry = cursorInternalStats.getCursorLedgerLastEntry(); + cs.individuallyDeletedMessages = cursorInternalStats.getIndividuallyDeletedMessages(); + cs.lastLedgerSwitchTimestamp = cursorInternalStats.getLastLedgerSwitchTimestamp(); + cs.state = cursorInternalStats.getState(); + cs.active = cursorInternalStats.isActive(); + cs.numberOfEntriesSinceFirstNotAckedMessage = + cursorInternalStats.getNumberOfEntriesSinceFirstNotAckedMessage(); + cs.totalNonContiguousDeletedMessagesRange = + cursorInternalStats.getTotalNonContiguousDeletedMessagesRange(); + cs.properties = cursorInternalStats.getProperties(); + // subscription metrics + PersistentSubscription sub = subscriptions.get(Codec.decode(c.getName())); + if (sub != null) { + if (sub.getDispatcher() instanceof PersistentDispatcherMultipleConsumers) { + PersistentDispatcherMultipleConsumers dispatcher = + (PersistentDispatcherMultipleConsumers) sub.getDispatcher(); + cs.subscriptionHavePendingRead = dispatcher.havePendingRead; + cs.subscriptionHavePendingReplayRead = dispatcher.havePendingReplayRead; + } else if (sub.getDispatcher() instanceof PersistentDispatcherSingleActiveConsumer) { + PersistentDispatcherSingleActiveConsumer dispatcher = + (PersistentDispatcherSingleActiveConsumer) sub.getDispatcher(); + cs.subscriptionHavePendingRead = dispatcher.havePendingRead; + } } - }) - ); - - // Add ledger info for compacted topic ledger if exist. - LedgerInfo info = new LedgerInfo(); - info.ledgerId = -1; - info.entries = -1; - info.size = -1; - - futures.add(getCompactedTopicContextAsync().thenAccept(v -> { - if (v != null) { - info.ledgerId = v.getLedger().getId(); - info.entries = v.getLedger().getLastAddConfirmed() + 1; - info.size = v.getLedger().getLength(); - } - })); + stats.cursors.put(c.getName(), cs); + }); - stats.compactedLedger = info; - - stats.cursors = new HashMap<>(); - ml.getCursors().forEach(c -> { - ManagedCursorImpl cursor = (ManagedCursorImpl) c; - CursorStats cs = new CursorStats(); - cs.markDeletePosition = cursor.getMarkDeletedPosition().toString(); - cs.readPosition = cursor.getReadPosition().toString(); - cs.waitingReadOp = cursor.hasPendingReadRequest(); - cs.pendingReadOps = cursor.getPendingReadOpsCount(); - cs.messagesConsumedCounter = cursor.getMessagesConsumedCounter(); - cs.cursorLedger = cursor.getCursorLedger(); - cs.cursorLedgerLastEntry = cursor.getCursorLedgerLastEntry(); - cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages(); - cs.lastLedgerSwitchTimestamp = DateFormatter.format(cursor.getLastLedgerSwitchTimestamp()); - cs.state = cursor.getState(); - cs.active = cursor.isActive(); - cs.numberOfEntriesSinceFirstNotAckedMessage = cursor.getNumberOfEntriesSinceFirstNotAckedMessage(); - cs.totalNonContiguousDeletedMessagesRange = cursor.getTotalNonContiguousDeletedMessagesRange(); - cs.properties = cursor.getProperties(); - // subscription metrics - PersistentSubscription sub = subscriptions.get(Codec.decode(c.getName())); - if (sub != null) { - if (sub.getDispatcher() instanceof PersistentDispatcherMultipleConsumers) { - PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) sub - .getDispatcher(); - cs.subscriptionHavePendingRead = dispatcher.havePendingRead; - cs.subscriptionHavePendingReplayRead = dispatcher.havePendingReplayRead; - } else if (sub.getDispatcher() instanceof PersistentDispatcherSingleActiveConsumer) { - PersistentDispatcherSingleActiveConsumer dispatcher = (PersistentDispatcherSingleActiveConsumer) sub - .getDispatcher(); - cs.subscriptionHavePendingRead = dispatcher.havePendingRead; + //Schema store ledgers + String schemaId; + try { + schemaId = TopicName.get(topic).getSchemaName(); + } catch (Throwable t) { + statFuture.completeExceptionally(t); + return null; } - } - stats.cursors.put(cursor.getName(), cs); - }); - //Schema store ledgers - String schemaId; - try { - schemaId = TopicName.get(topic).getSchemaName(); - } catch (Throwable t) { - statFuture.completeExceptionally(t); - return statFuture; - } - - - CompletableFuture schemaStoreLedgersFuture = new CompletableFuture<>(); - stats.schemaLedgers = Collections.synchronizedList(new ArrayList<>()); - if (brokerService.getPulsar().getSchemaStorage() != null - && brokerService.getPulsar().getSchemaStorage() instanceof BookkeeperSchemaStorage) { - ((BookkeeperSchemaStorage) brokerService.getPulsar().getSchemaStorage()) - .getStoreLedgerIdsBySchemaId(schemaId) - .thenAccept(ledgers -> { - List> getLedgerMetadataFutures = new ArrayList<>(); - ledgers.forEach(ledgerId -> { - CompletableFuture completableFuture = new CompletableFuture<>(); - getLedgerMetadataFutures.add(completableFuture); - CompletableFuture metadataFuture = null; - try { - metadataFuture = brokerService.getPulsar().getBookKeeperClient() - .getLedgerMetadata(ledgerId); - } catch (NullPointerException e) { - // related to bookkeeper issue https://github.com/apache/bookkeeper/issues/2741 - if (log.isDebugEnabled()) { - log.debug("{{}} Failed to get ledger metadata for the schema ledger {}", + + CompletableFuture schemaStoreLedgersFuture = new CompletableFuture<>(); + stats.schemaLedgers = Collections.synchronizedList(new ArrayList<>()); + if (brokerService.getPulsar().getSchemaStorage() != null + && brokerService.getPulsar().getSchemaStorage() instanceof BookkeeperSchemaStorage) { + ((BookkeeperSchemaStorage) brokerService.getPulsar().getSchemaStorage()) + .getStoreLedgerIdsBySchemaId(schemaId) + .thenAccept(ledgers -> { + List> getLedgerMetadataFutures = new ArrayList<>(); + ledgers.forEach(ledgerId -> { + CompletableFuture completableFuture = new CompletableFuture<>(); + getLedgerMetadataFutures.add(completableFuture); + CompletableFuture metadataFuture = null; + try { + metadataFuture = brokerService.getPulsar().getBookKeeperClient() + .getLedgerMetadata(ledgerId); + } catch (NullPointerException e) { + // related to bookkeeper issue https://github.com/apache/bookkeeper/issues/2741 + if (log.isDebugEnabled()) { + log.debug("{{}} Failed to get ledger metadata for the schema ledger {}", topic, ledgerId, e); - } - } - if (metadataFuture != null) { - metadataFuture.thenAccept(metadata -> { - LedgerInfo schemaLedgerInfo = new LedgerInfo(); - schemaLedgerInfo.ledgerId = metadata.getLedgerId(); - schemaLedgerInfo.entries = metadata.getLastEntryId() + 1; - schemaLedgerInfo.size = metadata.getLength(); - if (includeLedgerMetadata) { - info.metadata = metadata.toSafeString(); } - stats.schemaLedgers.add(schemaLedgerInfo); - completableFuture.complete(null); - }).exceptionally(e -> { - log.error("[{}] Failed to get ledger metadata for the schema ledger {}", + } + if (metadataFuture != null) { + metadataFuture.thenAccept(metadata -> { + LedgerInfo schemaLedgerInfo = new LedgerInfo(); + schemaLedgerInfo.ledgerId = metadata.getLedgerId(); + schemaLedgerInfo.entries = metadata.getLastEntryId() + 1; + schemaLedgerInfo.size = metadata.getLength(); + if (includeLedgerMetadata) { + info.metadata = metadata.toSafeString(); + } + stats.schemaLedgers.add(schemaLedgerInfo); + completableFuture.complete(null); + }).exceptionally(e -> { + log.error("[{}] Failed to get ledger metadata for the schema ledger {}", topic, ledgerId, e); - if ((e.getCause() instanceof BKNoSuchLedgerExistsOnMetadataServerException) + if ((e.getCause() instanceof BKNoSuchLedgerExistsOnMetadataServerException) || (e.getCause() instanceof BKNoSuchLedgerExistsException)) { - completableFuture.complete(null); + completableFuture.complete(null); + return null; + } + completableFuture.completeExceptionally(e); return null; - } - completableFuture.completeExceptionally(e); - return null; - }); - } else { - completableFuture.complete(null); - } - }); - FutureUtil.waitForAll(getLedgerMetadataFutures).thenRun(() -> { - schemaStoreLedgersFuture.complete(null); + }); + } else { + completableFuture.complete(null); + } + }); + FutureUtil.waitForAll(getLedgerMetadataFutures).thenRun(() -> { + schemaStoreLedgersFuture.complete(null); + }).exceptionally(e -> { + schemaStoreLedgersFuture.completeExceptionally(e); + return null; + }); }).exceptionally(e -> { schemaStoreLedgersFuture.completeExceptionally(e); return null; }); - }).exceptionally(e -> { - schemaStoreLedgersFuture.completeExceptionally(e); + } else { + schemaStoreLedgersFuture.complete(null); + } + schemaStoreLedgersFuture.whenComplete((r, ex) -> { + if (ex != null) { + statFuture.completeExceptionally(ex); + } else { + statFuture.complete(stats); + } + }); return null; - }); - } else { - schemaStoreLedgersFuture.complete(null); - } - schemaStoreLedgersFuture.thenRun(() -> - FutureUtil.waitForAll(futures).handle((res, ex) -> { - statFuture.complete(stats); + }) + .exceptionally(ex -> { + statFuture.completeExceptionally(ex); return null; - })).exceptionally(e -> { - statFuture.completeExceptionally(e); - return null; - }); + }); return statFuture; } @@ -3638,8 +3580,8 @@ public CompletableFuture checkTimeBacklogExceeded() { CompletableFuture future = new CompletableFuture<>(); // Check if first unconsumed message(first message after mark delete position) // for slowest cursor's has expired. - Position position = ((ManagedLedgerImpl) ledger).getNextValidPosition(oldestMarkDeletePosition); - ((ManagedLedgerImpl) ledger).asyncReadEntry(position, + Position position = ledger.getNextValidPosition(oldestMarkDeletePosition); + ledger.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { @Override public void readEntryComplete(Entry entry, Object ctx) { @@ -3714,11 +3656,10 @@ private EstimateTimeBasedBacklogQuotaCheckResult estimatedTimeBasedBacklogQuotaC Position markDeletePosition) throws ExecutionException, InterruptedException { int backlogQuotaLimitInSecond = getBacklogQuota(BacklogQuotaType.message_age).getLimitTime(); - ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) ledger; // The ledger timestamp is only known when ledger is closed, hence when the mark-delete // is at active ledger (open) we can't estimate it. - if (managedLedger.getLedgersInfo().lastKey().equals(markDeletePosition.getLedgerId())) { + if (ledger.getLedgersInfo().lastKey().equals(markDeletePosition.getLedgerId())) { return new EstimateTimeBasedBacklogQuotaCheckResult(false, null); } @@ -3731,14 +3672,14 @@ private EstimateTimeBasedBacklogQuotaCheckResult estimatedTimeBasedBacklogQuotaC // if the mark-delete position is the last entry it means all entries for // that ledger are acknowledged if (markDeletePosition.getEntryId() == markDeletePositionLedgerInfo.getEntries() - 1) { - Position positionToCheck = managedLedger.getNextValidPosition(markDeletePosition); + Position positionToCheck = ledger.getNextValidPosition(markDeletePosition); positionToCheckLedgerInfo = ledger.getLedgerInfo(positionToCheck.getLedgerId()).get(); } if (positionToCheckLedgerInfo != null && positionToCheckLedgerInfo.hasTimestamp() && positionToCheckLedgerInfo.getTimestamp() > 0) { - long estimateMsgAgeMs = managedLedger.getClock().millis() - positionToCheckLedgerInfo.getTimestamp(); + long estimateMsgAgeMs = clock.millis() - positionToCheckLedgerInfo.getTimestamp(); boolean shouldTruncateBacklog = estimateMsgAgeMs > SECONDS.toMillis(backlogQuotaLimitInSecond); if (log.isDebugEnabled()) { log.debug("Time based backlog quota exceeded, quota {}[ms], age of ledger " @@ -3890,24 +3831,22 @@ public CompletableFuture getLastDispatchablePosition() { if (lastDispatchablePosition != null) { return CompletableFuture.completedFuture(lastDispatchablePosition); } - return ManagedLedgerImplUtils - .asyncGetLastValidPosition((ManagedLedgerImpl) ledger, entry -> { - MessageMetadata md = Commands.parseMessageMetadata(entry.getDataBuffer()); - // If a messages has marker will filter by AbstractBaseDispatcher.filterEntriesForConsumer - if (Markers.isServerOnlyMarker(md)) { - return false; - } else if (md.hasTxnidMostBits() && md.hasTxnidLeastBits()) { - // Filter-out transaction aborted messages. - TxnID txnID = new TxnID(md.getTxnidMostBits(), md.getTxnidLeastBits()); - return !isTxnAborted(txnID, entry.getPosition()); - } - return true; - }, getMaxReadPosition()) - .thenApply(position -> { - // Update lastDispatchablePosition to the given position - updateLastDispatchablePosition(position); - return position; - }); + return ledger.getLastDispatchablePosition(entry -> { + MessageMetadata md = Commands.parseMessageMetadata(entry.getDataBuffer()); + // If a messages has marker will filter by AbstractBaseDispatcher.filterEntriesForConsumer + if (Markers.isServerOnlyMarker(md)) { + return false; + } else if (md.hasTxnidMostBits() && md.hasTxnidLeastBits()) { + // Filter-out transaction aborted messages. + TxnID txnID = new TxnID(md.getTxnidMostBits(), md.getTxnidLeastBits()); + return !isTxnAborted(txnID, entry.getPosition()); + } + return true; + }, getMaxReadPosition()).thenApply(position -> { + // Update lastDispatchablePosition to the given position + updateLastDispatchablePosition(position); + return position; + }); } /** @@ -3952,13 +3891,13 @@ public CompletableFuture getLastMessageId() { .complete(new MessageIdImpl(position.getLedgerId(), position.getEntryId(), partitionIndex)); return completableFuture; } - ManagedLedgerImpl ledgerImpl = (ManagedLedgerImpl) ledger; - if (!ledgerImpl.ledgerExists(position.getLedgerId())) { + + if (!ledger.getLedgersInfo().containsKey(position.getLedgerId())) { completableFuture .complete(MessageId.earliest); return completableFuture; } - ledgerImpl.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { + ledger.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { @Override public void readEntryComplete(Entry entry, Object ctx) { try { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/AbstractMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/AbstractMetrics.java index f87abcf495308..489d37dd0a307 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/AbstractMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/AbstractMetrics.java @@ -24,9 +24,8 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerFactoryMXBean; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.common.policies.data.TopicStats; @@ -132,7 +131,7 @@ protected Metrics createMetrics(Map dimensionMap) { * @return */ protected ManagedLedgerFactoryMXBean getManagedLedgerCacheStats() { - return ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getCacheStats(); + return pulsar.getManagedLedgerFactory().getCacheStats(); } /** @@ -140,8 +139,8 @@ protected ManagedLedgerFactoryMXBean getManagedLedgerCacheStats() { * * @return */ - protected Map getManagedLedgers() { - return ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getManagedLedgers(); + protected Map getManagedLedgers() { + return pulsar.getManagedLedgerFactory().getManagedLedgers(); } protected String getLocalClusterName() { @@ -235,8 +234,8 @@ protected void populateMaxMap(Map map, String mkey, long value) { * @param metrics * @param ledger */ - protected void populateDimensionMap(Map> ledgersByDimensionMap, Metrics metrics, - ManagedLedgerImpl ledger) { + protected void populateDimensionMap(Map> ledgersByDimensionMap, Metrics metrics, + ManagedLedger ledger) { ledgersByDimensionMap.computeIfAbsent(metrics, __ -> new ArrayList<>()).add(ledger); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java index 424a7cb2f81ac..639f51ead6cee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedCursorMetrics.java @@ -25,9 +25,7 @@ import java.util.Map; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedCursorMXBean; -import org.apache.bookkeeper.mledger.impl.ManagedCursorContainer; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.common.stats.Metrics; @@ -55,16 +53,15 @@ public synchronized List generate() { */ private List aggregate() { metricsCollection.clear(); - for (Map.Entry e : getManagedLedgers().entrySet()) { + for (Map.Entry e : getManagedLedgers().entrySet()) { String ledgerName = e.getKey(); - ManagedLedgerImpl ledger = e.getValue(); + ManagedLedger ledger = e.getValue(); String namespace = parseNamespaceFromLedgerName(ledgerName); - ManagedCursorContainer cursorContainer = ledger.getCursors(); - Iterator cursorIterator = cursorContainer.iterator(); + Iterator cursorIterator = ledger.getCursors().iterator(); while (cursorIterator.hasNext()) { - ManagedCursorImpl cursor = (ManagedCursorImpl) cursorIterator.next(); + ManagedCursor cursor = cursorIterator.next(); ManagedCursorMXBean cStats = cursor.getStats(); dimensionMap.clear(); dimensionMap.put("namespace", namespace); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java index 36004bc1281bb..52c69265c2f1f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/metrics/ManagedLedgerMetrics.java @@ -23,16 +23,15 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerMXBean; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.common.stats.Metrics; public class ManagedLedgerMetrics extends AbstractMetrics { private List metricsCollection; - private Map> ledgersByDimensionMap; + private Map> ledgersByDimensionMap; // temp map to prepare aggregation metrics private Map tempAggregatedMetricsMap; private static final Buckets @@ -53,7 +52,7 @@ public ManagedLedgerMetrics(PulsarService pulsar) { this.metricsCollection = new ArrayList<>(); this.ledgersByDimensionMap = new HashMap<>(); this.tempAggregatedMetricsMap = new HashMap<>(); - this.statsPeriodSeconds = ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()) + this.statsPeriodSeconds = pulsar.getManagedLedgerFactory() .getConfig().getStatsPeriodSeconds(); } @@ -71,20 +70,20 @@ public synchronized List generate() { * @param ledgersByDimension * @return */ - private List aggregate(Map> ledgersByDimension) { + private List aggregate(Map> ledgersByDimension) { metricsCollection.clear(); - for (Entry> e : ledgersByDimension.entrySet()) { + for (Entry> e : ledgersByDimension.entrySet()) { Metrics metrics = e.getKey(); - List ledgers = e.getValue(); + List ledgers = e.getValue(); // prepare aggregation map tempAggregatedMetricsMap.clear(); // generate the collections by each metrics and then apply the aggregation - for (ManagedLedgerImpl ledger : ledgers) { + for (ManagedLedger ledger : ledgers) { ManagedLedgerMXBean lStats = ledger.getStats(); populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntryBytesRate", @@ -151,17 +150,17 @@ private List aggregate(Map> ledgersByD * * @return */ - private Map> groupLedgersByDimension() { + private Map> groupLedgersByDimension() { ledgersByDimensionMap.clear(); // get the current topics statistics from StatsBrokerFilter // Map : topic-name->dest-stat - for (Entry e : getManagedLedgers().entrySet()) { + for (Entry e : getManagedLedgers().entrySet()) { String ledgerName = e.getKey(); - ManagedLedgerImpl ledger = e.getValue(); + ManagedLedger ledger = e.getValue(); // we want to aggregate by NS dimension diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java index 1649349e3e6f6..a0ffa121b8999 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SingleSnapshotAbortedTxnProcessorImpl.java @@ -24,7 +24,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.collections4.map.LinkedMap; import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -69,8 +68,8 @@ public void putAbortedTxnAndPosition(TxnID abortedTxnId, Position abortedMarkerP //In this implementation we clear the invalid aborted txn ID one by one. @Override public void trimExpiredAbortedTxns() { - while (!aborts.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) - .ledgerExists(aborts.get(aborts.firstKey()).getLedgerId())) { + while (!aborts.isEmpty() && !topic.getManagedLedger().getLedgersInfo() + .containsKey(aborts.get(aborts.firstKey()).getLedgerId())) { if (log.isDebugEnabled()) { log.debug("[{}] Topic transaction buffer clear aborted transaction, TxnId : {}, Position : {}", topic.getName(), aborts.firstKey(), aborts.get(aborts.firstKey())); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 4ca27f77a87f5..88a3968b7b430 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -38,8 +38,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; +import org.apache.bookkeeper.mledger.ReadOnlyManagedLedger; import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -189,8 +188,8 @@ public boolean checkAbortedTransaction(TxnID txnID) { public void trimExpiredAbortedTxns() { //Checking whether there are some segment expired. List positionsNeedToDelete = new ArrayList<>(); - while (!segmentIndex.isEmpty() && !((ManagedLedgerImpl) topic.getManagedLedger()) - .ledgerExists(segmentIndex.firstKey().getLedgerId())) { + while (!segmentIndex.isEmpty() && !topic.getManagedLedger().getLedgersInfo() + .containsKey(segmentIndex.firstKey().getLedgerId())) { if (log.isDebugEnabled()) { log.debug("[{}] Topic transaction buffer clear aborted transactions, maxReadPosition : {}", topic.getName(), segmentIndex.firstKey()); @@ -275,8 +274,8 @@ private void readSegmentEntries(TopicName topicName, TransactionBufferSnapshotIn entry.release(); } } catch (Throwable throwable) { - if (((ManagedLedgerImpl) topic.getManagedLedger()) - .ledgerExists(index.getAbortedMarkLedgerID())) { + if (topic.getManagedLedger().getLedgersInfo() + .containsKey(index.getAbortedMarkLedgerID())) { log.error("[{}] Failed to read snapshot segment [{}:{}]", topic.getName(), index.segmentLedgerID, index.segmentEntryID, throwable); @@ -293,11 +292,11 @@ private void readSegmentEntries(TopicName topicName, TransactionBufferSnapshotIn } } - private ReadOnlyManagedLedgerImpl openReadOnlyManagedLedger(TopicName topicName) throws Exception { - final var future = new CompletableFuture(); + private ReadOnlyManagedLedger openReadOnlyManagedLedger(TopicName topicName) throws Exception { + final var future = new CompletableFuture(); final var callback = new AsyncCallbacks.OpenReadOnlyManagedLedgerCallback() { @Override - public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl managedLedger, Object ctx) { + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger managedLedger, Object ctx) { future.complete(managedLedger); } @@ -317,7 +316,7 @@ public String toString() { return wait(future, "open read only ml for " + topicName); } - private Entry readEntry(ReadOnlyManagedLedgerImpl managedLedger, Position position) throws Exception { + private Entry readEntry(ReadOnlyManagedLedger managedLedger, Position position) throws Exception { final var future = new CompletableFuture(); managedLedger.asyncReadEntry(position, new AsyncCallbacks.ReadEntryCallback() { @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 2f90ff8922a81..41977e6b61d88 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -38,7 +38,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.collections4.map.LinkedMap; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; @@ -320,7 +319,7 @@ private void handleTransactionMessage(TxnID txnId, Position position) { ongoingTxns.put(txnId, position); Position firstPosition = ongoingTxns.get(ongoingTxns.firstKey()); // max read position is less than first ongoing transaction message position - updateMaxReadPosition(((ManagedLedgerImpl) topic.getManagedLedger()).getPreviousPosition(firstPosition), + updateMaxReadPosition(topic.getManagedLedger().getPreviousPosition(firstPosition), false); } } @@ -488,7 +487,7 @@ void removeTxnAndUpdateMaxReadPosition(TxnID txnID) { ongoingTxns.remove(txnID); if (!ongoingTxns.isEmpty()) { Position position = ongoingTxns.get(ongoingTxns.firstKey()); - updateMaxReadPosition(((ManagedLedgerImpl) topic.getManagedLedger()).getPreviousPosition(position), false); + updateMaxReadPosition(topic.getManagedLedger().getPreviousPosition(position), false); } else { updateMaxReadPosition(topic.getManagedLedger().getLastConfirmedEntry(), false); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java index f8143cfc4c125..25c7727259db3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStore.java @@ -32,6 +32,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Predicate; @@ -44,7 +45,6 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; import org.apache.pulsar.broker.transaction.pendingack.PendingAckReplyCallBack; @@ -121,7 +121,7 @@ public class MLPendingAckStore implements PendingAckStore { public MLPendingAckStore(ManagedLedger managedLedger, ManagedCursor cursor, ManagedCursor subManagedCursor, long transactionPendingAckLogIndexMinLag, TxnLogBufferedWriterConfig bufferedWriterConfig, - Timer timer, TxnLogBufferedWriterMetricsStats bufferedWriterMetrics) { + Timer timer, TxnLogBufferedWriterMetricsStats bufferedWriterMetrics, Executor executor) { this.managedLedger = managedLedger; this.cursor = cursor; this.currentLoadPosition = this.cursor.getMarkDeletedPosition(); @@ -131,7 +131,7 @@ public MLPendingAckStore(ManagedLedger managedLedger, ManagedCursor cursor, this.subManagedCursor = subManagedCursor; this.logIndexBackoff = new LogIndexLagBackoff(transactionPendingAckLogIndexMinLag, Long.MAX_VALUE, 1); this.maxIndexLag = logIndexBackoff.next(0); - this.bufferedWriter = new TxnLogBufferedWriter(managedLedger, ((ManagedLedgerImpl) managedLedger).getExecutor(), + this.bufferedWriter = new TxnLogBufferedWriter(managedLedger, executor, timer, PendingAckLogSerializer.INSTANCE, bufferedWriterConfig.getBatchedWriteMaxRecords(), bufferedWriterConfig.getBatchedWriteMaxSize(), bufferedWriterConfig.getBatchedWriteMaxDelayInMillis(), bufferedWriterConfig.isBatchEnabled(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java index 5308648b80c1d..6fc61d423ce85 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java @@ -134,7 +134,12 @@ public void openCursorComplete(ManagedCursor cursor, Object ctx) { .getConfiguration() .getTransactionPendingAckLogIndexMinLag(), txnLogBufferedWriterConfig, - brokerClientSharedTimer, bufferedWriterMetrics)); + brokerClientSharedTimer, bufferedWriterMetrics, + originPersistentTopic + .getBrokerService() + .getPulsar() + .getOrderedExecutor() + .chooseThread())); if (log.isDebugEnabled()) { log.debug("{},{} open MLPendingAckStore cursor success", originPersistentTopic.getName(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java index 27408854b0198..591842927f35b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/PendingAckHandleImpl.java @@ -47,7 +47,6 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.AckSetState; import org.apache.bookkeeper.mledger.impl.AckSetStateUtil; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -235,8 +234,7 @@ public void internalIndividualAcknowledgeMessage(TxnID txnID, List findStartPoint(cursorPosition, context.ledger.getLastAddConfirmed(), context.cache) @@ -143,7 +141,7 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, for (Entry entry : entries) { entriesSize += entry.getLength(); } - managedCursor.updateReadStats(entries.size(), entriesSize); + cursor.updateReadStats(entries.size(), entriesSize); Entry lastEntry = entries.get(entries.size() - 1); // The compaction task depends on the last snapshot and the incremental diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java index 5023180e0b979..a7a5fd4ef1113 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicUtils.java @@ -31,7 +31,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer; @@ -75,8 +74,7 @@ public static void asyncReadCompactedEntries(TopicCompactionService topicCompact return CompletableFuture.completedFuture(null); } - ManagedCursorImpl managedCursor = (ManagedCursorImpl) cursor; - int numberOfEntriesToRead = managedCursor.applyMaxSizeCap(maxEntries, bytesToRead); + int numberOfEntriesToRead = cursor.applyMaxSizeCap(maxEntries, bytesToRead); return topicCompactionService.readCompactedEntries(readPosition, numberOfEntriesToRead) .thenAccept(entries -> { @@ -94,7 +92,7 @@ public static void asyncReadCompactedEntries(TopicCompactionService topicCompact for (Entry entry : entries) { entriesSize += entry.getLength(); } - managedCursor.updateReadStats(entries.size(), entriesSize); + cursor.updateReadStats(entries.size(), entriesSize); Entry lastEntry = entries.get(entries.size() - 1); cursor.seek(lastEntry.getPosition().getNext(), true); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java index e0e679b113f33..cf0b3c45d7023 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java @@ -32,6 +32,7 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; public class MockManagedCursor implements ManagedCursor { @@ -414,4 +415,34 @@ public boolean checkAndUpdateReadPositionChanged() { public boolean isClosed() { return false; } + + @Override + public ManagedLedgerInternalStats.CursorStats getCursorStats() { + return null; + } + + @Override + public boolean isMessageDeleted(Position position) { + return false; + } + + @Override + public ManagedCursor duplicateNonDurableCursor(String nonDurableCursorName) throws ManagedLedgerException { + return null; + } + + @Override + public long[] getBatchPositionAckSet(Position position) { + return new long[0]; + } + + @Override + public int applyMaxSizeCap(int maxEntries, long maxSizeBytes) { + return 0; + } + + @Override + public void updateReadStats(int readEntriesCount, long readEntriesSize) { + + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CurrentLedgerRolloverIfFullTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CurrentLedgerRolloverIfFullTest.java index 375fe41f143cd..4f83d25a29210 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CurrentLedgerRolloverIfFullTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CurrentLedgerRolloverIfFullTest.java @@ -81,7 +81,9 @@ public void testCurrentLedgerRolloverIfFull() throws Exception { } ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); - Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2 + 1); + }); for (int i = 0; i < msgNum; i++) { Message msg = consumer.receive(2, TimeUnit.SECONDS); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java index ff8e418c024a0..20ea33fb3e1ed 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.broker.service.persistent; -import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; +import static org.apache.bookkeeper.mledger.ManagedCursor.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.Metric; import static org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsClient.parseMetrics; import static org.testng.Assert.assertEquals; @@ -38,7 +38,6 @@ import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.ManagedCursor; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.pulsar.PrometheusMetricsTestUtil; import org.apache.pulsar.broker.BrokerTestUtil; @@ -104,7 +103,7 @@ public void testBucketDelayedDeliveryWithAllConsumersDisconnecting() throws Exce Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 1000)); List bucketKeys = ((PersistentDispatcherMultipleConsumers) dispatcher).getCursor().getCursorProperties().keySet().stream() - .filter(x -> x.startsWith(ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX)).toList(); + .filter(x -> x.startsWith(CURSOR_INTERNAL_PROPERTY_PREFIX)).toList(); c1.close(); @@ -119,7 +118,7 @@ public void testBucketDelayedDeliveryWithAllConsumersDisconnecting() throws Exce Dispatcher dispatcher2 = pulsar.getBrokerService().getTopicReference(topic).get().getSubscription("sub").getDispatcher(); List bucketKeys2 = ((PersistentDispatcherMultipleConsumers) dispatcher2).getCursor().getCursorProperties().keySet().stream() - .filter(x -> x.startsWith(ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX)).toList(); + .filter(x -> x.startsWith(CURSOR_INTERNAL_PROPERTY_PREFIX)).toList(); Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher2.getNumberOfDelayedMessages(), 1000)); Assert.assertEquals(bucketKeys, bucketKeys2); @@ -384,7 +383,7 @@ public void testDeleteTopicIfCursorPropsEmpty(SubscriptionType subscriptionType) admin.topics().createSubscription(topic, subscriptionName, MessageId.earliest); pulsarClient.newConsumer().topic(topic).subscriptionName(subscriptionName) .subscriptionType(subscriptionType).subscribe().close(); - ManagedCursorImpl cursor = findCursor(topic, subscriptionName); + ManagedCursor cursor = findCursor(topic, subscriptionName); assertNotNull(cursor); assertTrue(cursor.getCursorProperties() == null || cursor.getCursorProperties().isEmpty()); // Test topic deletion is successful. @@ -404,7 +403,7 @@ public void testDeletePartitionedTopicIfCursorPropsEmpty(SubscriptionType subscr admin.topics().createSubscription(topic, subscriptionName, MessageId.earliest); pulsarClient.newConsumer().topic(topic).subscriptionName(subscriptionName) .subscriptionType(subscriptionType).subscribe().close(); - ManagedCursorImpl cursor = findCursor(topic + "-partition-0", subscriptionName); + ManagedCursor cursor = findCursor(topic + "-partition-0", subscriptionName); assertNotNull(cursor); assertTrue(cursor.getCursorProperties() == null || cursor.getCursorProperties().isEmpty()); // Test topic deletion is successful. @@ -424,7 +423,7 @@ public void testDeleteTopicIfCursorPropsNotEmpty(SubscriptionType subscriptionTy admin.topics().createSubscription(topic, subscriptionName, MessageId.earliest); pulsarClient.newConsumer().topic(topic).subscriptionName(subscriptionName) .subscriptionType(subscriptionType).subscribe().close(); - ManagedCursorImpl cursor = findCursor(topic, subscriptionName); + ManagedCursor cursor = findCursor(topic, subscriptionName); assertNotNull(cursor); assertTrue(cursor.getCursorProperties() == null || cursor.getCursorProperties().isEmpty()); // Put a subscription prop. @@ -451,7 +450,7 @@ public void testDeletePartitionedTopicIfCursorPropsNotEmpty(SubscriptionType sub pulsarClient.newConsumer().topic(topic).subscriptionName(subscriptionName) .subscriptionType(subscriptionType).subscribe().close(); - ManagedCursorImpl cursor = findCursor(topic + "-partition-0", subscriptionName); + ManagedCursor cursor = findCursor(topic + "-partition-0", subscriptionName); assertNotNull(cursor); assertTrue(cursor.getCursorProperties() == null || cursor.getCursorProperties().isEmpty()); // Put a subscription prop. @@ -464,7 +463,7 @@ public void testDeletePartitionedTopicIfCursorPropsNotEmpty(SubscriptionType sub } - private ManagedCursorImpl findCursor(String topic, String subscriptionName) { + private ManagedCursor findCursor(String topic, String subscriptionName) { PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); Iterator cursorIterator = persistentTopic.getManagedLedger().getCursors().iterator(); @@ -473,7 +472,7 @@ private ManagedCursorImpl findCursor(String topic, String subscriptionName) { if (managedCursor == null || !managedCursor.getName().equals(subscriptionName)) { continue; } - return (ManagedCursorImpl) managedCursor; + return managedCursor; } return null; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java index b9c0ab08e4ea1..d0fd384ba78fb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/ManagedLedgerMetricsTest.java @@ -29,9 +29,9 @@ import java.util.Map.Entry; import java.util.concurrent.TimeUnit; import lombok.Cleanup; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl; import org.apache.bookkeeper.mledger.impl.OpenTelemetryManagedLedgerStats; import org.apache.pulsar.broker.BrokerTestUtil; @@ -106,7 +106,7 @@ public void testManagedLedgerMetrics() throws Exception { } var managedLedgerFactory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory(); - for (Entry ledger : managedLedgerFactory.getManagedLedgers().entrySet()) { + for (Entry ledger : managedLedgerFactory.getManagedLedgers().entrySet()) { ManagedLedgerMBeanImpl stats = (ManagedLedgerMBeanImpl) ledger.getValue().getStats(); stats.refreshStats(1, TimeUnit.SECONDS); } @@ -118,7 +118,7 @@ public void testManagedLedgerMetrics() throws Exception { String message = "my-message-" + i; producer.send(message.getBytes()); } - for (Entry ledger : managedLedgerFactory.getManagedLedgers().entrySet()) { + for (Entry ledger : managedLedgerFactory.getManagedLedgers().entrySet()) { ManagedLedgerMBeanImpl stats = (ManagedLedgerMBeanImpl) ledger.getValue().getStats(); stats.refreshStats(1, TimeUnit.SECONDS); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java index 8ab9d58f57076..3924281c094b1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TopicTransactionBufferRecoverTest.java @@ -49,8 +49,8 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.PositionFactory; +import org.apache.bookkeeper.mledger.ReadOnlyManagedLedger; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl; import org.apache.bookkeeper.mledger.proto.MLDataFormats; import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.lang3.RandomUtils; @@ -796,7 +796,7 @@ public void testTransactionBufferSegmentSystemTopic() throws Exception { AsyncCallbacks.OpenReadOnlyManagedLedgerCallback callback = new AsyncCallbacks .OpenReadOnlyManagedLedgerCallback() { @Override - public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) { + public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger readOnlyManagedLedger, Object ctx) { readOnlyManagedLedger.asyncReadEntry( PositionFactory.create(messageId.getLedgerId(), messageId.getEntryId()), new AsyncCallbacks.ReadEntryCallback() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index 3b3eaf7bb2292..d3e0391443f0f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -864,7 +864,8 @@ public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception{ doReturn(CompletableFuture.completedFuture( new MLPendingAckStore(persistentTopic.getManagedLedger(), managedCursor, null, 500, bufferedWriterConfig, transactionTimer, - DISABLED_BUFFERED_WRITER_METRICS))) + DISABLED_BUFFERED_WRITER_METRICS, persistentTopic.getBrokerService().getPulsar() + .getOrderedExecutor().chooseThread()))) .when(pendingAckStoreProvider).newPendingAckStore(any()); doReturn(CompletableFuture.completedFuture(true)).when(pendingAckStoreProvider).checkInitializedBefore(any()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java index efe83cebc3661..f01ea7ac67ff9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckMetadataTest.java @@ -18,9 +18,19 @@ */ package org.apache.pulsar.broker.transaction.pendingack; +import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.State.WriteFailed; +import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; +import static org.testng.Assert.assertTrue; +import static org.testng.AssertJUnit.fail; import io.netty.util.HashedWheelTimer; import io.netty.util.concurrent.DefaultThreadFactory; +import java.lang.reflect.Field; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import lombok.Cleanup; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -36,14 +46,6 @@ import org.apache.pulsar.common.api.proto.CommandAck; import org.apache.pulsar.transaction.coordinator.impl.TxnLogBufferedWriterConfig; import org.testng.annotations.Test; -import java.lang.reflect.Field; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; -import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.State.WriteFailed; -import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; -import static org.testng.Assert.assertTrue; -import static org.testng.AssertJUnit.fail; public class PendingAckMetadataTest extends MockedBookKeeperTestCase { @@ -80,9 +82,13 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { ManagedCursor cursor = completableFuture.get().openCursor("test"); ManagedCursor subCursor = completableFuture.get().openCursor("test"); + + @Cleanup("shutdownNow") + ExecutorService executorService = Executors.newSingleThreadExecutor(); + MLPendingAckStore pendingAckStore = new MLPendingAckStore(completableFuture.get(), cursor, subCursor, 500, - bufferedWriterConfig, transactionTimer, DISABLED_BUFFERED_WRITER_METRICS); + bufferedWriterConfig, transactionTimer, DISABLED_BUFFERED_WRITER_METRICS, executorService); Field field = MLPendingAckStore.class.getDeclaredField("managedLedger"); field.setAccessible(true); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java index 95a45d37d9556..b68b6308c8f3b 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ManagedLedgerInternalStats.java @@ -20,10 +20,14 @@ import java.util.List; import java.util.Map; +import lombok.AccessLevel; +import lombok.Getter; + /** * ManagedLedger internal statistics. */ +@Getter(AccessLevel.PUBLIC) public class ManagedLedgerInternalStats { /** Messages published since this broker loaded this managedLedger. */ @@ -82,6 +86,7 @@ public static class LedgerInfo { /** * Pulsar cursor statistics. */ + @Getter(AccessLevel.PUBLIC) public static class CursorStats { public String markDeletePosition; public String readPosition; diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java index 8f52d20c5ee83..60dcbb8b3acd8 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/MockManagedLedger.java @@ -18,8 +18,10 @@ */ package org.apache.bookkeeper.mledger.offload.jcloud.impl; +import com.google.common.collect.Range; import io.netty.buffer.ByteBuf; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Predicate; @@ -32,6 +34,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerMXBean; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.PositionBound; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo; import org.apache.pulsar.common.api.proto.CommandSubscribe; @@ -177,6 +180,11 @@ public long getNumberOfEntries() { return 0; } + @Override + public long getNumberOfEntries(Range range) { + return 0; + } + @Override public long getNumberOfActiveEntries() { return 0; @@ -381,6 +389,51 @@ public void checkCursorsToCacheEntries() { // no-op } + @Override + public void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx) { + + } + + @Override + public NavigableMap getLedgersInfo() { + return null; + } + + @Override + public Position getNextValidPosition(Position position) { + return null; + } + + @Override + public Position getPreviousPosition(Position position) { + return null; + } + + @Override + public long getEstimatedBacklogSize(Position position) { + return 0; + } + + @Override + public Position getPositionAfterN(Position startPosition, long n, PositionBound startRange) { + return null; + } + + @Override + public int getPendingAddEntriesCount() { + return 0; + } + + @Override + public long getCacheSize() { + return 0; + } + + @Override + public Position getFirstPosition() { + return null; + } + @Override public CompletableFuture asyncMigrate() { // no-op From 212ccf8fe029052aa8f15e22597873b95d96288e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 17 Sep 2024 07:58:51 +0300 Subject: [PATCH 2/5] Decouple OpAddEntry from ManagedLedgerInterceptor --- .../bookkeeper/mledger/impl/OpAddEntry.java | 6 ++-- .../intercept/ManagedLedgerInterceptor.java | 35 ++++++++++++++----- .../ManagedLedgerInterceptorImpl.java | 10 +++--- .../MangedLedgerInterceptorImplTest.java | 6 ++-- .../MLTransactionSequenceIdGenerator.java | 5 ++- 5 files changed, 38 insertions(+), 24 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java index 3f0699657b5d4..036ce9223e89d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java @@ -45,7 +45,7 @@ * */ @Slf4j -public class OpAddEntry implements AddCallback, CloseCallback, Runnable { +public class OpAddEntry implements AddCallback, CloseCallback, Runnable, ManagedLedgerInterceptor.AddEntryOperation { protected ManagedLedgerImpl ml; LedgerHandle ledger; long entryId; @@ -139,8 +139,8 @@ public void initiate() { lastInitTime = System.nanoTime(); if (ml.getManagedLedgerInterceptor() != null) { long originalDataLen = data.readableBytes(); - payloadProcessorHandle = ml.getManagedLedgerInterceptor().processPayloadBeforeLedgerWrite(this, - duplicateBuffer); + payloadProcessorHandle = ml.getManagedLedgerInterceptor() + .processPayloadBeforeLedgerWrite(this.getCtx(), duplicateBuffer); if (payloadProcessorHandle != null) { duplicateBuffer = payloadProcessorHandle.getProcessedPayload(); // If data len of entry changes, correct "dataLength" and "currentLedgerSize". diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java index d26a5e15735aa..11c05a463e5c8 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java @@ -24,7 +24,6 @@ import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.common.annotation.InterfaceAudience; import org.apache.bookkeeper.common.annotation.InterfaceStability; -import org.apache.bookkeeper.mledger.impl.OpAddEntry; /** * Interceptor for ManagedLedger. @@ -32,14 +31,34 @@ @InterfaceAudience.LimitedPrivate @InterfaceStability.Stable public interface ManagedLedgerInterceptor { + /** + * An operation to add an entry to a ledger. + */ + interface AddEntryOperation { + /** + * Get the data to be written to the ledger. + * @return data to be written to the ledger + */ + ByteBuf getData(); + /** + * Set the data to be written to the ledger. + * @param data data to be written to the ledger + */ + void setData(ByteBuf data); + /** + * Get the operation context object. + * @return context the context object + */ + Object getCtx(); + } /** - * Intercept an OpAddEntry and return an OpAddEntry. - * @param op an OpAddEntry to be intercepted. + * Intercept adding an entry to a ledger. + * + * @param op an operation to be intercepted. * @param numberOfMessages - * @return an OpAddEntry. */ - OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages); + void beforeAddEntry(AddEntryOperation op, int numberOfMessages); /** * Intercept When add entry failed. @@ -93,12 +112,12 @@ default PayloadProcessorHandle processPayloadBeforeEntryCache(ByteBuf dataReadFr /** * Intercept before payload gets written to ledger. - * @param ledgerWriteOp OpAddEntry used to trigger ledger write. + * @param ctx the operation context object * @param dataToBeStoredInLedger data to be stored in ledger * @return handle to the processor */ - default PayloadProcessorHandle processPayloadBeforeLedgerWrite(OpAddEntry ledgerWriteOp, - ByteBuf dataToBeStoredInLedger){ + default PayloadProcessorHandle processPayloadBeforeLedgerWrite(Object ctx, + ByteBuf dataToBeStoredInLedger) { return null; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java index 02c6c575fd919..03403e8d4231a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java @@ -25,7 +25,6 @@ import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.api.LedgerEntry; -import org.apache.bookkeeper.mledger.impl.OpAddEntry; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; @@ -85,12 +84,11 @@ public long getIndex() { } @Override - public OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages) { + public void beforeAddEntry(AddEntryOperation op, int numberOfMessages) { if (op == null || numberOfMessages <= 0) { - return op; + return; } op.setData(Commands.addBrokerEntryMetadata(op.getData(), brokerEntryMetadataInterceptors, numberOfMessages)); - return op; } @Override @@ -189,11 +187,11 @@ public void release() { }; } @Override - public PayloadProcessorHandle processPayloadBeforeLedgerWrite(OpAddEntry op, ByteBuf ledgerData) { + public PayloadProcessorHandle processPayloadBeforeLedgerWrite(Object ctx, ByteBuf ledgerData) { if (this.inputProcessors == null || this.inputProcessors.size() == 0) { return null; } - return processPayload(this.inputProcessors, op.getCtx(), ledgerData); + return processPayload(this.inputProcessors, ctx, ledgerData); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java index 16953d76ade45..74a88382b0e0e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/MangedLedgerInterceptorImplTest.java @@ -41,7 +41,6 @@ import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.bookkeeper.mledger.impl.OpAddEntry; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; @@ -391,16 +390,15 @@ public MockManagedLedgerInterceptorImpl( } @Override - public OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages) { + public void beforeAddEntry(AddEntryOperation op, int numberOfMessages) { if (op == null || numberOfMessages <= 0) { - return op; + return; } op.setData(Commands.addBrokerEntryMetadata(op.getData(), brokerEntryMetadataInterceptors, numberOfMessages)); if (op != null) { throw new RuntimeException("throw exception before add entry for test"); } - return op; } } diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java index 204555a1cfc67..611b68453db0f 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java @@ -24,7 +24,6 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.api.LedgerEntry; -import org.apache.bookkeeper.mledger.impl.OpAddEntry; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.transaction.coordinator.proto.TransactionMetadataEntry; @@ -42,8 +41,8 @@ public class MLTransactionSequenceIdGenerator implements ManagedLedgerIntercepto private final AtomicLong sequenceId = new AtomicLong(TC_ID_NOT_USED); @Override - public OpAddEntry beforeAddEntry(OpAddEntry op, int numberOfMessages) { - return op; + public void beforeAddEntry(AddEntryOperation op, int numberOfMessages) { + // do nothing } // When all of ledger have been deleted, we will generate sequenceId from managedLedger properties From 408f88413fe67ea46e8b4d464bfe9e9d3757f0c2 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 17 Sep 2024 09:11:50 +0300 Subject: [PATCH 3/5] Decouple LedgerHandle --- .../mledger/impl/ManagedLedgerImpl.java | 52 +++++++++++++++--- .../mledger/impl/ShadowManagedLedgerImpl.java | 3 +- .../intercept/ManagedLedgerInterceptor.java | 22 ++++++-- .../ManagedLedgerInterceptorImpl.java | 52 ++++++------------ .../MLTransactionSequenceIdGenerator.java | 53 ++++++------------- 5 files changed, 96 insertions(+), 86 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 7ca15d835aed2..cb19bd94bce01 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -78,6 +78,7 @@ import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.client.api.LedgerEntry; import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.client.api.ReadHandle; import org.apache.bookkeeper.common.util.Backoff; @@ -431,13 +432,14 @@ public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) { .setTimestamp(clock.millis()).build(); ledgers.put(id, info); if (managedLedgerInterceptor != null) { - managedLedgerInterceptor.onManagedLedgerLastLedgerInitialize(name, lh) - .thenRun(() -> initializeBookKeeper(callback)) - .exceptionally(ex -> { - callback.initializeFailed( - new ManagedLedgerInterceptException(ex.getCause())); - return null; - }); + managedLedgerInterceptor + .onManagedLedgerLastLedgerInitialize(name, createLastEntryHandle(lh)) + .thenRun(() -> initializeBookKeeper(callback)) + .exceptionally(ex -> { + callback.initializeFailed( + new ManagedLedgerInterceptException(ex.getCause())); + return null; + }); } else { initializeBookKeeper(callback); } @@ -477,6 +479,42 @@ public void operationFailed(MetaStoreException e) { scheduleTimeoutTask(); } + protected ManagedLedgerInterceptor.LastEntryHandle createLastEntryHandle(LedgerHandle lh) { + return () -> { + CompletableFuture> promise = new CompletableFuture<>(); + if (lh.getLastAddConfirmed() >= 0) { + lh.readAsync(lh.getLastAddConfirmed(), lh.getLastAddConfirmed()) + .whenComplete((entries, ex) -> { + if (ex != null) { + promise.completeExceptionally(ex); + } else { + if (entries != null) { + try { + LedgerEntry ledgerEntry = + entries.getEntry(lh.getLastAddConfirmed()); + if (ledgerEntry != null) { + promise.complete( + Optional.of(EntryImpl.create(ledgerEntry))); + } else { + promise.complete(Optional.empty()); + } + entries.close(); + } catch (Exception e) { + entries.close(); + promise.completeExceptionally(e); + } + } else { + promise.complete(Optional.empty()); + } + } + }); + } else { + promise.complete(Optional.empty()); + } + return promise; + }; + } + protected synchronized void initializeBookKeeper(final ManagedLedgerInitializeLedgerCallback callback) { if (log.isDebugEnabled()) { log.debug("[{}] initializing bookkeeper; ledgers {}", name, ledgers); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java index 546733f909e21..4b03cad8e0a1d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java @@ -131,7 +131,8 @@ public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat) currentLedger = lh; if (managedLedgerInterceptor != null) { - managedLedgerInterceptor.onManagedLedgerLastLedgerInitialize(name, lh) + managedLedgerInterceptor + .onManagedLedgerLastLedgerInitialize(name, createLastEntryHandle(lh)) .thenRun(() -> ShadowManagedLedgerImpl.super.initialize(callback, ctx)) .exceptionally(ex -> { callback.initializeFailed( diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java index 11c05a463e5c8..0ca6fa9dd866c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/intercept/ManagedLedgerInterceptor.java @@ -20,10 +20,11 @@ import io.netty.buffer.ByteBuf; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; -import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.common.annotation.InterfaceAudience; import org.apache.bookkeeper.common.annotation.InterfaceStability; +import org.apache.bookkeeper.mledger.Entry; /** * Interceptor for ManagedLedger. @@ -74,12 +75,25 @@ default void afterFailedAddEntry(int numberOfMessages){ */ void onManagedLedgerPropertiesInitialize(Map propertiesMap); + /** + * A handle for reading the last ledger entry. + */ + interface LastEntryHandle { + /** + * Read the last entry from the ledger. + * The caller is responsible for releasing the entry. + * @return the last entry from the ledger, if any + */ + CompletableFuture> readLastEntryAsync(); + } + /** * Intercept when ManagedLedger is initialized. - * @param name name of ManagedLedger - * @param ledgerHandle a LedgerHandle. + * + * @param name name of ManagedLedger + * @param lastEntryHandle a LedgerHandle. */ - CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LedgerHandle ledgerHandle); + CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LastEntryHandle lastEntryHandle); /** * @param propertiesMap map of properties. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java index 03403e8d4231a..db138989a8eee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/ManagedLedgerInterceptorImpl.java @@ -23,8 +23,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; -import org.apache.bookkeeper.client.LedgerHandle; -import org.apache.bookkeeper.client.api.LedgerEntry; +import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; @@ -113,43 +112,22 @@ public void onManagedLedgerPropertiesInitialize(Map propertiesMa } @Override - public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LedgerHandle lh) { - CompletableFuture promise = new CompletableFuture<>(); - boolean hasAppendIndexMetadataInterceptor = appendIndexMetadataInterceptor != null; - if (hasAppendIndexMetadataInterceptor && lh.getLastAddConfirmed() >= 0) { - lh.readAsync(lh.getLastAddConfirmed(), lh.getLastAddConfirmed()).whenComplete((entries, ex) -> { - if (ex != null) { - log.error("[{}] Read last entry error.", name, ex); - promise.completeExceptionally(ex); - } else { - if (entries != null) { - try { - LedgerEntry ledgerEntry = entries.getEntry(lh.getLastAddConfirmed()); - if (ledgerEntry != null) { - BrokerEntryMetadata brokerEntryMetadata = - Commands.parseBrokerEntryMetadataIfExist(ledgerEntry.getEntryBuffer()); - if (brokerEntryMetadata != null && brokerEntryMetadata.hasIndex()) { - appendIndexMetadataInterceptor.recoveryIndexGenerator( - brokerEntryMetadata.getIndex()); - } - } - entries.close(); - promise.complete(null); - } catch (Exception e) { - entries.close(); - log.error("[{}] Failed to recover the index generator from the last add confirmed entry.", - name, e); - promise.completeExceptionally(e); - } - } else { - promise.complete(null); + public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LastEntryHandle lh) { + return lh.readLastEntryAsync().thenAccept(lastEntryOptional -> { + if (lastEntryOptional.isPresent()) { + Entry lastEntry = lastEntryOptional.get(); + try { + BrokerEntryMetadata brokerEntryMetadata = + Commands.parseBrokerEntryMetadataIfExist(lastEntry.getDataBuffer()); + if (brokerEntryMetadata != null && brokerEntryMetadata.hasIndex()) { + appendIndexMetadataInterceptor.recoveryIndexGenerator( + brokerEntryMetadata.getIndex()); } + } finally { + lastEntry.release(); } - }); - } else { - promise.complete(null); - } - return promise; + } + }); } @Override diff --git a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java index 611b68453db0f..a6605046eeff6 100644 --- a/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java +++ b/pulsar-transaction/coordinator/src/main/java/org/apache/pulsar/transaction/coordinator/impl/MLTransactionSequenceIdGenerator.java @@ -22,8 +22,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; -import org.apache.bookkeeper.client.LedgerHandle; -import org.apache.bookkeeper.client.api.LedgerEntry; +import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor; import org.apache.commons.collections4.CollectionUtils; import org.apache.pulsar.transaction.coordinator.proto.TransactionMetadataEntry; @@ -59,43 +58,23 @@ public void onManagedLedgerPropertiesInitialize(Map propertiesMa // When we don't roll over ledger, we can init sequenceId from the getLastAddConfirmed transaction metadata entry @Override - public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LedgerHandle lh) { - CompletableFuture promise = new CompletableFuture<>(); - if (lh.getLastAddConfirmed() >= 0) { - lh.readAsync(lh.getLastAddConfirmed(), lh.getLastAddConfirmed()).whenComplete((entries, ex) -> { - if (ex != null) { - log.error("[{}] Read last entry error.", name, ex); - promise.completeExceptionally(ex); - } else { - if (entries != null) { - try { - LedgerEntry ledgerEntry = entries.getEntry(lh.getLastAddConfirmed()); - if (ledgerEntry != null) { - List transactionLogs = - MLTransactionLogImpl.deserializeEntry(ledgerEntry.getEntryBuffer()); - if (!CollectionUtils.isEmpty(transactionLogs)){ - TransactionMetadataEntry lastConfirmEntry = - transactionLogs.get(transactionLogs.size() - 1); - this.sequenceId.set(lastConfirmEntry.getMaxLocalTxnId()); - } - } - entries.close(); - promise.complete(null); - } catch (Exception e) { - entries.close(); - log.error("[{}] Failed to recover the tc sequenceId from the last add confirmed entry.", - name, e); - promise.completeExceptionally(e); - } - } else { - promise.complete(null); + public CompletableFuture onManagedLedgerLastLedgerInitialize(String name, LastEntryHandle lh) { + return lh.readLastEntryAsync().thenAccept(lastEntryOptional -> { + if (lastEntryOptional.isPresent()) { + Entry lastEntry = lastEntryOptional.get(); + try { + List transactionLogs = + MLTransactionLogImpl.deserializeEntry(lastEntry.getDataBuffer()); + if (!CollectionUtils.isEmpty(transactionLogs)) { + TransactionMetadataEntry lastConfirmEntry = + transactionLogs.get(transactionLogs.size() - 1); + this.sequenceId.set(lastConfirmEntry.getMaxLocalTxnId()); } + } finally { + lastEntry.release(); } - }); - } else { - promise.complete(null); - } - return promise; + } + }); } // roll over ledger will update sequenceId to managedLedger properties From 42210e2e69efd1240dfd3965828bde09f4ebf943 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 17 Sep 2024 00:21:23 -0700 Subject: [PATCH 4/5] Adjust indent Co-authored-by: Yunze Xu --- .../org/apache/bookkeeper/mledger/ManagedLedgerFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java index 9854b5de34ecc..d9c887fac468e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java @@ -242,7 +242,7 @@ void asyncDelete(String name, CompletableFuture mlConfigFut void estimateUnloadedTopicBacklog(PersistentOfflineTopicStats offlineTopicStats, TopicName topicName, - boolean accurate, Object ctx) throws Exception; + boolean accurate, Object ctx) throws Exception; ManagedLedgerFactoryConfig getConfig(); } From 3d5de42c9d7c7326759b85008c5de8140a70b909 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 17 Sep 2024 00:21:45 -0700 Subject: [PATCH 5/5] Adjust indent Co-authored-by: Yunze Xu --- .../main/java/org/apache/bookkeeper/mledger/PositionBound.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java index f6f0cd3445cf0..a7ab4a48a9b02 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java @@ -20,5 +20,5 @@ public enum PositionBound { // define boundaries for position based seeks and searches - startIncluded, startExcluded + startIncluded, startExcluded }