From 7e544b79112650dc56c370b51522beec1a7e94e5 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 24 Sep 2024 17:57:12 +0800 Subject: [PATCH 01/23] Add tests to reproduce --- .../ExtensibleLoadManagerCloseTest.java | 46 +++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java index 41413f3e3a913..f345fae5d0a07 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java @@ -22,13 +22,15 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -41,21 +43,26 @@ public class ExtensibleLoadManagerCloseTest { private static final String clusterName = "test"; private final LocalBookkeeperEnsemble bk = new LocalBookkeeperEnsemble(1, 0, () -> 0); private final List brokers = new ArrayList<>(); - private PulsarAdmin admin; @BeforeClass(alwaysRun = true) public void setup() throws Exception { bk.start(); - for (int i = 0; i < 3; i++) { + } + + private void setupBrokers(int numBrokers) throws Exception { + brokers.clear(); + for (int i = 0; i < numBrokers; i++) { final var broker = new PulsarService(brokerConfig()); broker.start(); brokers.add(broker); } - admin = brokers.get(0).getAdminClient(); - admin.clusters().createCluster(clusterName, ClusterData.builder().build()); - admin.tenants().createTenant("public", TenantInfo.builder() - .allowedClusters(Collections.singleton(clusterName)).build()); - admin.namespaces().createNamespace("public/default"); + final var admin = brokers.get(0).getAdminClient(); + if (!admin.clusters().getClusters().contains(clusterName)) { + admin.clusters().createCluster(clusterName, ClusterData.builder().build()); + admin.tenants().createTenant("public", TenantInfo.builder() + .allowedClusters(Collections.singleton(clusterName)).build()); + admin.namespaces().createNamespace("public/default"); + } } @@ -85,7 +92,9 @@ private ServiceConfiguration brokerConfig() { @Test public void testCloseAfterLoadingBundles() throws Exception { + setupBrokers(3); final var topic = "test"; + final var admin = brokers.get(0).getAdminClient(); admin.topics().createPartitionedTopic(topic, 20); admin.lookups().lookupPartitionedTopic(topic); final var client = PulsarClient.builder().serviceUrl(brokers.get(0).getBrokerServiceUrl()).build(); @@ -104,4 +113,25 @@ public void testCloseAfterLoadingBundles() throws Exception { Assert.assertTrue(closeTimeMs < 5000L); } } + + @Test + public void testLookup() throws Exception { + setupBrokers(1); + final var topic = "test-lookup"; + final var numPartitions = 16; + final var admin = brokers.get(0).getAdminClient(); + admin.topics().createPartitionedTopic(topic, numPartitions); + + final var futures = new ArrayList>(); + for (int i = 0; i < numPartitions; i++) { + futures.add(admin.lookups().lookupTopicAsync(topic + TopicName.PARTITIONED_TOPIC_SUFFIX + i)); + } + FutureUtil.waitForAll(futures).get(); + + final var start = System.currentTimeMillis(); + brokers.get(0).close(); + final var closeTimeMs = System.currentTimeMillis() - start; + log.info("Broker close time: {}", closeTimeMs); + Assert.assertTrue(closeTimeMs < 5000L); + } } From d7b2d80073bb92c2821a5722700badb5f9400c96 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 24 Sep 2024 22:26:25 +0800 Subject: [PATCH 02/23] Fix Free event cannot be sent --- .../extensions/channel/ServiceUnitStateChannelImpl.java | 6 +++++- .../extensions/channel/ServiceUnitStateData.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index ddbc9eacac921..a80b38c7df8af 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -1273,7 +1273,11 @@ private void handleBrokerDeletionEvent(String broker) { return; } } catch (Exception e) { - log.error("Failed to handle broker deletion event.", e); + if (e instanceof ExecutionException && e.getCause() instanceof IllegalStateException) { + log.warn("Failed to handle broker deletion event due to {}", e.getMessage()); + } else { + log.error("Failed to handle broker deletion event.", e); + } return; } MetadataState state = getMetadataState(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java index e85134e611632..4a990ddbc9b21 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateData.java @@ -34,7 +34,7 @@ public record ServiceUnitStateData( public ServiceUnitStateData { Objects.requireNonNull(state); - if (StringUtils.isBlank(dstBroker) && StringUtils.isBlank(sourceBroker)) { + if (state != ServiceUnitState.Free && StringUtils.isBlank(dstBroker) && StringUtils.isBlank(sourceBroker)) { throw new IllegalArgumentException("Empty broker"); } } From 590e8dd02edc3bf9ab82eca587279db71681adc0 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 24 Sep 2024 22:38:49 +0800 Subject: [PATCH 03/23] Close TopicPoliciesService before unload --- .../main/java/org/apache/pulsar/broker/PulsarService.java | 7 +++---- .../service/SystemTopicBasedTopicPoliciesService.java | 8 +++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index a2f6fb9e9773b..6c768a078974f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -513,6 +513,9 @@ public CompletableFuture closeAsync() { return closeFuture; } LOG.info("Closing PulsarService"); + if (topicPoliciesService != null) { + topicPoliciesService.close(); + } if (brokerService != null) { brokerService.unloadNamespaceBundlesGracefully(); } @@ -633,10 +636,6 @@ public CompletableFuture closeAsync() { transactionBufferClient.close(); } - if (topicPoliciesService != null) { - topicPoliciesService.close(); - topicPoliciesService = null; - } if (client != null) { client.close(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 18b4c610a5c9b..59bb5f2337fff 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -254,7 +254,7 @@ public CompletableFuture> getTopicPoliciesAsync(TopicNam // initialization by calling this method. At the moment, the load manager does not start so the lookup // for "__change_events" will fail. In this case, just return an empty policies to avoid deadlock. final var loadManager = pulsarService.getLoadManager().get(); - if (loadManager == null || !loadManager.started()) { + if (loadManager == null || !loadManager.started() || closed.get()) { return CompletableFuture.completedFuture(Optional.empty()); } final CompletableFuture preparedFuture = prepareInitPoliciesCacheAsync(topicName.getNamespaceObject()); @@ -308,6 +308,9 @@ public void addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { @VisibleForTesting @Nonnull CompletableFuture prepareInitPoliciesCacheAsync(@Nonnull NamespaceName namespace) { requireNonNull(namespace); + if (closed.get()) { + return CompletableFuture.completedFuture(false); + } return pulsarService.getPulsarResources().getNamespaceResources().getPoliciesAsync(namespace) .thenCompose(namespacePolicies -> { if (namespacePolicies.isEmpty() || namespacePolicies.get().deleted) { @@ -331,6 +334,9 @@ public void addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { }); initFuture.exceptionally(ex -> { try { + if (closed.get()) { + return null; + } log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); cleanCacheAndCloseReader(namespace, false); From c4de2c69ae8987ad717117d03698d78ce3e11cb9 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 24 Sep 2024 23:44:16 +0800 Subject: [PATCH 04/23] Avoid blocking the thread that the topic policies reader is created --- .../SystemTopicBasedTopicPoliciesService.java | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 59bb5f2337fff..6ff6408916b1c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -687,14 +687,22 @@ public void close() throws Exception { if (closed.compareAndSet(false, true)) { writerCaches.synchronous().invalidateAll(); readerCaches.values().forEach(future -> { - if (future != null && !future.isCompletedExceptionally()) { - future.thenAccept(reader -> { - try { - reader.close(); - } catch (Exception e) { - log.error("Failed to close reader.", e); - } - }); + try { + final var reader = future.getNow(null); + if (reader != null) { + reader.close(); + log.info("Closed the reader for topic policies"); + } else { + // Avoid blocking the thread that the reader is created + future.thenAccept(SystemTopicClient.Reader::closeAsync).whenComplete((__, e) -> { + if (e == null) { + log.info("Closed the reader for topic policies"); + } else { + log.error("Failed to close the reader for topic policies", e); + } + }); + } + } catch (Throwable ignored) { } }); readerCaches.clear(); From 810bd04dc16d8c478e7b99dec6bcc3543ce0036f Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 25 Sep 2024 00:57:23 +0800 Subject: [PATCH 05/23] Tombstone the bundle gracefully --- .../channel/ServiceUnitStateChannelImpl.java | 31 ++++++++++++++++--- .../ServiceUnitStateTableViewImpl.java | 19 ++++++++++-- .../pulsar/client/impl/TableViewImpl.java | 7 ++--- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index a80b38c7df8af..5c260ad9c2ccb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -412,9 +412,7 @@ public CompletableFuture isChannelOwnerAsync() { if (owner.isPresent()) { return isTargetBroker(owner.get()); } else { - String msg = "There is no channel owner now."; - log.error(msg); - throw new IllegalStateException(msg); + throw new IllegalStateException("There is no channel owner now."); } }); } @@ -851,7 +849,7 @@ private void handleSplitEvent(String serviceUnit, ServiceUnitStateData data) { } } - private void handleFreeEvent(String serviceUnit, ServiceUnitStateData data) { + private CompletableFuture handleFreeEvent(String serviceUnit, ServiceUnitStateData data) { var getOwnerRequest = getOwnerRequests.remove(serviceUnit); if (getOwnerRequest != null) { getOwnerRequest.complete(null); @@ -865,8 +863,10 @@ private void handleFreeEvent(String serviceUnit, ServiceUnitStateData data) { : CompletableFuture.completedFuture(0)).thenApply(__ -> null); stateChangeListeners.notifyOnCompletion(future, serviceUnit, data) .whenComplete((__, e) -> log(e, serviceUnit, data, null)); + return future; } else { stateChangeListeners.notify(serviceUnit, data, null); + return CompletableFuture.completedFuture(null); } } @@ -1385,8 +1385,10 @@ private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanDa private void waitForCleanups(String broker, boolean excludeSystemTopics, int maxWaitTimeInMillis) { long started = System.currentTimeMillis(); + final var futures = new HashMap>(); while (System.currentTimeMillis() - started < maxWaitTimeInMillis) { boolean cleaned = true; + futures.clear(); for (var etr : tableview.entrySet()) { var serviceUnit = etr.getKey(); var data = etr.getValue(); @@ -1395,7 +1397,9 @@ private void waitForCleanups(String broker, boolean excludeSystemTopics, int max continue; } - if (data.state() == Owned && broker.equals(data.dstBroker())) { + if (data.state() == Free) { + futures.put(serviceUnit, handleFreeEvent(serviceUnit, data)); + } else if (data.state() == Owned && broker.equals(data.dstBroker())) { cleaned = false; break; } @@ -1411,6 +1415,18 @@ private void waitForCleanups(String broker, boolean excludeSystemTopics, int max } } } + var waitTimeMs = started + maxWaitTimeInMillis - System.currentTimeMillis(); + if (waitTimeMs < 0) { + waitTimeMs = 0; + } + try { + FutureUtil.waitForAll(futures.values()).get(waitTimeMs, MILLISECONDS); + } catch (ExecutionException e) { + log.error("Failed to tombstone {}", futures.keySet(), e.getCause()); + } catch (TimeoutException __) { + log.warn("Failed to tombstone {} in {} ms", futures.keySet(), waitTimeMs); + } catch (InterruptedException ignored) { + } log.info("Finished cleanup waiting for orphan broker:{}. Elapsed {} ms", brokerId, System.currentTimeMillis() - started); } @@ -1432,6 +1448,11 @@ private synchronized void doCleanup(String broker, boolean gracefully) { log.info("Started ownership cleanup for the inactive broker:{}", broker); int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); + try { + tableview.flush(OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS); + } catch (Exception e) { + log.error("Failed to flush", e); + } Map orphanSystemServiceUnits = new HashMap<>(); for (var etr : tableview.entrySet()) { var stateData = etr.getValue(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java index 8dfaddcdabca1..12cf87445a3dd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java @@ -33,6 +33,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.common.naming.TopicDomain; @@ -144,8 +145,13 @@ public CompletableFuture put(String key, ServiceUnitStateData value) { .sendAsync() .whenComplete((messageId, e) -> { if (e != null) { - log.error("Failed to publish the message: serviceUnit:{}, data:{}", - key, value, e); + if (e instanceof PulsarClientException.AlreadyClosedException) { + log.info("Skip publishing the message since the producer is closed, serviceUnit: {}, data: " + + "{}", key, value); + } else { + log.error("Failed to publish the message: serviceUnit:{}, data:{}", + key, value, e); + } future.completeExceptionally(e); } else { future.complete(null); @@ -159,7 +165,14 @@ public void flush(long waitDurationInMillis) throws InterruptedException, Timeou if (!isValidState()) { throw new IllegalStateException(INVALID_STATE_ERROR_MSG); } - producer.flushAsync().get(waitDurationInMillis, MILLISECONDS); + final var deadline = System.currentTimeMillis() + waitDurationInMillis; + var waitTimeMs = waitDurationInMillis; + producer.flushAsync().get(waitTimeMs, MILLISECONDS); + waitTimeMs = deadline - System.currentTimeMillis(); + if (waitTimeMs < 0) { + waitTimeMs = 0; + } + tableview.refreshAsync().get(waitTimeMs, MILLISECONDS); } @Override diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java index d5d4174ee10a9..4f52060497864 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java @@ -364,8 +364,8 @@ private void readAllExistingMessages(Reader reader, CompletableFuture f } }).exceptionally(ex -> { if (ex.getCause() instanceof PulsarClientException.AlreadyClosedException) { - log.error("Reader {} was closed while reading existing messages.", - reader.getTopic(), ex); + log.info("Reader {} was closed while reading existing messages.", + reader.getTopic()); } else { log.warn("Reader {} was interrupted while reading existing messages. ", reader.getTopic(), ex); @@ -393,8 +393,7 @@ private void readTailMessages(Reader reader) { readTailMessages(reader); }).exceptionally(ex -> { if (ex.getCause() instanceof PulsarClientException.AlreadyClosedException) { - log.error("Reader {} was closed while reading tail messages.", - reader.getTopic(), ex); + log.info("Reader {} was closed while reading tail messages.", reader.getTopic()); // Fail all refresh request when no more messages can be read. pendingRefreshRequests.keySet().forEach(future -> { pendingRefreshRequests.remove(future); From c6540d9a73b86f6965b6cf43b3a3c0bf3ff79713 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 25 Sep 2024 12:01:24 +0800 Subject: [PATCH 06/23] Fix load data store related issues --- .../extensions/ExtensibleLoadManagerImpl.java | 31 +++++++++++++++++++ .../channel/ServiceUnitStateChannelImpl.java | 6 +++- .../ServiceUnitStateDataConflictResolver.java | 4 +-- .../filter/BrokerMaxTopicCountFilter.java | 7 ++++- .../ExtensibleLoadManagerCloseTest.java | 1 + 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 98ef6bf36edac..7be7ca1f89827 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -182,6 +182,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager, BrokerS private SplitManager splitManager; volatile boolean started = false; + boolean disabling = false; private boolean configuredSystemTopics = false; @@ -808,6 +809,9 @@ public static boolean isInternalTopic(String topic) { @VisibleForTesting synchronized void playLeader() { + if (disabling) { + return; + } log.info("This broker:{} is setting the role from {} to {}", pulsar.getBrokerId(), role, Leader); int retry = 0; @@ -835,6 +839,10 @@ synchronized void playLeader() { } break; } catch (Throwable e) { + if (disabling) { + log.warn("The broker:{} failed to playLeader, exit because it's disabled", pulsar.getBrokerId()); + return; + } log.warn("The broker:{} failed to set the role. Retrying {} th ...", pulsar.getBrokerId(), ++retry, e); try { @@ -846,6 +854,9 @@ synchronized void playLeader() { } } } + if (disabling) { + return; + } if (becameFollower) { log.warn("The broker:{} became follower while initializing leader role.", pulsar.getBrokerId()); @@ -863,6 +874,9 @@ synchronized void playLeader() { @VisibleForTesting synchronized void playFollower() { + if (disabling) { + return; + } log.info("This broker:{} is setting the role from {} to {}", pulsar.getBrokerId(), role, Follower); int retry = 0; @@ -885,6 +899,10 @@ synchronized void playFollower() { serviceUnitStateTableViewSyncer.close(); break; } catch (Throwable e) { + if (disabling) { + log.warn("The broker:{} failed to playFollower, exit because it's disabled", pulsar.getBrokerId()); + return; + } log.warn("The broker:{} failed to set the role. Retrying {} th ...", pulsar.getBrokerId(), ++retry, e); try { @@ -896,6 +914,9 @@ synchronized void playFollower() { } } } + if (disabling) { + return; + } if (becameLeader) { log.warn("This broker:{} became leader while initializing follower role.", pulsar.getBrokerId()); @@ -982,9 +1003,19 @@ protected void monitor() { } public void disableBroker() throws Exception { + // TopicDoesNotExistException might be thrown and it's not recoverable. Enable this flag to exit playFollower() + // or playLeader() quickly. + synchronized (this) { + disabling = true; + } serviceUnitStateChannel.cleanOwnerships(); leaderElectionService.close(); brokerRegistry.unregister(); + final var availableBrokers = brokerRegistry.getAvailableBrokersAsync() + .get(conf.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); + if (availableBrokers.isEmpty()) { + close(); + } // Close the internal topics (if owned any) after giving up the possible leader role, // so that the subsequent lookups could hit the next leader. closeInternalTopics(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 5c260ad9c2ccb..cdfddc2d62b4a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -1408,10 +1408,14 @@ private void waitForCleanups(String broker, boolean excludeSystemTopics, int max break; } else { try { - MILLISECONDS.sleep(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS); + tableview.flush(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS); } catch (InterruptedException e) { log.warn("Interrupted while delaying the next service unit clean-up. Cleaning broker:{}", brokerId); + } catch (ExecutionException e) { + log.error("Failed to flush table view", e.getCause()); + } catch (TimeoutException e) { + log.warn("Failed to flush the table view in {} ms", OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS); } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java index b1dbb6fac8709..d812842c26271 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java @@ -67,7 +67,7 @@ public void checkBrokers(boolean check) { @Override public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to) { - if (to == null) { + if (to == null || to.state() == ServiceUnitState.Free) { return false; } @@ -145,4 +145,4 @@ private boolean invalidUnload(ServiceUnitStateData from, ServiceUnitStateData to || !from.dstBroker().equals(to.sourceBroker()) || from.dstBroker().equals(to.dstBroker()); } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java index 48213c18e6376..9863d05ee751e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java @@ -41,7 +41,12 @@ public CompletableFuture> filterAsync(Map { - Optional brokerLoadDataOpt = context.brokerLoadDataStore().get(broker); + final Optional brokerLoadDataOpt; + try { + brokerLoadDataOpt = context.brokerLoadDataStore().get(broker); + } catch (IllegalStateException ignored) { + return false; + } long topics = brokerLoadDataOpt.map(BrokerLoadData::getTopics).orElse(0L); // TODO: The broker load data might be delayed, so the max topic check might not accurate. return topics >= loadBalancerBrokerMaxTopics; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java index f345fae5d0a07..ecaa72d40e9b3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java @@ -38,6 +38,7 @@ import org.testng.annotations.Test; @Slf4j +@Test(groups = "broker") public class ExtensibleLoadManagerCloseTest { private static final String clusterName = "test"; From 9b5d390f888fff34d9f7fdb108411cfa40cc768a Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 25 Sep 2024 13:16:04 +0800 Subject: [PATCH 07/23] Cleanup BK for each test --- .../loadbalance/extensions/ExtensibleLoadManagerCloseTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java index ecaa72d40e9b3..fa63ce566c603 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerCloseTest.java @@ -42,11 +42,12 @@ public class ExtensibleLoadManagerCloseTest { private static final String clusterName = "test"; - private final LocalBookkeeperEnsemble bk = new LocalBookkeeperEnsemble(1, 0, () -> 0); private final List brokers = new ArrayList<>(); + private LocalBookkeeperEnsemble bk; @BeforeClass(alwaysRun = true) public void setup() throws Exception { + bk = new LocalBookkeeperEnsemble(1, 0, () -> 0); bk.start(); } From 95bc70515721f8974772d4bda462fe052460ea16 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 25 Sep 2024 17:10:15 +0800 Subject: [PATCH 08/23] Restore handleSkippedEvent and revert the change to shouldKeepLeft --- .../channel/ServiceUnitStateChannelImpl.java | 8 ++++++-- .../ServiceUnitStateDataConflictResolver.java | 2 +- .../ServiceUnitStateMetadataStoreTableViewImpl.java | 4 +++- .../channel/ServiceUnitStateTableView.java | 8 +++++++- .../channel/ServiceUnitStateTableViewImpl.java | 13 +++++++++++-- .../channel/ServiceUnitStateTableViewSyncer.java | 12 ++++++++---- 6 files changed, 36 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index cdfddc2d62b4a..5725249769740 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -307,7 +307,7 @@ public synchronized void start() throws PulsarServerException { pulsar.getConfiguration().getDefaultNumberOfNamespaceBundles()); tableview = createServiceUnitStateTableView(); - tableview.start(pulsar, this::handleEvent, this::handleExisting); + tableview.start(pulsar, this::handleEvent, this::handleExisting, this::handleSkippedEvent); if (debug) { log.info("Successfully started the channel tableview."); @@ -772,7 +772,11 @@ brokerId, getLogEventTag(data), serviceUnit, } } - private void handleSkippedEvent(String serviceUnit) { + private void handleSkippedEvent(String serviceUnit, ServiceUnitStateData skippedData) { + if (skippedData.state() == Free) { + handleFreeEvent(serviceUnit, skippedData); + return; + } var getOwnerRequest = getOwnerRequests.get(serviceUnit); if (getOwnerRequest != null) { var data = tableview.get(serviceUnit); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java index d812842c26271..3e43237f4c00e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java @@ -67,7 +67,7 @@ public void checkBrokers(boolean check) { @Override public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to) { - if (to == null || to.state() == ServiceUnitState.Free) { + if (to == null) { return false; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java index f488b31c77415..283381a2e14bf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java @@ -56,11 +56,13 @@ public class ServiceUnitStateMetadataStoreTableViewImpl extends ServiceUnitState public void start(PulsarService pulsar, BiConsumer tailItemListener, - BiConsumer existingItemListener) + BiConsumer existingItemListener, + BiConsumer skippedItemListener) throws MetadataStoreException { init(pulsar); conflictResolver = new ServiceUnitStateDataConflictResolver(); conflictResolver.setStorageType(MetadataStore); + conflictResolver.setSkippedMsgHandler(skippedItemListener); tableview = new MetadataStoreTableViewImpl<>(ServiceUnitStateData.class, pulsar.getBrokerId(), pulsar.getLocalMetadataStore(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableView.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableView.java index 5ac57fe5c19c6..b55e44a0cc7e0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableView.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableView.java @@ -27,6 +27,8 @@ import java.util.concurrent.TimeoutException; import java.util.function.BiConsumer; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.common.classification.InterfaceAudience; +import org.apache.pulsar.common.classification.InterfaceStability; import org.apache.pulsar.common.naming.NamespaceBundle; /** @@ -37,6 +39,8 @@ * ServiceUnitStateTableView receives notifications whenever ownership states are updated in the remote store, and * upon notification, it applies the updates to its local tableview with the listener logic. */ +@InterfaceStability.Evolving +@InterfaceAudience.LimitedPrivate public interface ServiceUnitStateTableView extends Closeable { /** @@ -46,11 +50,13 @@ public interface ServiceUnitStateTableView extends Closeable { * @param pulsar pulsar service reference * @param tailItemListener listener to listen tail(newly updated) items * @param existingItemListener listener to listen existing items + * @param skippedItemListener listener for items that are skipped by the topic compaction strategy * @throws IOException if it fails to init the tableview. */ void start(PulsarService pulsar, BiConsumer tailItemListener, - BiConsumer existingItemListener) throws IOException; + BiConsumer existingItemListener, + BiConsumer skippedItemListener) throws IOException; /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java index 12cf87445a3dd..91f4d12df80d9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java @@ -21,6 +21,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE; +import static org.apache.pulsar.common.topics.TopicCompactionStrategy.TABLE_VIEW_TAG; import java.io.IOException; import java.util.Map; import java.util.Set; @@ -38,6 +39,7 @@ import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.topics.TopicCompactionStrategy; @Slf4j public class ServiceUnitStateTableViewImpl extends ServiceUnitStateTableViewBase { @@ -53,7 +55,8 @@ public class ServiceUnitStateTableViewImpl extends ServiceUnitStateTableViewBase public void start(PulsarService pulsar, BiConsumer tailItemListener, - BiConsumer existingItemListener) throws IOException { + BiConsumer existingItemListener, + BiConsumer skippedItemListener) throws IOException { boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log); init(pulsar); @@ -98,7 +101,13 @@ public void start(PulsarService pulsar, tableview.listen(tailItemListener); tableview.forEach(this::updateOwnedServiceUnits); tableview.forEach(existingItemListener); - + final var strategy = (ServiceUnitStateDataConflictResolver) TopicCompactionStrategy.getInstance(TABLE_VIEW_TAG); + if (strategy == null) { + String err = TABLE_VIEW_TAG + "tag TopicCompactionStrategy is null."; + log.error(err); + throw new IllegalStateException(err); + } + strategy.setSkippedMsgHandler(skippedItemListener); } private boolean isValidState() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java index 10ab39a66d279..42550eaa43963 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java @@ -100,7 +100,8 @@ private void syncExistingItems() metadataStoreTableView.start( pulsar, this::dummy, - this::dummy + this::dummy, + (__, ___) -> {} ); @Cleanup @@ -108,7 +109,8 @@ private void syncExistingItems() systemTopicTableView.start( pulsar, this::dummy, - this::dummy + this::dummy, + (__, ___) -> {} ); @@ -152,7 +154,8 @@ private void syncTailItems() throws InterruptedException, IOException, TimeoutEx this.metadataStoreTableView.start( pulsar, this::syncToSystemTopic, - this::dummy + this::dummy, + (__, ___) -> {} ); log.info("Started MetadataStoreTableView"); @@ -160,7 +163,8 @@ private void syncTailItems() throws InterruptedException, IOException, TimeoutEx this.systemTopicTableView.start( pulsar, this::syncToMetadataStore, - this::dummy + this::dummy, + (__, ___) -> {} ); log.info("Started SystemTopicTableView"); From a9e9893c63f2fa806c7d810ba343bc8b750be224 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 25 Sep 2024 19:54:59 +0800 Subject: [PATCH 09/23] Fail all the lookup requests after clearOwnerships() is called --- .../channel/ServiceUnitStateChannelImpl.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 5725249769740..0c274c972bcea 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -141,6 +141,7 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { private volatile long lastOwnEventHandledAt = 0; private long lastOwnedServiceUnitCountAt = 0; private int totalOwnedServiceUnitCnt = 0; + private volatile boolean disablePubOwnedEvent = false; public enum EventType { Assign, @@ -255,6 +256,7 @@ public void cancelOwnershipMonitor() { @Override public void cleanOwnerships() { + disablePubOwnedEvent = true; doCleanup(brokerId, true); } @@ -817,6 +819,15 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { private void handleAssignEvent(String serviceUnit, ServiceUnitStateData data) { if (isTargetBroker(data.dstBroker())) { + if (disablePubOwnedEvent) { + log.info("Skip assigning self({}) as the owner after cleanOwnerships", serviceUnit); + final var getOwnerRequest = getOwnerRequests.remove(serviceUnit); + if (getOwnerRequest != null) { + getOwnerRequest.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException( + "lookup during ownership cleanup")); + } + return; + } ServiceUnitStateData next = new ServiceUnitStateData( Owned, data.dstBroker(), data.sourceBroker(), getNextVersionId(data)); stateChangeListeners.notifyOnCompletion(pubAsync(serviceUnit, next), serviceUnit, data) @@ -1439,7 +1450,7 @@ private void waitForCleanups(String broker, boolean excludeSystemTopics, int max System.currentTimeMillis() - started); } - private synchronized void doCleanup(String broker, boolean gracefully) { + private void doCleanup(String broker, boolean gracefully) { try { if (getChannelOwnerAsync().get(MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS, TimeUnit.SECONDS) .isEmpty()) { From ca51fda32fda2c3cddd409ed7f33c5378c32338d Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 26 Sep 2024 15:04:16 +0800 Subject: [PATCH 10/23] Add enum state to replace started and disabling --- .../extensions/ExtensibleLoadManagerImpl.java | 68 ++++++++++++------- .../ExtensibleLoadManagerWrapper.java | 2 +- 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 7be7ca1f89827..a723fd764268c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -181,8 +181,14 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager, BrokerS private SplitManager splitManager; - volatile boolean started = false; - boolean disabling = false; + enum State { + INIT, + RUNNING, + // It's removing visibility of the current broker from other brokers. In this state, it cannot play as a leader + // or follower. + DISABLED, + } + private final AtomicReference state = new AtomicReference<>(State.INIT); private boolean configuredSystemTopics = false; @@ -215,7 +221,7 @@ public CompletableFuture> getOwnedServiceUnitsAsync() { } public Set getOwnedServiceUnits() { - if (!started) { + if (!running()) { log.warn("Failed to get owned service units, load manager is not started."); return Collections.emptySet(); } @@ -345,7 +351,7 @@ public static CompletableFuture> getAssignedBrokerLoo @Override public void start() throws PulsarServerException { - if (this.started) { + if (state.get() != State.INIT) { return; } try { @@ -444,7 +450,9 @@ public void start() throws PulsarServerException { this.splitScheduler.start(); this.initWaiter.complete(true); - this.started = true; + if (!state.compareAndSet(State.INIT, State.RUNNING)) { + failForUnexpectedState("start"); + } log.info("Started load manager."); } catch (Throwable e) { failStarting(e); @@ -756,7 +764,7 @@ private CompletableFuture splitAsync(SplitDecision decision, @Override public void close() throws PulsarServerException { - if (!this.started) { + if (state.get() == State.INIT) { return; } try { @@ -792,7 +800,7 @@ public void close() throws PulsarServerException { } catch (Exception e) { throw new PulsarServerException(e); } finally { - this.started = false; + state.set(State.INIT); } } @@ -809,22 +817,22 @@ public static boolean isInternalTopic(String topic) { @VisibleForTesting synchronized void playLeader() { - if (disabling) { - return; - } log.info("This broker:{} is setting the role from {} to {}", pulsar.getBrokerId(), role, Leader); int retry = 0; boolean becameFollower = false; while (!Thread.currentThread().isInterrupted()) { try { - if (!initWaiter.get()) { + if (!initWaiter.get() || disabled()) { return; } if (!serviceUnitStateChannel.isChannelOwner()) { becameFollower = true; break; } + if (disabled()) { + return; + } // Confirm the system topics have been created or create them if they do not exist. // If the leader has changed, the new leader need to reset // the local brokerService.topics (by this topic creations). @@ -839,8 +847,9 @@ synchronized void playLeader() { } break; } catch (Throwable e) { - if (disabling) { - log.warn("The broker:{} failed to playLeader, exit because it's disabled", pulsar.getBrokerId()); + if (disabled()) { + log.warn("The broker:{} failed to set the role but exit because it's disabled", + pulsar.getBrokerId(), e); return; } log.warn("The broker:{} failed to set the role. Retrying {} th ...", @@ -854,7 +863,7 @@ synchronized void playLeader() { } } } - if (disabling) { + if (disabled()) { return; } @@ -874,22 +883,22 @@ synchronized void playLeader() { @VisibleForTesting synchronized void playFollower() { - if (disabling) { - return; - } log.info("This broker:{} is setting the role from {} to {}", pulsar.getBrokerId(), role, Follower); int retry = 0; boolean becameLeader = false; while (!Thread.currentThread().isInterrupted()) { try { - if (!initWaiter.get()) { + if (!initWaiter.get() || disabled()) { return; } if (serviceUnitStateChannel.isChannelOwner()) { becameLeader = true; break; } + if (disabled()) { + return; + } unloadScheduler.close(); serviceUnitStateChannel.cancelOwnershipMonitor(); closeInternalTopics(); @@ -899,8 +908,9 @@ synchronized void playFollower() { serviceUnitStateTableViewSyncer.close(); break; } catch (Throwable e) { - if (disabling) { - log.warn("The broker:{} failed to playFollower, exit because it's disabled", pulsar.getBrokerId()); + if (disabled()) { + log.warn("The broker:{} failed to set the role but exit because it's disabled", + pulsar.getBrokerId(), e); return; } log.warn("The broker:{} failed to set the role. Retrying {} th ...", @@ -914,7 +924,7 @@ synchronized void playFollower() { } } } - if (disabling) { + if (disabled()) { return; } @@ -1005,8 +1015,8 @@ protected void monitor() { public void disableBroker() throws Exception { // TopicDoesNotExistException might be thrown and it's not recoverable. Enable this flag to exit playFollower() // or playLeader() quickly. - synchronized (this) { - disabling = true; + if (!state.compareAndSet(State.RUNNING, State.DISABLED)) { + failForUnexpectedState("disableBroker"); } serviceUnitStateChannel.cleanOwnerships(); leaderElectionService.close(); @@ -1049,4 +1059,16 @@ protected BrokerRegistry createBrokerRegistry(PulsarService pulsar) { protected ServiceUnitStateChannel createServiceUnitStateChannel(PulsarService pulsar) { return new ServiceUnitStateChannelImpl(pulsar); } + + private void failForUnexpectedState(String msg) { + throw new IllegalStateException("Failed to " + msg + ", state: " + state.get()); + } + + boolean running() { + return state.get() == State.RUNNING; + } + + private boolean disabled() { + return state.get() == State.DISABLED; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java index 6a48607977ba9..35f6cfcbcf549 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java @@ -51,7 +51,7 @@ public void start() throws PulsarServerException { } public boolean started() { - return loadManager.started && loadManager.getServiceUnitStateChannel().started(); + return loadManager.running() && loadManager.getServiceUnitStateChannel().started(); } @Override From 15fee179a7ffaaf3e4bce9885f4818b8af9fc35e Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 26 Sep 2024 15:28:33 +0800 Subject: [PATCH 11/23] Reuse the Disabled channel state to skip handling all events --- .../channel/ServiceUnitStateChannelImpl.java | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 0c274c972bcea..6666e09f04271 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -256,7 +256,7 @@ public void cancelOwnershipMonitor() { @Override public void cleanOwnerships() { - disablePubOwnedEvent = true; + disable(); doCleanup(brokerId, true); } @@ -679,11 +679,15 @@ private void handleEvent(String serviceUnit, ServiceUnitStateData data) { brokerId, serviceUnit, data, totalHandledRequests); } + ServiceUnitState state = state(data); if (channelState == Disabled) { + final var request = getOwnerRequests.remove(serviceUnit); + if (request != null) { + request.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException( + "cancel the lookup request for " + serviceUnit + " when receiving " + state)); + } return; } - - ServiceUnitState state = state(data); try { switch (state) { case Owned -> handleOwnEvent(serviceUnit, data); @@ -819,15 +823,6 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { private void handleAssignEvent(String serviceUnit, ServiceUnitStateData data) { if (isTargetBroker(data.dstBroker())) { - if (disablePubOwnedEvent) { - log.info("Skip assigning self({}) as the owner after cleanOwnerships", serviceUnit); - final var getOwnerRequest = getOwnerRequests.remove(serviceUnit); - if (getOwnerRequest != null) { - getOwnerRequest.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException( - "lookup during ownership cleanup")); - } - return; - } ServiceUnitStateData next = new ServiceUnitStateData( Owned, data.dstBroker(), data.sourceBroker(), getNextVersionId(data)); stateChangeListeners.notifyOnCompletion(pubAsync(serviceUnit, next), serviceUnit, data) @@ -1423,7 +1418,8 @@ private void waitForCleanups(String broker, boolean excludeSystemTopics, int max break; } else { try { - tableview.flush(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS); + tableview.flush(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS / 2); + Thread.sleep(OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS / 2); } catch (InterruptedException e) { log.warn("Interrupted while delaying the next service unit clean-up. Cleaning broker:{}", brokerId); From 774e32dc74d36f976a6a77d228670a8dea7dd9bf Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 26 Sep 2024 15:28:38 +0800 Subject: [PATCH 12/23] Revert "Restore handleSkippedEvent and revert the change to shouldKeepLeft" This reverts commit 95bc70515721f8974772d4bda462fe052460ea16. --- .../channel/ServiceUnitStateChannelImpl.java | 8 ++------ .../ServiceUnitStateDataConflictResolver.java | 2 +- .../ServiceUnitStateMetadataStoreTableViewImpl.java | 4 +--- .../channel/ServiceUnitStateTableView.java | 8 +------- .../channel/ServiceUnitStateTableViewImpl.java | 13 ++----------- .../channel/ServiceUnitStateTableViewSyncer.java | 12 ++++-------- 6 files changed, 11 insertions(+), 36 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 6666e09f04271..86fadff31e786 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -309,7 +309,7 @@ public synchronized void start() throws PulsarServerException { pulsar.getConfiguration().getDefaultNumberOfNamespaceBundles()); tableview = createServiceUnitStateTableView(); - tableview.start(pulsar, this::handleEvent, this::handleExisting, this::handleSkippedEvent); + tableview.start(pulsar, this::handleEvent, this::handleExisting); if (debug) { log.info("Successfully started the channel tableview."); @@ -778,11 +778,7 @@ brokerId, getLogEventTag(data), serviceUnit, } } - private void handleSkippedEvent(String serviceUnit, ServiceUnitStateData skippedData) { - if (skippedData.state() == Free) { - handleFreeEvent(serviceUnit, skippedData); - return; - } + private void handleSkippedEvent(String serviceUnit) { var getOwnerRequest = getOwnerRequests.get(serviceUnit); if (getOwnerRequest != null) { var data = tableview.get(serviceUnit); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java index 3e43237f4c00e..d812842c26271 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java @@ -67,7 +67,7 @@ public void checkBrokers(boolean check) { @Override public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to) { - if (to == null) { + if (to == null || to.state() == ServiceUnitState.Free) { return false; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java index 283381a2e14bf..f488b31c77415 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateMetadataStoreTableViewImpl.java @@ -56,13 +56,11 @@ public class ServiceUnitStateMetadataStoreTableViewImpl extends ServiceUnitState public void start(PulsarService pulsar, BiConsumer tailItemListener, - BiConsumer existingItemListener, - BiConsumer skippedItemListener) + BiConsumer existingItemListener) throws MetadataStoreException { init(pulsar); conflictResolver = new ServiceUnitStateDataConflictResolver(); conflictResolver.setStorageType(MetadataStore); - conflictResolver.setSkippedMsgHandler(skippedItemListener); tableview = new MetadataStoreTableViewImpl<>(ServiceUnitStateData.class, pulsar.getBrokerId(), pulsar.getLocalMetadataStore(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableView.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableView.java index b55e44a0cc7e0..5ac57fe5c19c6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableView.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableView.java @@ -27,8 +27,6 @@ import java.util.concurrent.TimeoutException; import java.util.function.BiConsumer; import org.apache.pulsar.broker.PulsarService; -import org.apache.pulsar.common.classification.InterfaceAudience; -import org.apache.pulsar.common.classification.InterfaceStability; import org.apache.pulsar.common.naming.NamespaceBundle; /** @@ -39,8 +37,6 @@ * ServiceUnitStateTableView receives notifications whenever ownership states are updated in the remote store, and * upon notification, it applies the updates to its local tableview with the listener logic. */ -@InterfaceStability.Evolving -@InterfaceAudience.LimitedPrivate public interface ServiceUnitStateTableView extends Closeable { /** @@ -50,13 +46,11 @@ public interface ServiceUnitStateTableView extends Closeable { * @param pulsar pulsar service reference * @param tailItemListener listener to listen tail(newly updated) items * @param existingItemListener listener to listen existing items - * @param skippedItemListener listener for items that are skipped by the topic compaction strategy * @throws IOException if it fails to init the tableview. */ void start(PulsarService pulsar, BiConsumer tailItemListener, - BiConsumer existingItemListener, - BiConsumer skippedItemListener) throws IOException; + BiConsumer existingItemListener) throws IOException; /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java index 91f4d12df80d9..12cf87445a3dd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewImpl.java @@ -21,7 +21,6 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE; -import static org.apache.pulsar.common.topics.TopicCompactionStrategy.TABLE_VIEW_TAG; import java.io.IOException; import java.util.Map; import java.util.Set; @@ -39,7 +38,6 @@ import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.topics.TopicCompactionStrategy; @Slf4j public class ServiceUnitStateTableViewImpl extends ServiceUnitStateTableViewBase { @@ -55,8 +53,7 @@ public class ServiceUnitStateTableViewImpl extends ServiceUnitStateTableViewBase public void start(PulsarService pulsar, BiConsumer tailItemListener, - BiConsumer existingItemListener, - BiConsumer skippedItemListener) throws IOException { + BiConsumer existingItemListener) throws IOException { boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log); init(pulsar); @@ -101,13 +98,7 @@ public void start(PulsarService pulsar, tableview.listen(tailItemListener); tableview.forEach(this::updateOwnedServiceUnits); tableview.forEach(existingItemListener); - final var strategy = (ServiceUnitStateDataConflictResolver) TopicCompactionStrategy.getInstance(TABLE_VIEW_TAG); - if (strategy == null) { - String err = TABLE_VIEW_TAG + "tag TopicCompactionStrategy is null."; - log.error(err); - throw new IllegalStateException(err); - } - strategy.setSkippedMsgHandler(skippedItemListener); + } private boolean isValidState() { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java index 42550eaa43963..10ab39a66d279 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateTableViewSyncer.java @@ -100,8 +100,7 @@ private void syncExistingItems() metadataStoreTableView.start( pulsar, this::dummy, - this::dummy, - (__, ___) -> {} + this::dummy ); @Cleanup @@ -109,8 +108,7 @@ private void syncExistingItems() systemTopicTableView.start( pulsar, this::dummy, - this::dummy, - (__, ___) -> {} + this::dummy ); @@ -154,8 +152,7 @@ private void syncTailItems() throws InterruptedException, IOException, TimeoutEx this.metadataStoreTableView.start( pulsar, this::syncToSystemTopic, - this::dummy, - (__, ___) -> {} + this::dummy ); log.info("Started MetadataStoreTableView"); @@ -163,8 +160,7 @@ private void syncTailItems() throws InterruptedException, IOException, TimeoutEx this.systemTopicTableView.start( pulsar, this::syncToMetadataStore, - this::dummy, - (__, ___) -> {} + this::dummy ); log.info("Started SystemTopicTableView"); From 0e4ccc721aaef858a7bae7773d8cb93c8233a96b Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 26 Sep 2024 15:29:29 +0800 Subject: [PATCH 13/23] Don't handle Free events specially in ServiceUnitStateDataConflictResolver --- .../channel/ServiceUnitStateDataConflictResolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java index d812842c26271..3e43237f4c00e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateDataConflictResolver.java @@ -67,7 +67,7 @@ public void checkBrokers(boolean check) { @Override public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to) { - if (to == null || to.state() == ServiceUnitState.Free) { + if (to == null) { return false; } From 8589e8bde78999c0a48fd936e6a3a49ebdd8f0a8 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 26 Sep 2024 15:32:59 +0800 Subject: [PATCH 14/23] Allow getOwnedServiceUnits() in DISABLED state --- .../loadbalance/extensions/ExtensibleLoadManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index a723fd764268c..d216f28059376 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -221,7 +221,7 @@ public CompletableFuture> getOwnedServiceUnitsAsync() { } public Set getOwnedServiceUnits() { - if (!running()) { + if (state.get() == State.INIT) { log.warn("Failed to get owned service units, load manager is not started."); return Collections.emptySet(); } From bdff85621b21de0debcd026b68d93950f36a9cb7 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 26 Sep 2024 18:20:29 +0800 Subject: [PATCH 15/23] Stop load data report tasks in disableBroker() --- .../extensions/ExtensibleLoadManagerImpl.java | 59 ++++++++++--------- .../channel/ServiceUnitStateChannelImpl.java | 4 +- .../store/TableViewLoadDataStoreImpl.java | 10 +++- .../pulsar/broker/service/BrokerService.java | 7 ++- 4 files changed, 48 insertions(+), 32 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index d216f28059376..2e2b96ce0dfb7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -624,21 +624,13 @@ public CompletableFuture> selectAsync(ServiceUnitId bundle, filter.filterAsync(availableBrokerCandidates, bundle, context); futures.add(future); } - CompletableFuture> result = new CompletableFuture<>(); - FutureUtil.waitForAll(futures).whenComplete((__, ex) -> { - if (ex != null) { - // TODO: We may need to revisit this error case. - log.error("Failed to filter out brokers when select bundle: {}", bundle, ex); - } + return FutureUtil.waitForAll(futures).thenApply(__ -> { if (availableBrokerCandidates.isEmpty()) { - result.complete(Optional.empty()); - return; + return Optional.empty(); } Set candidateBrokers = availableBrokerCandidates.keySet(); - - result.complete(getBrokerSelectionStrategy().select(candidateBrokers, bundle, context)); + return getBrokerSelectionStrategy().select(candidateBrokers, bundle, context); }); - return result; }); } @@ -676,6 +668,9 @@ public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle, boolean force, long timeout, TimeUnit timeoutUnit) { + if (state.get() == State.INIT) { + return CompletableFuture.completedFuture(null); + } if (NamespaceService.isSLAOrHeartbeatNamespace(bundle.getNamespaceObject().toString())) { log.info("Skip unloading namespace bundle: {}.", bundle); return CompletableFuture.completedFuture(null); @@ -768,20 +763,7 @@ public void close() throws PulsarServerException { return; } try { - if (brokerLoadDataReportTask != null) { - brokerLoadDataReportTask.cancel(true); - } - - if (topBundlesLoadDataReportTask != null) { - topBundlesLoadDataReportTask.cancel(true); - } - - if (monitorTask != null) { - monitorTask.cancel(true); - } - - this.brokerLoadDataStore.shutdown(); - this.topBundlesLoadDataStore.shutdown(); + stopLoadDataReportTasks(); this.unloadScheduler.close(); this.splitScheduler.close(); this.serviceUnitStateTableViewSyncer.close(); @@ -808,6 +790,28 @@ public void close() throws PulsarServerException { } } + private void stopLoadDataReportTasks() { + if (brokerLoadDataReportTask != null) { + brokerLoadDataReportTask.cancel(true); + } + if (topBundlesLoadDataReportTask != null) { + topBundlesLoadDataReportTask.cancel(true); + } + if (monitorTask != null) { + monitorTask.cancel(true); + } + try { + brokerLoadDataStore.shutdown(); + } catch (IOException e) { + log.warn("Failed to shutdown brokerLoadDataStore", e); + } + try { + topBundlesLoadDataStore.shutdown(); + } catch (IOException e) { + log.warn("Failed to shutdown brokerLoadDataStore", e); + } + } + public static boolean isInternalTopic(String topic) { return INTERNAL_TOPICS.contains(topic) || topic.startsWith(TOPIC) @@ -1018,9 +1022,10 @@ public void disableBroker() throws Exception { if (!state.compareAndSet(State.RUNNING, State.DISABLED)) { failForUnexpectedState("disableBroker"); } + stopLoadDataReportTasks(); serviceUnitStateChannel.cleanOwnerships(); - leaderElectionService.close(); brokerRegistry.unregister(); + leaderElectionService.close(); final var availableBrokers = brokerRegistry.getAvailableBrokersAsync() .get(conf.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); if (availableBrokers.isEmpty()) { @@ -1033,7 +1038,7 @@ public void disableBroker() throws Exception { private void closeInternalTopics() { List> futures = new ArrayList<>(); - for (String name : INTERNAL_TOPICS) { + for (String name : Set.of(BROKER_LOAD_DATA_STORE_TOPIC, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC)) { pulsar.getBrokerService() .getTopicReference(name) .ifPresent(topic -> futures.add(topic.close(true) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 86fadff31e786..0b3989b1dc9db 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -680,7 +680,7 @@ private void handleEvent(String serviceUnit, ServiceUnitStateData data) { } ServiceUnitState state = state(data); - if (channelState == Disabled) { + if (channelState == Disabled && (data == null || !data.force())) { final var request = getOwnerRequests.remove(serviceUnit); if (request != null) { request.completeExceptionally(new BrokerServiceException.ServiceUnitNotReadyException( @@ -1442,7 +1442,7 @@ private void waitForCleanups(String broker, boolean excludeSystemTopics, int max System.currentTimeMillis() - started); } - private void doCleanup(String broker, boolean gracefully) { + private synchronized void doCleanup(String broker, boolean gracefully) { try { if (getChannelOwnerAsync().get(MAX_CHANNEL_OWNER_ELECTION_WAITING_TIME_IN_SECS, TimeUnit.SECONDS) .isEmpty()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java index c9d18676cfa99..3ce44a1e65a73 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java @@ -92,7 +92,11 @@ public synchronized CompletableFuture removeAsync(String key) { public synchronized Optional get(String key) { String msg = validateTableView(); if (StringUtils.isNotBlank(msg)) { - throw new IllegalStateException(msg); + if (msg.equals(SHUTDOWN_ERR_MSG)) { + return Optional.empty(); + } else { + throw new IllegalStateException(msg); + } } return Optional.ofNullable(tableView.get(key)); } @@ -193,7 +197,9 @@ public synchronized void startProducer() throws LoadDataStoreException { @Override public synchronized void close() throws IOException { - validateState(); + if (isShutdown) { + return; + } closeProducer(); closeTableView(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 09f04d878c4e5..bfa99eedcadce 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -973,7 +973,12 @@ public void unloadNamespaceBundlesGracefully(int maxConcurrentUnload, boolean cl pulsar.getNamespaceService().unloadNamespaceBundle(su, timeout, MILLISECONDS, closeWithoutWaitingClientDisconnect).get(timeout, MILLISECONDS); } catch (Exception e) { - log.warn("Failed to unload namespace bundle {}", su, e); + if (e instanceof ExecutionException + && e.getCause() instanceof ServiceUnitNotReadyException) { + log.warn("Failed to unload namespace bundle {}: {}", su, e.getMessage()); + } else { + log.warn("Failed to unload namespace bundle {}", su, e); + } } } }); From 79fa08d32f3fd4f5e567ac8841d7ce7b27eead38 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 26 Sep 2024 18:43:26 +0800 Subject: [PATCH 16/23] Remove unused field --- .../extensions/channel/ServiceUnitStateChannelImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 0b3989b1dc9db..6f7bd6e948254 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -141,7 +141,6 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { private volatile long lastOwnEventHandledAt = 0; private long lastOwnedServiceUnitCountAt = 0; private int totalOwnedServiceUnitCnt = 0; - private volatile boolean disablePubOwnedEvent = false; public enum EventType { Assign, From 83a0b88fa83d0cd301a63707ed0635e65d7c40d4 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 26 Sep 2024 21:02:23 +0800 Subject: [PATCH 17/23] Fix LoadDataStoreTest#testShutdown --- .../broker/loadbalance/extensions/store/LoadDataStoreTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java index 3267e67ad2c3e..820307637be67 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java @@ -198,7 +198,7 @@ public void testShutdown() throws Exception { Assert.assertTrue(loadDataStore.pushAsync("2", 2).isCompletedExceptionally()); Assert.assertTrue(loadDataStore.removeAsync("2").isCompletedExceptionally()); - assertThrows(IllegalStateException.class, () -> loadDataStore.get("2")); + assertTrue(loadDataStore.get("2").isEmpty()); assertThrows(IllegalStateException.class, loadDataStore::size); assertThrows(IllegalStateException.class, loadDataStore::entrySet); assertThrows(IllegalStateException.class, () -> loadDataStore.forEach((k, v) -> {})); @@ -206,7 +206,6 @@ public void testShutdown() throws Exception { assertThrows(IllegalStateException.class, loadDataStore::start); assertThrows(IllegalStateException.class, loadDataStore::startProducer); assertThrows(IllegalStateException.class, loadDataStore::startTableView); - assertThrows(IllegalStateException.class, loadDataStore::close); assertThrows(IllegalStateException.class, loadDataStore::closeTableView); } From e0fd852273c4cc3e85e0206b115e4899f948547f Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 26 Sep 2024 21:03:46 +0800 Subject: [PATCH 18/23] Fix wrong logs --- .../loadbalance/extensions/ExtensibleLoadManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 2e2b96ce0dfb7..439ee6eae2659 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -808,7 +808,7 @@ private void stopLoadDataReportTasks() { try { topBundlesLoadDataStore.shutdown(); } catch (IOException e) { - log.warn("Failed to shutdown brokerLoadDataStore", e); + log.warn("Failed to shutdown topBundlesLoadDataStore", e); } } From 7d31290579658e227d59f1886194621066d72f7b Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 26 Sep 2024 21:27:11 +0800 Subject: [PATCH 19/23] Revert changes on closeInternalTopics --- .../loadbalance/extensions/ExtensibleLoadManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 439ee6eae2659..64a90d8d3d56e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -1038,7 +1038,7 @@ public void disableBroker() throws Exception { private void closeInternalTopics() { List> futures = new ArrayList<>(); - for (String name : Set.of(BROKER_LOAD_DATA_STORE_TOPIC, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC)) { + for (String name : INTERNAL_TOPICS) { pulsar.getBrokerService() .getTopicReference(name) .ifPresent(topic -> futures.add(topic.close(true) From 8068f1dca12d074590bf65731c93cd115f04622f Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 26 Sep 2024 21:46:28 +0800 Subject: [PATCH 20/23] Skip scheduleCleanup if the state is Disabled --- .../extensions/channel/ServiceUnitStateChannelImpl.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 6f7bd6e948254..31c0e51f1d51a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -1302,6 +1302,11 @@ private void handleBrokerDeletionEvent(String broker) { private void scheduleCleanup(String broker, long delayInSecs) { var scheduled = new MutableObject>(); try { + final var channelState = this.channelState; + if (channelState == Disabled || channelState == Closed) { + log.warn("[{}] Skip scheduleCleanup because the state is {} now", brokerId, channelState); + return; + } cleanupJobs.computeIfAbsent(broker, k -> { Executor delayed = CompletableFuture .delayedExecutor(delayInSecs, TimeUnit.SECONDS, pulsar.getLoadManagerExecutor()); From 09f40ff0b80aa606c46af819ab591f824fb89717 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 26 Sep 2024 22:21:36 +0800 Subject: [PATCH 21/23] Revert Free events handling on waitForCleanups --- .../channel/ServiceUnitStateChannelImpl.java | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 31c0e51f1d51a..a6df9e21baa8e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -1395,10 +1395,8 @@ private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanDa private void waitForCleanups(String broker, boolean excludeSystemTopics, int maxWaitTimeInMillis) { long started = System.currentTimeMillis(); - final var futures = new HashMap>(); while (System.currentTimeMillis() - started < maxWaitTimeInMillis) { boolean cleaned = true; - futures.clear(); for (var etr : tableview.entrySet()) { var serviceUnit = etr.getKey(); var data = etr.getValue(); @@ -1407,9 +1405,7 @@ private void waitForCleanups(String broker, boolean excludeSystemTopics, int max continue; } - if (data.state() == Free) { - futures.put(serviceUnit, handleFreeEvent(serviceUnit, data)); - } else if (data.state() == Owned && broker.equals(data.dstBroker())) { + if (data.state() == Owned && broker.equals(data.dstBroker())) { cleaned = false; break; } @@ -1430,18 +1426,6 @@ private void waitForCleanups(String broker, boolean excludeSystemTopics, int max } } } - var waitTimeMs = started + maxWaitTimeInMillis - System.currentTimeMillis(); - if (waitTimeMs < 0) { - waitTimeMs = 0; - } - try { - FutureUtil.waitForAll(futures.values()).get(waitTimeMs, MILLISECONDS); - } catch (ExecutionException e) { - log.error("Failed to tombstone {}", futures.keySet(), e.getCause()); - } catch (TimeoutException __) { - log.warn("Failed to tombstone {} in {} ms", futures.keySet(), waitTimeMs); - } catch (InterruptedException ignored) { - } log.info("Finished cleanup waiting for orphan broker:{}. Elapsed {} ms", brokerId, System.currentTimeMillis() - started); } From d429420a21edbb5e055f86a93608fff4b7f21427 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 26 Sep 2024 22:22:25 +0800 Subject: [PATCH 22/23] Add log for possible reason that blocks the waitForCleanups --- .../extensions/channel/ServiceUnitStateChannelImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index a6df9e21baa8e..ce975495feb2a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -1407,6 +1407,7 @@ private void waitForCleanups(String broker, boolean excludeSystemTopics, int max if (data.state() == Owned && broker.equals(data.dstBroker())) { cleaned = false; + log.info("[{}] bundle {} is still owned by this, data: {}", broker, serviceUnit, data); break; } } From 5ad8309f315c7a838a6287b505dac133780e9a57 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 27 Sep 2024 13:28:25 +0800 Subject: [PATCH 23/23] Fix regression that does not swallow the exception from filters --- .../loadbalance/extensions/ExtensibleLoadManagerImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 64a90d8d3d56e..841f9bfb669d4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -624,7 +624,11 @@ public CompletableFuture> selectAsync(ServiceUnitId bundle, filter.filterAsync(availableBrokerCandidates, bundle, context); futures.add(future); } - return FutureUtil.waitForAll(futures).thenApply(__ -> { + return FutureUtil.waitForAll(futures).exceptionally(e -> { + // TODO: We may need to revisit this error case. + log.error("Failed to filter out brokers when select bundle: {}", bundle, e); + return null; + }).thenApply(__ -> { if (availableBrokerCandidates.isEmpty()) { return Optional.empty(); }