Skip to content

[broker] Optimize deduplicationSnapshotIntervalSeconds with HierarchyTopicPolicies #13769

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
codelipenghui merged 2 commits into from
Jan 19, 2022
Merged

[broker] Optimize deduplicationSnapshotIntervalSeconds with HierarchyTopicPolicies #13769

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4cfef64
optmize deduplicationSnapshotIntervalSeconds
Jan 16, 2022
d12494e
remove unused import
Jan 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,8 @@ protected void updateTopicPolicy(TopicPolicies data) {
topicPolicies.getMaxConsumersPerSubscription().updateTopicValue(data.getMaxConsumersPerSubscription());
topicPolicies.getInactiveTopicPolicies().updateTopicValue(data.getInactiveTopicPolicies());
topicPolicies.getDeduplicationEnabled().updateTopicValue(data.getDeduplicationEnabled());
topicPolicies.getDeduplicationSnapshotIntervalSeconds().updateTopicValue(
data.getDeduplicationSnapshotIntervalSeconds());
topicPolicies.getSubscriptionTypesEnabled().updateTopicValue(
CollectionUtils.isEmpty(data.getSubscriptionTypesEnabled()) ? null :
EnumSet.copyOf(data.getSubscriptionTypesEnabled()));
Expand Down Expand Up @@ -194,6 +196,8 @@ protected void updateTopicPolicyByNamespacePolicy(Policies namespacePolicies) {
.updateNamespaceValue(namespacePolicies.max_consumers_per_subscription);
topicPolicies.getInactiveTopicPolicies().updateNamespaceValue(namespacePolicies.inactive_topic_policies);
topicPolicies.getDeduplicationEnabled().updateNamespaceValue(namespacePolicies.deduplicationEnabled);
topicPolicies.getDeduplicationSnapshotIntervalSeconds().updateNamespaceValue(
namespacePolicies.deduplicationSnapshotIntervalSeconds);
topicPolicies.getDelayedDeliveryEnabled().updateNamespaceValue(
Optional.ofNullable(namespacePolicies.delayed_delivery_policies)
.map(DelayedDeliveryPolicies::isActive).orElse(null));
Expand All @@ -220,6 +224,8 @@ private void updateTopicPolicyByBrokerConfig() {
topicPolicies.getMaxConsumerPerTopic().updateBrokerValue(config.getMaxConsumersPerTopic());
topicPolicies.getMaxConsumersPerSubscription().updateBrokerValue(config.getMaxConsumersPerSubscription());
topicPolicies.getDeduplicationEnabled().updateBrokerValue(config.isBrokerDeduplicationEnabled());
topicPolicies.getDeduplicationSnapshotIntervalSeconds().updateBrokerValue(
config.getBrokerDeduplicationSnapshotIntervalSeconds());
//init backlogQuota
topicPolicies.getBackLogQuotaMap()
.get(BacklogQuota.BacklogQuotaType.destination_storage)
Expand Down
25 changes: 1 addition & 24 deletions ...roker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.TreeMap;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
Expand All @@ -40,9 +39,6 @@
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.service.Topic.PublishContext;
import org.apache.pulsar.common.api.proto.MessageMetadata;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.Policies;
import org.apache.pulsar.common.policies.data.TopicPolicies;
import org.apache.pulsar.common.protocol.Commands;
import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
import org.slf4j.Logger;
Expand Down Expand Up @@ -472,26 +468,7 @@ public long getLastPublishedSequenceId(String producerName) {
}

public void takeSnapshot() {
// try to get topic-level policies
Integer interval = topic.getTopicPolicies()
.map(TopicPolicies::getDeduplicationSnapshotIntervalSeconds)
.orElse(null);
try {
//if topic-level policies not exists, try to get namespace-level policies
if (interval == null) {
final Optional<Policies> policies = pulsar.getPulsarResources().getNamespaceResources()
.getPolicies(TopicName.get(topic.getName()).getNamespaceObject());
if (policies.isPresent()) {
interval = policies.get().deduplicationSnapshotIntervalSeconds;
}
}
} catch (Exception e) {
log.error("Failed to get namespace policies", e);
}
//There is no other level of policies, use the broker-level by default
if (interval == null) {
interval = pulsar.getConfiguration().getBrokerDeduplicationSnapshotIntervalSeconds();
}
Integer interval = topic.getHierarchyTopicPolicies().getDeduplicationSnapshotIntervalSeconds().get();
long currentTimeStamp = System.currentTimeMillis();
if (interval == null || interval <= 0
|| currentTimeStamp - lastSnapshotTimestamp < TimeUnit.SECONDS.toMillis(interval)) {
Expand Down
4 changes: 2 additions & 2 deletions ...roker/src/test/java/org/apache/pulsar/broker/service/persistent/TopicDuplicationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ public void testTopicPolicyTakeSnapshot() throws Exception {
Producer<String> producer = pulsarClient
.newProducer(Schema.STRING).topic(topicName).enableBatching(false).producerName(producerName).create();
waitCacheInit(topicName);
admin.topics().setDeduplicationSnapshotInterval(topicName, 3);
admin.topicPolicies().setDeduplicationSnapshotInterval(topicName, 3);
admin.namespaces().setDeduplicationSnapshotInterval(myNamespace, 5);

int msgNum = 10;
Expand All @@ -264,7 +264,7 @@ public void testTopicPolicyTakeSnapshot() throws Exception {
assertEquals(position, markDeletedPosition);

//remove topic-level policies, namespace-level should be used, interval becomes 5 seconds
admin.topics().removeDeduplicationSnapshotInterval(topicName);
admin.topicPolicies().removeDeduplicationSnapshotInterval(topicName);
producer.newMessage().value("msg").send();
//zk update time + 5 second interval time
Awaitility.await()
Expand Down
2 changes: 2 additions & 0 deletions ...r-common/src/main/java/org/apache/pulsar/common/policies/data/HierarchyTopicPolicies.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
public class HierarchyTopicPolicies {
final PolicyHierarchyValue<List<String>> replicationClusters;
final PolicyHierarchyValue<Boolean> deduplicationEnabled;
final PolicyHierarchyValue<Integer> deduplicationSnapshotIntervalSeconds;
final PolicyHierarchyValue<InactiveTopicPolicies> inactiveTopicPolicies;
final PolicyHierarchyValue<EnumSet<SubType>> subscriptionTypesEnabled;
final PolicyHierarchyValue<Integer> maxSubscriptionsPerTopic;
Expand All @@ -51,6 +52,7 @@ public class HierarchyTopicPolicies {
public HierarchyTopicPolicies() {
replicationClusters = new PolicyHierarchyValue<>();
deduplicationEnabled = new PolicyHierarchyValue<>();
deduplicationSnapshotIntervalSeconds = new PolicyHierarchyValue<>();
inactiveTopicPolicies = new PolicyHierarchyValue<>();
subscriptionTypesEnabled = new PolicyHierarchyValue<>();
maxSubscriptionsPerTopic = new PolicyHierarchyValue<>();
Expand Down