From cf122372b2285eb324835b797d0abd577300fa98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=80=9D=E6=A5=A0?= Date: Sat, 3 Sep 2022 23:43:39 +0800 Subject: [PATCH 01/20] Fix the problem that using schema to create consumer fails after using AUTO_CONSUME consumer to subscribe topic --- .../broker/service/nonpersistent/NonPersistentTopic.java | 3 ++- .../pulsar/broker/service/persistent/PersistentTopic.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 0b5a1ed973154..6535a57bf21d8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -87,6 +87,7 @@ import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -1125,7 +1126,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem .sum(); if (hasSchema || (!producers.isEmpty()) - || (numActiveConsumers != 0) + || (numActiveConsumers != 0 && schema.getType() == SchemaType.AUTO_CONSUME) || ENTRIES_ADDED_COUNTER_UPDATER.get(this) != 0) { return checkSchemaCompatibleForConsumer(schema); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 4d63ddafd0105..66a920861e12e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -149,6 +149,7 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; @@ -2854,7 +2855,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem .sum(); if (hasSchema || (!producers.isEmpty()) - || (numActiveConsumers != 0) + || (numActiveConsumers != 0 && schema.getType() == SchemaType.AUTO_CONSUME) || (ledger.getTotalSize() != 0)) { return checkSchemaCompatibleForConsumer(schema); } else { From d1c40ffb9e0a2d01d66c556f19f0775cace7cae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=80=9D=E6=A5=A0?= Date: Sat, 3 Sep 2022 23:43:39 +0800 Subject: [PATCH 02/20] Fix the problem that using schema to create consumer fails after using AUTO_CONSUME consumer to subscribe topic --- .../broker/service/nonpersistent/NonPersistentTopic.java | 3 ++- .../pulsar/broker/service/persistent/PersistentTopic.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 0b5a1ed973154..6535a57bf21d8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -87,6 +87,7 @@ import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -1125,7 +1126,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem .sum(); if (hasSchema || (!producers.isEmpty()) - || (numActiveConsumers != 0) + || (numActiveConsumers != 0 && schema.getType() == SchemaType.AUTO_CONSUME) || ENTRIES_ADDED_COUNTER_UPDATER.get(this) != 0) { return checkSchemaCompatibleForConsumer(schema); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 4d63ddafd0105..66a920861e12e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -149,6 +149,7 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; @@ -2854,7 +2855,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem .sum(); if (hasSchema || (!producers.isEmpty()) - || (numActiveConsumers != 0) + || (numActiveConsumers != 0 && schema.getType() == SchemaType.AUTO_CONSUME) || (ledger.getTotalSize() != 0)) { return checkSchemaCompatibleForConsumer(schema); } else { From 22d6a06945f032354f634bc440fb41dd8271fe86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=80=9D=E6=A5=A0?= Date: Tue, 6 Sep 2022 20:12:51 +0800 Subject: [PATCH 03/20] add test for Fix the problem that using schema to create consumer fails after using AUTO_CONSUME consumer to subscribe topic --- .../pulsar/client/api/SimpleSchemaTest.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java index cb8b8728ceee8..6486a38961523 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java @@ -1216,6 +1216,52 @@ public void testAutoCreatedSchema(String domain) throws Exception { Assert.assertEquals(admin.schemas().getSchemaInfo(topic2).getType(), SchemaType.STRING); } + @Test(dataProvider = "topicDomain") + public void testSubscribeWithSchemaAfterAutoConsumeNewTopic(String domain) throws Exception { + final String topic = domain + "my-property/my-ns/testSubscribeWithSchemaAfterAutoConsume-1"; + + Consumer autoConsumer1 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("autoConsumer1") + .subscribe(); + + Consumer autoConsumer2 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("autoConsumer2") + .subscribe(); + try { + log.info("The autoConsumer1 isConnected: " + autoConsumer1.isConnected()); + log.info("The autoConsumer2 isConnected: " + autoConsumer2.isConnected()); + admin.schemas().getSchemaInfo(topic); + fail("The schema of topic should not exist"); + } catch (PulsarAdminException e) { + assertEquals(e.getStatusCode(), 404); + } + + Consumer consumerWithSchema = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("consumerWithSchema") + .subscribe(); + try { + log.info(admin.schemas().getSchemaInfo(topic).toString()); + log.info("The autoConsumer1 isConnected: " + autoConsumer1.isConnected()); + log.info("The autoConsumer2 isConnected: " + autoConsumer2.isConnected()); + log.info("The consumerWithSchema isConnected: " + consumerWithSchema.isConnected()); + } catch (PulsarAdminException e) { + assertEquals(e.getStatusCode(), 404); + } + + autoConsumer1.close(); + autoConsumer2.close(); + consumerWithSchema.close(); + } + @DataProvider(name = "keyEncodingType") public static Object[] keyEncodingType() { return new Object[] { KeyValueEncodingType.SEPARATED, KeyValueEncodingType.INLINE }; From 81847dfbe8c7c13e73a4b0a939523af51cb977b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=80=9D=E6=A5=A0?= Date: Sat, 3 Sep 2022 23:43:39 +0800 Subject: [PATCH 04/20] Fix the problem that using schema to create consumer fails after using AUTO_CONSUME consumer to subscribe topic --- .../broker/service/nonpersistent/NonPersistentTopic.java | 3 ++- .../pulsar/broker/service/persistent/PersistentTopic.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 7d68bb9223d59..0b66292746343 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -87,6 +87,7 @@ import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -1126,7 +1127,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem .sum(); if (hasSchema || (!producers.isEmpty()) - || (numActiveConsumers != 0) + || (numActiveConsumers != 0 && schema.getType() == SchemaType.AUTO_CONSUME) || ENTRIES_ADDED_COUNTER_UPDATER.get(this) != 0) { return checkSchemaCompatibleForConsumer(schema); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 31dda5539a3e2..bbf77c8d3446f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -149,6 +149,7 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; @@ -2854,7 +2855,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem .sum(); if (hasSchema || (!producers.isEmpty()) - || (numActiveConsumers != 0) + || (numActiveConsumers != 0 && schema.getType() == SchemaType.AUTO_CONSUME) || (ledger.getTotalSize() != 0)) { return checkSchemaCompatibleForConsumer(schema); } else { From 350bef1e1649c61b7f8bb62c360903ee862314dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=80=9D=E6=A5=A0?= Date: Tue, 6 Sep 2022 20:12:51 +0800 Subject: [PATCH 05/20] add test for Fix the problem that using schema to create consumer fails after using AUTO_CONSUME consumer to subscribe topic --- .../pulsar/client/api/SimpleSchemaTest.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java index cb8b8728ceee8..6486a38961523 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java @@ -1216,6 +1216,52 @@ public void testAutoCreatedSchema(String domain) throws Exception { Assert.assertEquals(admin.schemas().getSchemaInfo(topic2).getType(), SchemaType.STRING); } + @Test(dataProvider = "topicDomain") + public void testSubscribeWithSchemaAfterAutoConsumeNewTopic(String domain) throws Exception { + final String topic = domain + "my-property/my-ns/testSubscribeWithSchemaAfterAutoConsume-1"; + + Consumer autoConsumer1 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("autoConsumer1") + .subscribe(); + + Consumer autoConsumer2 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("autoConsumer2") + .subscribe(); + try { + log.info("The autoConsumer1 isConnected: " + autoConsumer1.isConnected()); + log.info("The autoConsumer2 isConnected: " + autoConsumer2.isConnected()); + admin.schemas().getSchemaInfo(topic); + fail("The schema of topic should not exist"); + } catch (PulsarAdminException e) { + assertEquals(e.getStatusCode(), 404); + } + + Consumer consumerWithSchema = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("consumerWithSchema") + .subscribe(); + try { + log.info(admin.schemas().getSchemaInfo(topic).toString()); + log.info("The autoConsumer1 isConnected: " + autoConsumer1.isConnected()); + log.info("The autoConsumer2 isConnected: " + autoConsumer2.isConnected()); + log.info("The consumerWithSchema isConnected: " + consumerWithSchema.isConnected()); + } catch (PulsarAdminException e) { + assertEquals(e.getStatusCode(), 404); + } + + autoConsumer1.close(); + autoConsumer2.close(); + consumerWithSchema.close(); + } + @DataProvider(name = "keyEncodingType") public static Object[] keyEncodingType() { return new Object[] { KeyValueEncodingType.SEPARATED, KeyValueEncodingType.INLINE }; From 87b7b5540dd87d6fe08d5acceeecc9990414d71e Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Fri, 11 Nov 2022 23:31:42 +0800 Subject: [PATCH 06/20] Record the schema of all consumers for each subscription after the consumer successfully subscribes to the topic. --- .../pulsar/broker/service/ServerCnx.java | 9 +-- .../apache/pulsar/broker/service/Topic.java | 8 ++- .../nonpersistent/NonPersistentTopic.java | 42 +++++++++-- .../service/persistent/PersistentTopic.java | 72 +++++++++++++------ .../pulsar/client/api/SimpleSchemaTest.java | 20 ++++-- 5 files changed, 110 insertions(+), 41 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 70f2e2bc40d07..e73b2da7c2cb3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1113,11 +1113,12 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { .subscriptionProperties(subscriptionProperties) .consumerEpoch(consumerEpoch) .build(); - if (schema != null) { - return topic.addSchemaIfIdleOrCheckCompatible(schema) - .thenCompose(v -> topic.subscribe(option)); + if (schema != null && schema.getType() != SchemaType.AUTO_PUBLISH + && schema.getType() != SchemaType.AUTO_CONSUME) { + return topic.addSchemaIfIdleOrCheckCompatible(schema, subscriptionName) + .thenCompose(v -> topic.subscribe(option, schema.getType())); } else { - return topic.subscribe(option); + return topic.subscribe(option, null == schema ? null : schema.getType()); } }) .thenAccept(consumer -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java index b4f27adcc4a31..311b198e12fac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java @@ -44,6 +44,7 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.protocol.schema.SchemaVersion; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.utils.StatsOutputStream; @@ -158,14 +159,15 @@ CompletableFuture subscribe(TransportCnx cnx, String subscriptionName, Map metadata, boolean readCompacted, InitialPosition initialPosition, long startMessageRollbackDurationSec, boolean replicateSubscriptionState, - KeySharedMeta keySharedMeta); + KeySharedMeta keySharedMeta, + SchemaType schemaType); /** * Subscribe a topic. * @param option * @return */ - CompletableFuture subscribe(SubscriptionOption option); + CompletableFuture subscribe(SubscriptionOption option, SchemaType schemaType); CompletableFuture createSubscription(String subscriptionName, InitialPosition initialPosition, boolean replicateSubscriptionState, Map properties); @@ -287,7 +289,7 @@ CompletableFuture asyncGetStats(boolean getPreciseBack * add the passed schema to the topic. Otherwise, check that the passed schema is compatible * with what the topic already has. */ - CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema); + CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema, String subscriptionName); CompletableFuture deleteForcefully(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 0b66292746343..199b5ddc67eb8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -90,6 +90,7 @@ import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.utils.StatsOutputStream; @@ -103,6 +104,8 @@ public class NonPersistentTopic extends AbstractTopic implements Topic, TopicPol private final ConcurrentOpenHashMap replicators; + private final ConcurrentOpenHashMap> subscriptionsOnlyIncludedAutoSchema; + // Ever increasing counter of entries added private static final AtomicLongFieldUpdater ENTRIES_ADDED_COUNTER_UPDATER = AtomicLongFieldUpdater.newUpdater(NonPersistentTopic.class, "entriesAddedCounter"); @@ -151,6 +154,8 @@ public NonPersistentTopic(String topic, BrokerService brokerService) { .expectedItems(16) .concurrencyLevel(1) .build(); + this.subscriptionsOnlyIncludedAutoSchema = + ConcurrentOpenHashMap.>newBuilder().build(); this.isFenced = false; registerTopicPolicyListener(); } @@ -240,13 +245,13 @@ public CompletableFuture checkIfTransactionBufferRecoverCompletely(boolean } @Override - public CompletableFuture subscribe(SubscriptionOption option) { + public CompletableFuture subscribe(SubscriptionOption option, SchemaType schemaType) { return internalSubscribe(option.getCnx(), option.getSubscriptionName(), option.getConsumerId(), option.getSubType(), option.getPriorityLevel(), option.getConsumerName(), option.isDurable(), option.getStartMessageId(), option.getMetadata(), option.isReadCompacted(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), - option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(null)); + option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(null), schemaType); } @Override @@ -256,10 +261,11 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs Map metadata, boolean readCompacted, InitialPosition initialPosition, long resetStartMessageBackInSec, boolean replicateSubscriptionState, - KeySharedMeta keySharedMeta) { + KeySharedMeta keySharedMeta, + SchemaType schemaType) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, resetStartMessageBackInSec, - replicateSubscriptionState, keySharedMeta, null); + replicateSubscriptionState, keySharedMeta, null, schemaType); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -270,7 +276,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St long resetStartMessageBackInSec, boolean replicateSubscriptionState, KeySharedMeta keySharedMeta, - Map subscriptionProperties) { + Map subscriptionProperties, + SchemaType schemaType) { return brokerService.checkTopicNsOwnership(getName()).thenCompose(__ -> { final CompletableFuture future = new CompletableFuture<>(); @@ -339,6 +346,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St future.completeExceptionally( new BrokerServiceException("Connection was closed while the opening the cursor ")); } else { + addConsumerSchemaTypeForSubscriptions(subscriptionName, schemaType); log.info("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId); future.complete(consumer); } @@ -1120,14 +1128,14 @@ public CompletableFuture getLastMessageId() { private static final Logger log = LoggerFactory.getLogger(NonPersistentTopic.class); @Override - public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema) { + public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema, String subscriptionName) { return hasSchema().thenCompose((hasSchema) -> { int numActiveConsumers = subscriptions.values().stream() .mapToInt(subscription -> subscription.getConsumers().size()) .sum(); if (hasSchema || (!producers.isEmpty()) - || (numActiveConsumers != 0 && schema.getType() == SchemaType.AUTO_CONSUME) + || (numActiveConsumers != 0 && !subscriptionsOnlyIncludedAutoSchema.containsKey(subscriptionName)) || ENTRIES_ADDED_COUNTER_UPDATER.get(this) != 0) { return checkSchemaCompatibleForConsumer(schema); } else { @@ -1136,6 +1144,26 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem }); } + private synchronized void addConsumerSchemaTypeForSubscriptions(String subscriptionName, SchemaType schemaType) { + if (null == schemaType) { + schemaType = SchemaType.AUTO_CONSUME; + } + + ConcurrentOpenHashSet consumerSchemaTypeSet = + ConcurrentOpenHashSet.newBuilder().build(); + if (subscriptionsOnlyIncludedAutoSchema.containsKey(subscriptionName)) { + consumerSchemaTypeSet = subscriptionsOnlyIncludedAutoSchema.get(subscriptionName); + } + consumerSchemaTypeSet.add(schemaType); + subscriptionsOnlyIncludedAutoSchema.put(subscriptionName, consumerSchemaTypeSet); + + // map saves only the subscription that contains only AUTO Schema + if (consumerSchemaTypeSet.size() > 1 || (consumerSchemaTypeSet.size() == 1 + && !consumerSchemaTypeSet.contains(SchemaType.AUTO_CONSUME))) { + subscriptionsOnlyIncludedAutoSchema.remove(subscriptionName); + } + } + @Override public void publishTxnMessage(TxnID txnID, ByteBuf headersAndPayload, PublishContext publishContext) { throw new UnsupportedOperationException("PublishTxnMessage is not supported by non-persistent topic"); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index bbf77c8d3446f..75c3e75ebf00c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -154,6 +154,7 @@ import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.apache.pulsar.compaction.CompactedTopic; import org.apache.pulsar.compaction.CompactedTopicContext; import org.apache.pulsar.compaction.CompactedTopicImpl; @@ -175,6 +176,8 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal private final ConcurrentOpenHashMap replicators; + private final ConcurrentOpenHashMap> subscriptionsOnlyIncludedAutoSchema; + static final String DEDUPLICATION_CURSOR_NAME = "pulsar.dedup"; private static final String TOPIC_EPOCH_PROPERTY_NAME = "pulsar.topic.epoch"; @@ -260,6 +263,8 @@ public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerS .expectedItems(16) .concurrencyLevel(1) .build(); + this.subscriptionsOnlyIncludedAutoSchema = + ConcurrentOpenHashMap.>newBuilder().build(); this.backloggedCursorThresholdEntries = brokerService.pulsar().getConfiguration().getManagedLedgerCursorBackloggedThreshold(); registerTopicPolicyListener(); @@ -361,6 +366,8 @@ public CompletableFuture initialize() { .expectedItems(16) .concurrencyLevel(1) .build(); + this.subscriptionsOnlyIncludedAutoSchema = + ConcurrentOpenHashMap.>newBuilder().build(); this.compactedTopic = new CompactedTopicImpl(brokerService.pulsar().getBookKeeperClient()); this.backloggedCursorThresholdEntries = brokerService.pulsar().getConfiguration().getManagedLedgerCursorBackloggedThreshold(); @@ -671,13 +678,14 @@ protected void handleProducerRemoved(Producer producer) { } @Override - public CompletableFuture subscribe(SubscriptionOption option) { + public CompletableFuture subscribe(SubscriptionOption option, SchemaType schemaType) { return internalSubscribe(option.getCnx(), option.getSubscriptionName(), option.getConsumerId(), option.getSubType(), option.getPriorityLevel(), option.getConsumerName(), option.isDurable(), option.getStartMessageId(), option.getMetadata(), option.isReadCompacted(), option.getInitialPosition(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), option.getKeySharedMeta(), - option.getSubscriptionProperties().orElse(Collections.emptyMap()), option.getConsumerEpoch()); + option.getSubscriptionProperties().orElse(Collections.emptyMap()), + option.getConsumerEpoch(), schemaType); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -690,7 +698,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St boolean replicatedSubscriptionStateArg, KeySharedMeta keySharedMeta, Map subscriptionProperties, - long consumerEpoch) { + long consumerEpoch, + SchemaType schemaType) { if (readCompacted && !(subType == SubType.Failover || subType == SubType.Exclusive)) { return FutureUtil.failedFuture(new NotAllowedException( "readCompacted only allowed on failover or exclusive subscriptions")); @@ -805,6 +814,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St return FutureUtil.failedFuture( new BrokerServiceException("Connection was closed while the opening the cursor ")); } else { + addConsumerSchemaTypeForSubscriptions(subscriptionName, schemaType); checkReplicatedSubscriptionControllerState(); if (log.isDebugEnabled()) { log.debug("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId); @@ -852,10 +862,11 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs InitialPosition initialPosition, long startMessageRollbackDurationSec, boolean replicatedSubscriptionStateArg, - KeySharedMeta keySharedMeta) { + KeySharedMeta keySharedMeta, + SchemaType schemaType) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, initialPosition, startMessageRollbackDurationSec, - replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH); + replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH, schemaType); } private CompletableFuture getDurableSubscription(String subscriptionName, @@ -2847,22 +2858,41 @@ public synchronized OffloadProcessStatus offloadStatus() { private static final Logger log = LoggerFactory.getLogger(PersistentTopic.class); @Override - public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema) { - return hasSchema() - .thenCompose((hasSchema) -> { - int numActiveConsumers = subscriptions.values().stream() - .mapToInt(subscription -> subscription.getConsumers().size()) - .sum(); - if (hasSchema - || (!producers.isEmpty()) - || (numActiveConsumers != 0 && schema.getType() == SchemaType.AUTO_CONSUME) - || (ledger.getTotalSize() != 0)) { - return checkSchemaCompatibleForConsumer(schema); - } else { - return addSchema(schema).thenCompose(schemaVersion -> - CompletableFuture.completedFuture(null)); - } - }); + public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema, String subscriptionName) { + return hasSchema().thenCompose((hasSchema) -> { + int numActiveConsumers = subscriptions.values().stream() + .mapToInt(subscription -> subscription.getConsumers().size()) + .sum(); + if (hasSchema + || (!producers.isEmpty()) + || (numActiveConsumers != 0 && !subscriptionsOnlyIncludedAutoSchema.containsKey(subscriptionName)) + || (ledger.getTotalSize() != 0)) { + return checkSchemaCompatibleForConsumer(schema); + } else { + return addSchema(schema).thenCompose(schemaVersion -> + CompletableFuture.completedFuture(null)); + } + }); + } + + private synchronized void addConsumerSchemaTypeForSubscriptions(String subscriptionName, SchemaType schemaType) { + if (null == schemaType) { + schemaType = SchemaType.AUTO_CONSUME; + } + + ConcurrentOpenHashSet consumerSchemaTypeSet = + ConcurrentOpenHashSet.newBuilder().build(); + if (subscriptionsOnlyIncludedAutoSchema.containsKey(subscriptionName)) { + consumerSchemaTypeSet = subscriptionsOnlyIncludedAutoSchema.get(subscriptionName); + } + consumerSchemaTypeSet.add(schemaType); + subscriptionsOnlyIncludedAutoSchema.put(subscriptionName, consumerSchemaTypeSet); + + // map saves only the subscription that contains only AUTO Schema + if (consumerSchemaTypeSet.size() > 1 || (consumerSchemaTypeSet.size() == 1 + && !consumerSchemaTypeSet.contains(SchemaType.AUTO_CONSUME))) { + subscriptionsOnlyIncludedAutoSchema.remove(subscriptionName); + } } public synchronized void checkReplicatedSubscriptionControllerState() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java index 6486a38961523..0eb1c08c5abde 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java @@ -1220,14 +1220,14 @@ public void testAutoCreatedSchema(String domain) throws Exception { public void testSubscribeWithSchemaAfterAutoConsumeNewTopic(String domain) throws Exception { final String topic = domain + "my-property/my-ns/testSubscribeWithSchemaAfterAutoConsume-1"; - Consumer autoConsumer1 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + Consumer autoConsumer1 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) .topic(topic) .subscriptionType(SubscriptionType.Shared) .subscriptionName("sub0") .consumerName("autoConsumer1") .subscribe(); - Consumer autoConsumer2 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + Consumer autoConsumer2 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) .topic(topic) .subscriptionType(SubscriptionType.Shared) .subscriptionName("sub0") @@ -1242,24 +1242,32 @@ public void testSubscribeWithSchemaAfterAutoConsumeNewTopic(String domain) throw assertEquals(e.getStatusCode(), 404); } - Consumer consumerWithSchema = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + Consumer consumerWithSchema1 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) .topic(topic) .subscriptionType(SubscriptionType.Shared) .subscriptionName("sub0") - .consumerName("consumerWithSchema") + .consumerName("consumerWithSchema-1") + .subscribe(); + Consumer consumerWithSchema2 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub0") + .consumerName("consumerWithSchema-2") .subscribe(); try { log.info(admin.schemas().getSchemaInfo(topic).toString()); log.info("The autoConsumer1 isConnected: " + autoConsumer1.isConnected()); log.info("The autoConsumer2 isConnected: " + autoConsumer2.isConnected()); - log.info("The consumerWithSchema isConnected: " + consumerWithSchema.isConnected()); + log.info("The consumerWithSchema-1 isConnected: " + consumerWithSchema1.isConnected()); + log.info("The consumerWithSchema-2 isConnected: " + consumerWithSchema2.isConnected()); } catch (PulsarAdminException e) { assertEquals(e.getStatusCode(), 404); } autoConsumer1.close(); autoConsumer2.close(); - consumerWithSchema.close(); + consumerWithSchema1.close(); + consumerWithSchema2.close(); } @DataProvider(name = "keyEncodingType") From 7be59c351509f5586552b6962e3f834a9001730d Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Mon, 12 Dec 2022 22:51:28 +0800 Subject: [PATCH 07/20] when the schema is null, do not check whether the schema is AUTO_CONSUME. --- .../main/java/org/apache/pulsar/broker/service/ServerCnx.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index e73b2da7c2cb3..f4df9dd441b9a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1113,8 +1113,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { .subscriptionProperties(subscriptionProperties) .consumerEpoch(consumerEpoch) .build(); - if (schema != null && schema.getType() != SchemaType.AUTO_PUBLISH - && schema.getType() != SchemaType.AUTO_CONSUME) { + if (schema != null) { return topic.addSchemaIfIdleOrCheckCompatible(schema, subscriptionName) .thenCompose(v -> topic.subscribe(option, schema.getType())); } else { From 5f3a73ad33489055ce5ccebdf11e62f8010993c2 Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Tue, 20 Dec 2022 13:24:11 +0800 Subject: [PATCH 08/20] Record schemaData in Consumer. --- .../pulsar/broker/service/Consumer.java | 17 +++- .../pulsar/broker/service/ServerCnx.java | 6 +- .../apache/pulsar/broker/service/Topic.java | 7 +- .../nonpersistent/NonPersistentTopic.java | 48 +++-------- .../service/persistent/PersistentTopic.java | 50 +++-------- .../pulsar/broker/service/ConsumerTest.java | 2 +- .../service/MessageCumulativeAckTest.java | 6 +- ...sistentDispatcherFailoverConsumerTest.java | 18 ++-- .../PersistentTopicConcurrentTest.java | 8 +- .../broker/service/PersistentTopicTest.java | 85 ++++++++++--------- .../pulsar/client/api/SimpleSchemaTest.java | 32 +++++++ 11 files changed, 142 insertions(+), 137 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 7eae83ba25031..abb00a9b13b8d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -56,6 +56,8 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.stats.Rate; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; @@ -140,11 +142,13 @@ public class Consumer { private long negtiveUnackedMsgsTimestamp; + private SchemaData schemaData; + public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId, int priorityLevel, String consumerName, boolean isDurable, TransportCnx cnx, String appId, Map metadata, boolean readCompacted, - KeySharedMeta keySharedMeta, MessageId startMessageId, long consumerEpoch) { + KeySharedMeta keySharedMeta, MessageId startMessageId, long consumerEpoch, SchemaData schemaData) { this.subscription = subscription; this.subType = subType; @@ -202,6 +206,12 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.consumerEpoch = consumerEpoch; this.isAcknowledgmentAtBatchIndexLevelEnabled = subscription.getTopic().getBrokerService() .getPulsar().getConfiguration().isAcknowledgmentAtBatchIndexLevelEnabled(); + + if (schemaData == null) { + this.schemaData = SchemaData.builder().type(SchemaType.AUTO_CONSUME).build(); + } else { + this.schemaData = schemaData; + } } @VisibleForTesting @@ -229,6 +239,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.clientAddress = null; this.startMessageId = null; this.isAcknowledgmentAtBatchIndexLevelEnabled = false; + this.schemaData = null; MESSAGE_PERMITS_UPDATER.set(this, availablePermits); } @@ -1086,6 +1097,10 @@ public Map getMetadata() { return metadata; } + public SchemaData getSchemaData() { + return schemaData; + } + private int getStickyKeyHash(Entry entry) { final byte[] stickyKey; if (entry instanceof EntryAndMetadata) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index f4df9dd441b9a..758c062a8f4ed 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1114,10 +1114,10 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { .consumerEpoch(consumerEpoch) .build(); if (schema != null) { - return topic.addSchemaIfIdleOrCheckCompatible(schema, subscriptionName) - .thenCompose(v -> topic.subscribe(option, schema.getType())); + return topic.addSchemaIfIdleOrCheckCompatible(schema) + .thenCompose(v -> topic.subscribe(option, schema)); } else { - return topic.subscribe(option, null == schema ? null : schema.getType()); + return topic.subscribe(option, schema); } }) .thenAccept(consumer -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java index 311b198e12fac..54ce3cdec53fa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java @@ -44,7 +44,6 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.protocol.schema.SchemaVersion; -import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.utils.StatsOutputStream; @@ -160,14 +159,14 @@ CompletableFuture subscribe(TransportCnx cnx, String subscriptionName, InitialPosition initialPosition, long startMessageRollbackDurationSec, boolean replicateSubscriptionState, KeySharedMeta keySharedMeta, - SchemaType schemaType); + SchemaData schemaData); /** * Subscribe a topic. * @param option * @return */ - CompletableFuture subscribe(SubscriptionOption option, SchemaType schemaType); + CompletableFuture subscribe(SubscriptionOption option, SchemaData schemaData); CompletableFuture createSubscription(String subscriptionName, InitialPosition initialPosition, boolean replicateSubscriptionState, Map properties); @@ -289,7 +288,7 @@ CompletableFuture asyncGetStats(boolean getPreciseBack * add the passed schema to the topic. Otherwise, check that the passed schema is compatible * with what the topic already has. */ - CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema, String subscriptionName); + CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema); CompletableFuture deleteForcefully(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 199b5ddc67eb8..2e40108993424 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -90,7 +90,6 @@ import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.utils.StatsOutputStream; @@ -104,8 +103,6 @@ public class NonPersistentTopic extends AbstractTopic implements Topic, TopicPol private final ConcurrentOpenHashMap replicators; - private final ConcurrentOpenHashMap> subscriptionsOnlyIncludedAutoSchema; - // Ever increasing counter of entries added private static final AtomicLongFieldUpdater ENTRIES_ADDED_COUNTER_UPDATER = AtomicLongFieldUpdater.newUpdater(NonPersistentTopic.class, "entriesAddedCounter"); @@ -154,8 +151,6 @@ public NonPersistentTopic(String topic, BrokerService brokerService) { .expectedItems(16) .concurrencyLevel(1) .build(); - this.subscriptionsOnlyIncludedAutoSchema = - ConcurrentOpenHashMap.>newBuilder().build(); this.isFenced = false; registerTopicPolicyListener(); } @@ -245,13 +240,13 @@ public CompletableFuture checkIfTransactionBufferRecoverCompletely(boolean } @Override - public CompletableFuture subscribe(SubscriptionOption option, SchemaType schemaType) { + public CompletableFuture subscribe(SubscriptionOption option, SchemaData schemaData) { return internalSubscribe(option.getCnx(), option.getSubscriptionName(), option.getConsumerId(), option.getSubType(), option.getPriorityLevel(), option.getConsumerName(), option.isDurable(), option.getStartMessageId(), option.getMetadata(), option.isReadCompacted(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), - option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(null), schemaType); + option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(null), schemaData); } @Override @@ -262,10 +257,10 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs InitialPosition initialPosition, long resetStartMessageBackInSec, boolean replicateSubscriptionState, KeySharedMeta keySharedMeta, - SchemaType schemaType) { + SchemaData schemaData) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, resetStartMessageBackInSec, - replicateSubscriptionState, keySharedMeta, null, schemaType); + replicateSubscriptionState, keySharedMeta, null, schemaData); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -277,7 +272,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St boolean replicateSubscriptionState, KeySharedMeta keySharedMeta, Map subscriptionProperties, - SchemaType schemaType) { + SchemaData schemaData) { return brokerService.checkTopicNsOwnership(getName()).thenCompose(__ -> { final CompletableFuture future = new CompletableFuture<>(); @@ -321,7 +316,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, - MessageId.latest, DEFAULT_CONSUMER_EPOCH); + MessageId.latest, DEFAULT_CONSUMER_EPOCH, schemaData); addConsumerToSubscription(subscription, consumer).thenRun(() -> { if (!cnx.isActive()) { @@ -346,7 +341,6 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St future.completeExceptionally( new BrokerServiceException("Connection was closed while the opening the cursor ")); } else { - addConsumerSchemaTypeForSubscriptions(subscriptionName, schemaType); log.info("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId); future.complete(consumer); } @@ -1128,14 +1122,16 @@ public CompletableFuture getLastMessageId() { private static final Logger log = LoggerFactory.getLogger(NonPersistentTopic.class); @Override - public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema, String subscriptionName) { + public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema) { return hasSchema().thenCompose((hasSchema) -> { - int numActiveConsumers = subscriptions.values().stream() - .mapToInt(subscription -> subscription.getConsumers().size()) + int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() + .mapToInt(subscription -> subscription.getConsumers().stream() + .filter(consumer -> consumer.getSchemaData().getType() != SchemaType.AUTO_CONSUME) + .toList().size()) .sum(); if (hasSchema || (!producers.isEmpty()) - || (numActiveConsumers != 0 && !subscriptionsOnlyIncludedAutoSchema.containsKey(subscriptionName)) + || (numActiveConsumersWithoutAutoSchema != 0) || ENTRIES_ADDED_COUNTER_UPDATER.get(this) != 0) { return checkSchemaCompatibleForConsumer(schema); } else { @@ -1144,26 +1140,6 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem }); } - private synchronized void addConsumerSchemaTypeForSubscriptions(String subscriptionName, SchemaType schemaType) { - if (null == schemaType) { - schemaType = SchemaType.AUTO_CONSUME; - } - - ConcurrentOpenHashSet consumerSchemaTypeSet = - ConcurrentOpenHashSet.newBuilder().build(); - if (subscriptionsOnlyIncludedAutoSchema.containsKey(subscriptionName)) { - consumerSchemaTypeSet = subscriptionsOnlyIncludedAutoSchema.get(subscriptionName); - } - consumerSchemaTypeSet.add(schemaType); - subscriptionsOnlyIncludedAutoSchema.put(subscriptionName, consumerSchemaTypeSet); - - // map saves only the subscription that contains only AUTO Schema - if (consumerSchemaTypeSet.size() > 1 || (consumerSchemaTypeSet.size() == 1 - && !consumerSchemaTypeSet.contains(SchemaType.AUTO_CONSUME))) { - subscriptionsOnlyIncludedAutoSchema.remove(subscriptionName); - } - } - @Override public void publishTxnMessage(TxnID txnID, ByteBuf headersAndPayload, PublishContext publishContext) { throw new UnsupportedOperationException("PublishTxnMessage is not supported by non-persistent topic"); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 75c3e75ebf00c..ab297afc5723c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -154,7 +154,6 @@ import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet; import org.apache.pulsar.compaction.CompactedTopic; import org.apache.pulsar.compaction.CompactedTopicContext; import org.apache.pulsar.compaction.CompactedTopicImpl; @@ -176,8 +175,6 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal private final ConcurrentOpenHashMap replicators; - private final ConcurrentOpenHashMap> subscriptionsOnlyIncludedAutoSchema; - static final String DEDUPLICATION_CURSOR_NAME = "pulsar.dedup"; private static final String TOPIC_EPOCH_PROPERTY_NAME = "pulsar.topic.epoch"; @@ -263,8 +260,6 @@ public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerS .expectedItems(16) .concurrencyLevel(1) .build(); - this.subscriptionsOnlyIncludedAutoSchema = - ConcurrentOpenHashMap.>newBuilder().build(); this.backloggedCursorThresholdEntries = brokerService.pulsar().getConfiguration().getManagedLedgerCursorBackloggedThreshold(); registerTopicPolicyListener(); @@ -366,8 +361,6 @@ public CompletableFuture initialize() { .expectedItems(16) .concurrencyLevel(1) .build(); - this.subscriptionsOnlyIncludedAutoSchema = - ConcurrentOpenHashMap.>newBuilder().build(); this.compactedTopic = new CompactedTopicImpl(brokerService.pulsar().getBookKeeperClient()); this.backloggedCursorThresholdEntries = brokerService.pulsar().getConfiguration().getManagedLedgerCursorBackloggedThreshold(); @@ -678,14 +671,14 @@ protected void handleProducerRemoved(Producer producer) { } @Override - public CompletableFuture subscribe(SubscriptionOption option, SchemaType schemaType) { + public CompletableFuture subscribe(SubscriptionOption option, SchemaData schemaData) { return internalSubscribe(option.getCnx(), option.getSubscriptionName(), option.getConsumerId(), option.getSubType(), option.getPriorityLevel(), option.getConsumerName(), option.isDurable(), option.getStartMessageId(), option.getMetadata(), option.isReadCompacted(), option.getInitialPosition(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(Collections.emptyMap()), - option.getConsumerEpoch(), schemaType); + option.getConsumerEpoch(), schemaData); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -699,7 +692,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St KeySharedMeta keySharedMeta, Map subscriptionProperties, long consumerEpoch, - SchemaType schemaType) { + SchemaData schemaData) { if (readCompacted && !(subType == SubType.Failover || subType == SubType.Exclusive)) { return FutureUtil.failedFuture(new NotAllowedException( "readCompacted only allowed on failover or exclusive subscriptions")); @@ -787,7 +780,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St CompletableFuture future = subscriptionFuture.thenCompose(subscription -> { Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, isDurable, cnx, cnx.getAuthRole(), metadata, - readCompacted, keySharedMeta, startMessageId, consumerEpoch); + readCompacted, keySharedMeta, startMessageId, consumerEpoch, schemaData); return addConsumerToSubscription(subscription, consumer).thenCompose(v -> { checkBackloggedCursors(); @@ -814,7 +807,6 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St return FutureUtil.failedFuture( new BrokerServiceException("Connection was closed while the opening the cursor ")); } else { - addConsumerSchemaTypeForSubscriptions(subscriptionName, schemaType); checkReplicatedSubscriptionControllerState(); if (log.isDebugEnabled()) { log.debug("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId); @@ -863,10 +855,10 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs long startMessageRollbackDurationSec, boolean replicatedSubscriptionStateArg, KeySharedMeta keySharedMeta, - SchemaType schemaType) { + SchemaData schemaData) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, initialPosition, startMessageRollbackDurationSec, - replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH, schemaType); + replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH, schemaData); } private CompletableFuture getDurableSubscription(String subscriptionName, @@ -2858,14 +2850,16 @@ public synchronized OffloadProcessStatus offloadStatus() { private static final Logger log = LoggerFactory.getLogger(PersistentTopic.class); @Override - public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema, String subscriptionName) { + public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schema) { return hasSchema().thenCompose((hasSchema) -> { - int numActiveConsumers = subscriptions.values().stream() - .mapToInt(subscription -> subscription.getConsumers().size()) + int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() + .mapToInt(subscription -> subscription.getConsumers().stream() + .filter(consumer -> consumer.getSchemaData().getType() != SchemaType.AUTO_CONSUME) + .toList().size()) .sum(); if (hasSchema || (!producers.isEmpty()) - || (numActiveConsumers != 0 && !subscriptionsOnlyIncludedAutoSchema.containsKey(subscriptionName)) + || (numActiveConsumersWithoutAutoSchema != 0) || (ledger.getTotalSize() != 0)) { return checkSchemaCompatibleForConsumer(schema); } else { @@ -2875,26 +2869,6 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem }); } - private synchronized void addConsumerSchemaTypeForSubscriptions(String subscriptionName, SchemaType schemaType) { - if (null == schemaType) { - schemaType = SchemaType.AUTO_CONSUME; - } - - ConcurrentOpenHashSet consumerSchemaTypeSet = - ConcurrentOpenHashSet.newBuilder().build(); - if (subscriptionsOnlyIncludedAutoSchema.containsKey(subscriptionName)) { - consumerSchemaTypeSet = subscriptionsOnlyIncludedAutoSchema.get(subscriptionName); - } - consumerSchemaTypeSet.add(schemaType); - subscriptionsOnlyIncludedAutoSchema.put(subscriptionName, consumerSchemaTypeSet); - - // map saves only the subscription that contains only AUTO Schema - if (consumerSchemaTypeSet.size() > 1 || (consumerSchemaTypeSet.size() == 1 - && !consumerSchemaTypeSet.contains(SchemaType.AUTO_CONSUME))) { - subscriptionsOnlyIncludedAutoSchema.remove(subscriptionName); - } - } - public synchronized void checkReplicatedSubscriptionControllerState() { AtomicBoolean shouldBeEnabled = new AtomicBoolean(false); subscriptions.forEach((name, subscription) -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java index 1ad0642d9c5ba..f3dc9b96d8889 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java @@ -57,7 +57,7 @@ public void beforeMethod() { consumer = new Consumer(subscription, Exclusive, "topic", 1, 0, "Cons1", true, cnx, "myrole-1", emptyMap(), false, - new KeySharedMeta().setKeySharedMode(AUTO_SPLIT), latest, DEFAULT_CONSUMER_EPOCH); + new KeySharedMeta().setKeySharedMode(AUTO_SPLIT), latest, DEFAULT_CONSUMER_EPOCH, null); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java index 05e86e73038d8..79eb76f411a59 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java @@ -160,7 +160,7 @@ public static Object[][] notIndividualAckModes() { public void testAckWithIndividualAckMode(CommandSubscribe.SubType subType) throws Exception { Consumer consumer = new Consumer(sub, subType, "topic-1", consumerId, 0, "Cons1", true, serverCnx, "myrole-1", emptyMap(), false, null, - MessageId.latest, DEFAULT_CONSUMER_EPOCH); + MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); CommandAck commandAck = new CommandAck(); commandAck.setAckType(Cumulative); @@ -175,7 +175,7 @@ public void testAckWithIndividualAckMode(CommandSubscribe.SubType subType) throw public void testAckWithNotIndividualAckMode(CommandSubscribe.SubType subType) throws Exception { Consumer consumer = new Consumer(sub, subType, "topic-1", consumerId, 0, "Cons1", true, serverCnx, "myrole-1", emptyMap(), false, null, - MessageId.latest, DEFAULT_CONSUMER_EPOCH); + MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); CommandAck commandAck = new CommandAck(); commandAck.setAckType(Cumulative); @@ -190,7 +190,7 @@ public void testAckWithNotIndividualAckMode(CommandSubscribe.SubType subType) th public void testAckWithMoreThanNoneMessageIds() throws Exception { Consumer consumer = new Consumer(sub, Failover, "topic-1", consumerId, 0, "Cons1", true, serverCnx, "myrole-1", emptyMap(), false, null, - MessageId.latest, DEFAULT_CONSUMER_EPOCH); + MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); CommandAck commandAck = new CommandAck(); commandAck.setAckType(Cumulative); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java index f0c61a1fc0292..26415ee155ab0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java @@ -315,7 +315,7 @@ public void testConsumerGroupChangesWithOldNewConsumers() throws Exception { // 2. Add old consumer Consumer consumer1 = new Consumer(sub, SubType.Exclusive, topic.getName(), 1 /* consumer id */, 0, - "Cons1"/* consumer name */, true, serverCnxWithOldVersion, "myrole-1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + "Cons1"/* consumer name */, true, serverCnxWithOldVersion, "myrole-1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); pdfc.addConsumer(consumer1); List consumers = pdfc.getConsumers(); assertSame(consumers.get(0).consumerName(), consumer1.consumerName()); @@ -326,7 +326,7 @@ public void testConsumerGroupChangesWithOldNewConsumers() throws Exception { // 3. Add new consumer Consumer consumer2 = new Consumer(sub, SubType.Exclusive, topic.getName(), 2 /* consumer id */, 0, - "Cons2"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + "Cons2"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); pdfc.addConsumer(consumer2); consumers = pdfc.getConsumers(); assertSame(consumers.get(0).consumerName(), consumer1.consumerName()); @@ -355,7 +355,7 @@ public void testAddRemoveConsumer() throws Exception { // 2. Add consumer Consumer consumer1 = spy(new Consumer(sub, SubType.Exclusive, topic.getName(), 1 /* consumer id */, 0, "Cons1"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null)); pdfc.addConsumer(consumer1); List consumers = pdfc.getConsumers(); assertSame(consumers.get(0).consumerName(), consumer1.consumerName()); @@ -380,7 +380,7 @@ public void testAddRemoveConsumer() throws Exception { // 5. Add another consumer which does not change active consumer Consumer consumer2 = spy(new Consumer(sub, SubType.Exclusive, topic.getName(), 2 /* consumer id */, 0, "Cons2"/* consumer name */, - true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); + true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null)); pdfc.addConsumer(consumer2); consumers = pdfc.getConsumers(); assertSame(pdfc.getActiveConsumer().consumerName(), consumer1.consumerName()); @@ -395,7 +395,7 @@ public void testAddRemoveConsumer() throws Exception { // 6. Add a consumer which changes active consumer Consumer consumer0 = spy(new Consumer(sub, SubType.Exclusive, topic.getName(), 0 /* consumer id */, 0, "Cons0"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null)); pdfc.addConsumer(consumer0); consumers = pdfc.getConsumers(); assertSame(pdfc.getActiveConsumer().consumerName(), consumer0.consumerName()); @@ -487,7 +487,7 @@ public void testAddRemoveConsumerNonPartitionedTopic() throws Exception { // 2. Add a consumer Consumer consumer1 = spy(new Consumer(sub, SubType.Failover, topic.getName(), 1 /* consumer id */, 1, "Cons1"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null)); pdfc.addConsumer(consumer1); List consumers = pdfc.getConsumers(); assertEquals(1, consumers.size()); @@ -496,7 +496,7 @@ public void testAddRemoveConsumerNonPartitionedTopic() throws Exception { // 3. Add a consumer with same priority level and consumer name is smaller in lexicographic order. Consumer consumer2 = spy(new Consumer(sub, SubType.Failover, topic.getName(), 2 /* consumer id */, 1, "Cons2"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null)); pdfc.addConsumer(consumer2); // 4. Verify active consumer doesn't change @@ -509,7 +509,7 @@ public void testAddRemoveConsumerNonPartitionedTopic() throws Exception { // 5. Add another consumer which has higher priority level Consumer consumer3 = spy(new Consumer(sub, SubType.Failover, topic.getName(), 3 /* consumer id */, 0, "Cons3"/* consumer name */, - true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); + true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null)); pdfc.addConsumer(consumer3); consumers = pdfc.getConsumers(); assertEquals(3, consumers.size()); @@ -700,7 +700,7 @@ private Consumer createConsumer(PersistentTopic topic, int priority, int permit, PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock, false); Consumer consumer = new Consumer(sub, SubType.Shared, "test-topic", id, priority, ""+id, true, - serverCnx, "appId", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest,DEFAULT_CONSUMER_EPOCH); + serverCnx, "appId", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest,DEFAULT_CONSUMER_EPOCH, null); try { consumer.flowPermits(permit); } catch (Exception e) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java index cc987a2ddc0b7..e1e244d9aae2b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java @@ -129,7 +129,7 @@ public void testConcurrentTopicAndSubscriptionDelete() throws Exception { SubscriptionOption subscriptionOption = getSubscriptionOption(cmd); - Future f1 = topic.subscribe(subscriptionOption); + Future f1 = topic.subscribe(subscriptionOption, null); f1.get(); final CyclicBarrier barrier = new CyclicBarrier(2); @@ -187,7 +187,7 @@ public void testConcurrentTopicGCAndSubscriptionDelete() throws Exception { SubscriptionOption subscriptionOption = getSubscriptionOption(cmd); - Future f1 = topic.subscribe(subscriptionOption); + Future f1 = topic.subscribe(subscriptionOption, null); f1.get(); final CyclicBarrier barrier = new CyclicBarrier(2); @@ -251,7 +251,7 @@ public void testConcurrentTopicDeleteAndUnsubscribe() throws Exception { SubscriptionOption subscriptionOption = getSubscriptionOption(cmd); - Future f1 = topic.subscribe(subscriptionOption); + Future f1 = topic.subscribe(subscriptionOption, null); f1.get(); final CyclicBarrier barrier = new CyclicBarrier(2); @@ -309,7 +309,7 @@ public void testConcurrentTopicDeleteAndSubsUnsubscribe() throws Exception { SubscriptionOption subscriptionOption = getSubscriptionOption(cmd); - Future f1 = topic.subscribe(subscriptionOption); + Future f1 = topic.subscribe(subscriptionOption, null); f1.get(); final CyclicBarrier barrier = new CyclicBarrier(2); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 1404ce7c7988e..4eefe43e316f8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -709,7 +709,7 @@ public void testSubscribeFail() throws Exception { SubscriptionOption subscriptionOption = getSubscriptionOption(cmd); - Future f1 = topic.subscribe(subscriptionOption); + Future f1 = topic.subscribe(subscriptionOption, null); try { f1.get(); fail("should fail with exception"); @@ -744,11 +744,11 @@ public void testSubscribeUnsubscribe() throws Exception { .setSubType(SubType.Exclusive); // 1. simple subscribe - Future f1 = topic.subscribe(getSubscriptionOption(cmd)); + Future f1 = topic.subscribe(getSubscriptionOption(cmd), null); f1.get(); // 2. duplicate subscribe - Future f2 = topic.subscribe(getSubscriptionOption(cmd)); + Future f2 = topic.subscribe(getSubscriptionOption(cmd), null); try { f2.get(); fail("should fail with exception"); @@ -771,7 +771,8 @@ public void testChangeSubscriptionType() throws Exception { Consumer consumer = new Consumer(sub, SubType.Exclusive, topic.getName(), 1, 0, "Cons1", true, serverCnx, "myrole-1", Collections.emptyMap(), false, - new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT), MessageId.latest, DEFAULT_CONSUMER_EPOCH); + new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT), MessageId.latest, + DEFAULT_CONSUMER_EPOCH, null); sub.addConsumer(consumer); consumer.close(); @@ -782,7 +783,8 @@ public void testChangeSubscriptionType() throws Exception { consumer = new Consumer(sub, subType, topic.getName(), 1, 0, "Cons1", true, serverCnx, "myrole-1", Collections.emptyMap(), false, - new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT), MessageId.latest, DEFAULT_CONSUMER_EPOCH); + new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT), MessageId.latest, + DEFAULT_CONSUMER_EPOCH, null); sub.addConsumer(consumer); assertTrue(sub.getDispatcher().isConsumerConnected()); @@ -805,7 +807,8 @@ public void testAddRemoveConsumer() throws Exception { // 1. simple add consumer Consumer consumer = new Consumer(sub, SubType.Exclusive, topic.getName(), 1 /* consumer id */, 0, "Cons1"/* consumer name */, - true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, + DEFAULT_CONSUMER_EPOCH, null); sub.addConsumer(consumer); assertTrue(sub.getDispatcher().isConsumerConnected()); @@ -838,7 +841,7 @@ public void testAddRemoveConsumerDurableCursor() throws Exception { PersistentSubscription sub = new PersistentSubscription(topic, "non-durable-sub", cursorMock, false); Consumer consumer = new Consumer(sub, SubType.Exclusive, topic.getName(), 1, 0, "Cons1", true, serverCnx, - "myrole-1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + "myrole-1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); sub.addConsumer(consumer); assertFalse(sub.getDispatcher().isClosed()); @@ -880,14 +883,14 @@ private void testMaxConsumersShared() throws Exception { // 1. add consumer1 Consumer consumer = new Consumer(sub, SubType.Shared, topic.getName(), 1 /* consumer id */, 0, "Cons1"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); addConsumerToSubscription.invoke(topic, sub, consumer); assertEquals(sub.getConsumers().size(), 1); // 2. add consumer2 Consumer consumer2 = new Consumer(sub, SubType.Shared, topic.getName(), 2 /* consumer id */, 0, "Cons2"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); addConsumerToSubscription.invoke(topic, sub, consumer2); assertEquals(sub.getConsumers().size(), 2); @@ -895,7 +898,7 @@ private void testMaxConsumersShared() throws Exception { try { Consumer consumer3 = new Consumer(sub, SubType.Shared, topic.getName(), 3 /* consumer id */, 0, "Cons3"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub, consumer3)).get(); fail("should have failed"); } catch (ExecutionException e) { @@ -908,7 +911,7 @@ private void testMaxConsumersShared() throws Exception { // 4. add consumer4 to sub2 Consumer consumer4 = new Consumer(sub2, SubType.Shared, topic.getName(), 4 /* consumer id */, 0, "Cons4"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); addConsumerToSubscription.invoke(topic, sub2, consumer4); assertEquals(sub2.getConsumers().size(), 1); @@ -919,7 +922,7 @@ private void testMaxConsumersShared() throws Exception { try { Consumer consumer5 = new Consumer(sub2, SubType.Shared, topic.getName(), 5 /* consumer id */, 0, "Cons5"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer5)).get(); fail("should have failed"); } catch (ExecutionException e) { @@ -988,14 +991,14 @@ private void testMaxConsumersFailover() throws Exception { // 1. add consumer1 Consumer consumer = new Consumer(sub, SubType.Failover, topic.getName(), 1 /* consumer id */, 0, "Cons1"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); addConsumerToSubscription.invoke(topic, sub, consumer); assertEquals(sub.getConsumers().size(), 1); // 2. add consumer2 Consumer consumer2 = new Consumer(sub, SubType.Failover, topic.getName(), 2 /* consumer id */, 0, "Cons2"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); addConsumerToSubscription.invoke(topic, sub, consumer2); assertEquals(sub.getConsumers().size(), 2); @@ -1003,7 +1006,7 @@ private void testMaxConsumersFailover() throws Exception { try { Consumer consumer3 = new Consumer(sub, SubType.Failover, topic.getName(), 3 /* consumer id */, 0, "Cons3"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub, consumer3)).get(); fail("should have failed"); } catch (ExecutionException e) { @@ -1016,7 +1019,7 @@ private void testMaxConsumersFailover() throws Exception { // 4. add consumer4 to sub2 Consumer consumer4 = new Consumer(sub2, SubType.Failover, topic.getName(), 4 /* consumer id */, 0, "Cons4"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); addConsumerToSubscription.invoke(topic, sub2, consumer4); assertEquals(sub2.getConsumers().size(), 1); @@ -1027,7 +1030,7 @@ private void testMaxConsumersFailover() throws Exception { try { Consumer consumer5 = new Consumer(sub2, SubType.Failover, topic.getName(), 5 /* consumer id */, 0, "Cons5"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer5)).get(); fail("should have failed"); } catch (ExecutionException e) { @@ -1081,7 +1084,7 @@ private Consumer getMockedConsumerWithSpecificAddress(Topic topic, Subscription doReturn(new PulsarCommandSenderImpl(null, cnx)).when(cnx).getCommandSender(); return new Consumer(sub, SubType.Shared, topic.getName(), consumerId, 0, consumerNameBase + consumerId, true, - cnx, role, Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + cnx, role, Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); } @Test @@ -1193,7 +1196,8 @@ public void testUbsubscribeRaceConditions() throws Exception { PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock, false); Consumer consumer1 = new Consumer(sub, SubType.Exclusive, topic.getName(), 1 /* consumer id */, 0, "Cons1"/* consumer name */, - true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, + DEFAULT_CONSUMER_EPOCH, null); sub.addConsumer(consumer1); doAnswer(new Answer() { @@ -1217,7 +1221,8 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { Thread.sleep(10); /* delay to ensure that the ubsubscribe gets executed first */ sub.addConsumer(new Consumer(sub, SubType.Exclusive, topic.getName(), 2 /* consumer id */, 0, "Cons2"/* consumer name */, true, serverCnx, - "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)).get(); + "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, + DEFAULT_CONSUMER_EPOCH, null)).get(); fail(); } catch (Exception e) { assertTrue(e.getCause() instanceof BrokerServiceException.SubscriptionFencedException); @@ -1306,7 +1311,7 @@ public void testDeleteTopic() throws Exception { .setRequestId(1) .setSubType(SubType.Exclusive); - Future f1 = topic.subscribe(getSubscriptionOption(cmd)); + Future f1 = topic.subscribe(getSubscriptionOption(cmd), null); f1.get(); assertTrue(topic.delete().isCompletedExceptionally()); @@ -1328,7 +1333,7 @@ public void testDeleteAndUnsubscribeTopic() throws Exception { .setReadCompacted(false) .setSubType(SubType.Exclusive); - Future f1 = topic.subscribe(getSubscriptionOption(cmd)); + Future f1 = topic.subscribe(getSubscriptionOption(cmd), null); f1.get(); final CyclicBarrier barrier = new CyclicBarrier(2); @@ -1385,7 +1390,7 @@ public void testConcurrentTopicAndSubscriptionDelete() throws Exception { .setRequestId(1) .setSubType(SubType.Exclusive); - Future f1 = topic.subscribe(getSubscriptionOption(cmd)); + Future f1 = topic.subscribe(getSubscriptionOption(cmd), null); f1.get(); @@ -1479,7 +1484,7 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { .setRequestId(1) .setSubType(SubType.Exclusive); - Future f = topic.subscribe(getSubscriptionOption(cmd)); + Future f = topic.subscribe(getSubscriptionOption(cmd), null); try { f.get(); fail("should have failed"); @@ -1620,7 +1625,7 @@ public void testFailoverSubscription() throws Exception { .setSubType(SubType.Failover); // 1. Subscribe with non partition topic - Future f1 = topic1.subscribe(getSubscriptionOption(cmd1)); + Future f1 = topic1.subscribe(getSubscriptionOption(cmd1), null); f1.get(); // 2. Subscribe with partition topic @@ -1635,7 +1640,7 @@ public void testFailoverSubscription() throws Exception { .setRequestId(1) .setSubType(SubType.Failover); - Future f2 = topic2.subscribe(getSubscriptionOption(cmd2)); + Future f2 = topic2.subscribe(getSubscriptionOption(cmd2), null); f2.get(); // 3. Subscribe and create second consumer @@ -1648,7 +1653,7 @@ public void testFailoverSubscription() throws Exception { .setRequestId(1) .setSubType(SubType.Failover); - Future f3 = topic2.subscribe(getSubscriptionOption(cmd3)); + Future f3 = topic2.subscribe(getSubscriptionOption(cmd3), null); f3.get(); assertEquals( @@ -1672,7 +1677,7 @@ public void testFailoverSubscription() throws Exception { .setRequestId(1) .setSubType(SubType.Failover); - Future f4 = topic2.subscribe(getSubscriptionOption(cmd4)); + Future f4 = topic2.subscribe(getSubscriptionOption(cmd4), null); f4.get(); assertEquals( @@ -1701,7 +1706,7 @@ public void testFailoverSubscription() throws Exception { .setRequestId(1) .setSubType(SubType.Exclusive); - Future f5 = topic2.subscribe(getSubscriptionOption(cmd5)); + Future f5 = topic2.subscribe(getSubscriptionOption(cmd5), null); try { f5.get(); fail("should fail with exception"); @@ -1720,7 +1725,7 @@ public void testFailoverSubscription() throws Exception { .setRequestId(1) .setSubType(SubType.Exclusive); - Future f6 = topic2.subscribe(getSubscriptionOption(cmd6)); + Future f6 = topic2.subscribe(getSubscriptionOption(cmd6), null); f6.get(); // 7. unsubscribe exclusive sub @@ -1995,21 +2000,24 @@ public void testBacklogCursor() throws Exception { ManagedCursor cursor1 = ledger.openCursor("c1"); PersistentSubscription sub1 = new PersistentSubscription(topic, "sub-1", cursor1, false); Consumer consumer1 = new Consumer(sub1, SubType.Exclusive, topic.getName(), 1 /* consumer id */, 0, "Cons1"/* consumer name */, - true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, + DEFAULT_CONSUMER_EPOCH, null); topic.getSubscriptions().put(Codec.decode(cursor1.getName()), sub1); sub1.addConsumer(consumer1); // Open cursor2, add it into activeCursor-container and add it into subscription consumer list ManagedCursor cursor2 = ledger.openCursor("c2"); PersistentSubscription sub2 = new PersistentSubscription(topic, "sub-2", cursor2, false); Consumer consumer2 = new Consumer(sub2, SubType.Exclusive, topic.getName(), 2 /* consumer id */, 0, "Cons2"/* consumer name */, - true, serverCnx, "myrole-2", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + true, serverCnx, "myrole-2", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, + DEFAULT_CONSUMER_EPOCH, null); topic.getSubscriptions().put(Codec.decode(cursor2.getName()), sub2); sub2.addConsumer(consumer2); // Open cursor3, add it into activeCursor-container and do not add it into subscription consumer list ManagedCursor cursor3 = ledger.openCursor("c3"); PersistentSubscription sub3 = new PersistentSubscription(topic, "sub-3", cursor3, false); Consumer consumer3 = new Consumer(sub2, SubType.Exclusive, topic.getName(), 3 /* consumer id */, 0, "Cons2"/* consumer name */, - true, serverCnx, "myrole-3", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + true, serverCnx, "myrole-3", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, + DEFAULT_CONSUMER_EPOCH, null); topic.getSubscriptions().put(Codec.decode(cursor3.getName()), sub3); // Case1: cursors are active as haven't started deactivateBacklogCursor scan @@ -2123,7 +2131,8 @@ public void testCheckInactiveSubscriptions() throws Exception { addConsumerToSubscription.setAccessible(true); Consumer consumer = new Consumer(nonDeletableSubscription1, SubType.Shared, topic.getName(), 1, 0, "consumer1", - true, serverCnx, "app1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + true, serverCnx, "app1", Collections.emptyMap(), false, null, MessageId.latest, + DEFAULT_CONSUMER_EPOCH, null); addConsumerToSubscription.invoke(topic, nonDeletableSubscription1, consumer); NamespaceResources nsr = pulsar.getPulsarResources().getNamespaceResources(); @@ -2237,7 +2246,7 @@ public void testGetDurableSubscription() throws Exception { .setRequestId(1) .setSubType(SubType.Exclusive); - Future f1 = topic.subscribe(getSubscriptionOption(cmd)); + Future f1 = topic.subscribe(getSubscriptionOption(cmd), null); f1.get(); Future f2 = topic.unsubscribe(successSubName); @@ -2268,7 +2277,7 @@ public void testKeySharedMetadataExposedToStats() throws Exception { Consumer consumer1 = new Consumer(sub1, SubType.Key_Shared, topic.getName(), 1, 0, "Cons1", true, serverCnx, "myrole-1", Collections.emptyMap(), false, new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT).setAllowOutOfOrderDelivery(false), - MessageId.latest, DEFAULT_CONSUMER_EPOCH); + MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); sub1.addConsumer(consumer1); consumer1.close(); @@ -2279,7 +2288,7 @@ public void testKeySharedMetadataExposedToStats() throws Exception { Consumer consumer2 = new Consumer(sub2, SubType.Key_Shared, topic.getName(), 2, 0, "Cons2", true, serverCnx, "myrole-1", Collections.emptyMap(), false, new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT).setAllowOutOfOrderDelivery(true), - MessageId.latest, DEFAULT_CONSUMER_EPOCH); + MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); sub2.addConsumer(consumer2); consumer2.close(); @@ -2291,7 +2300,7 @@ public void testKeySharedMetadataExposedToStats() throws Exception { .setAllowOutOfOrderDelivery(false); ksm.addHashRange().setStart(0).setEnd(65535); Consumer consumer3 = new Consumer(sub3, SubType.Key_Shared, topic.getName(), 3, 0, "Cons3", true, serverCnx, - "myrole-1", Collections.emptyMap(), false, ksm, MessageId.latest, DEFAULT_CONSUMER_EPOCH); + "myrole-1", Collections.emptyMap(), false, ksm, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); sub3.addConsumer(consumer3); consumer3.close(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java index 0eb1c08c5abde..f0e0a5755ddd9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java @@ -1233,9 +1233,25 @@ public void testSubscribeWithSchemaAfterAutoConsumeNewTopic(String domain) throw .subscriptionName("sub0") .consumerName("autoConsumer2") .subscribe(); + + Consumer autoConsumer3 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub1") + .consumerName("autoConsumer3") + .subscribe(); + + Consumer autoConsumer4 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub1") + .consumerName("autoConsumer4") + .subscribe(); try { log.info("The autoConsumer1 isConnected: " + autoConsumer1.isConnected()); log.info("The autoConsumer2 isConnected: " + autoConsumer2.isConnected()); + log.info("The autoConsumer3 isConnected: " + autoConsumer3.isConnected()); + log.info("The autoConsumer4 isConnected: " + autoConsumer4.isConnected()); admin.schemas().getSchemaInfo(topic); fail("The schema of topic should not exist"); } catch (PulsarAdminException e) { @@ -1254,12 +1270,28 @@ public void testSubscribeWithSchemaAfterAutoConsumeNewTopic(String domain) throw .subscriptionName("sub0") .consumerName("consumerWithSchema-2") .subscribe(); + Consumer consumerWithSchema3 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub1") + .consumerName("consumerWithSchema-3") + .subscribe(); + Consumer consumerWithSchema4 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) + .topic(topic) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName("sub1") + .consumerName("consumerWithSchema-4") + .subscribe(); try { log.info(admin.schemas().getSchemaInfo(topic).toString()); log.info("The autoConsumer1 isConnected: " + autoConsumer1.isConnected()); log.info("The autoConsumer2 isConnected: " + autoConsumer2.isConnected()); + log.info("The autoConsumer3 isConnected: " + autoConsumer3.isConnected()); + log.info("The autoConsumer4 isConnected: " + autoConsumer4.isConnected()); log.info("The consumerWithSchema-1 isConnected: " + consumerWithSchema1.isConnected()); log.info("The consumerWithSchema-2 isConnected: " + consumerWithSchema2.isConnected()); + log.info("The consumerWithSchema-3 isConnected: " + consumerWithSchema3.isConnected()); + log.info("The consumerWithSchema-4 isConnected: " + consumerWithSchema4.isConnected()); } catch (PulsarAdminException e) { assertEquals(e.getStatusCode(), 404); } From 649d92c92bc3a459aef9159603395129e4c8cd61 Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Tue, 20 Dec 2022 15:32:06 +0800 Subject: [PATCH 09/20] SchemaType replaces SchemaData in Consumer --- .../apache/pulsar/broker/service/Consumer.java | 17 ++++++----------- .../nonpersistent/NonPersistentTopic.java | 6 +++--- .../service/persistent/PersistentTopic.java | 5 +++-- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index abb00a9b13b8d..137e6c3225367 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -56,7 +56,6 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.stats.Rate; import org.apache.pulsar.common.util.DateFormatter; @@ -142,13 +141,13 @@ public class Consumer { private long negtiveUnackedMsgsTimestamp; - private SchemaData schemaData; + private SchemaType schemaType; public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId, int priorityLevel, String consumerName, boolean isDurable, TransportCnx cnx, String appId, Map metadata, boolean readCompacted, - KeySharedMeta keySharedMeta, MessageId startMessageId, long consumerEpoch, SchemaData schemaData) { + KeySharedMeta keySharedMeta, MessageId startMessageId, long consumerEpoch, SchemaType schemaType) { this.subscription = subscription; this.subType = subType; @@ -207,11 +206,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.isAcknowledgmentAtBatchIndexLevelEnabled = subscription.getTopic().getBrokerService() .getPulsar().getConfiguration().isAcknowledgmentAtBatchIndexLevelEnabled(); - if (schemaData == null) { - this.schemaData = SchemaData.builder().type(SchemaType.AUTO_CONSUME).build(); - } else { - this.schemaData = schemaData; - } + this.schemaType = schemaType; } @VisibleForTesting @@ -239,7 +234,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.clientAddress = null; this.startMessageId = null; this.isAcknowledgmentAtBatchIndexLevelEnabled = false; - this.schemaData = null; + this.schemaType = SchemaType.AUTO_CONSUME; MESSAGE_PERMITS_UPDATER.set(this, availablePermits); } @@ -1097,8 +1092,8 @@ public Map getMetadata() { return metadata; } - public SchemaData getSchemaData() { - return schemaData; + public SchemaType getSchemaType() { + return schemaType; } private int getStickyKeyHash(Entry entry) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 2e40108993424..e61f2df2c5b76 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -315,8 +315,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St name -> new NonPersistentSubscription(this, subscriptionName, isDurable, subscriptionProperties)); Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, - false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, - MessageId.latest, DEFAULT_CONSUMER_EPOCH, schemaData); + false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest, + DEFAULT_CONSUMER_EPOCH, schemaData == null ? SchemaType.AUTO_CONSUME : schemaData.getType()); addConsumerToSubscription(subscription, consumer).thenRun(() -> { if (!cnx.isActive()) { @@ -1126,7 +1126,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem return hasSchema().thenCompose((hasSchema) -> { int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() .mapToInt(subscription -> subscription.getConsumers().stream() - .filter(consumer -> consumer.getSchemaData().getType() != SchemaType.AUTO_CONSUME) + .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) .toList().size()) .sum(); if (hasSchema diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index ab297afc5723c..96dd02e2c89f1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -780,7 +780,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St CompletableFuture future = subscriptionFuture.thenCompose(subscription -> { Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, isDurable, cnx, cnx.getAuthRole(), metadata, - readCompacted, keySharedMeta, startMessageId, consumerEpoch, schemaData); + readCompacted, keySharedMeta, startMessageId, consumerEpoch, + schemaData == null ? SchemaType.AUTO_CONSUME : schemaData.getType()); return addConsumerToSubscription(subscription, consumer).thenCompose(v -> { checkBackloggedCursors(); @@ -2854,7 +2855,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem return hasSchema().thenCompose((hasSchema) -> { int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() .mapToInt(subscription -> subscription.getConsumers().stream() - .filter(consumer -> consumer.getSchemaData().getType() != SchemaType.AUTO_CONSUME) + .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) .toList().size()) .sum(); if (hasSchema From 71ecde81464478329d32b214fb60574d962dbded Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Sat, 24 Dec 2022 23:49:39 +0800 Subject: [PATCH 10/20] upload AUTO schema --- .../java/org/apache/pulsar/broker/service/Consumer.java | 4 ++-- .../java/org/apache/pulsar/broker/service/ServerCnx.java | 2 +- .../broker/service/nonpersistent/NonPersistentTopic.java | 4 ++-- .../pulsar/broker/service/persistent/PersistentTopic.java | 4 ++-- .../java/org/apache/pulsar/client/impl/ConsumerImpl.java | 8 ++++++++ .../java/org/apache/pulsar/common/protocol/Commands.java | 4 ++-- pulsar-common/src/main/proto/PulsarApi.proto | 1 + 7 files changed, 18 insertions(+), 9 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 8c342003767ef..2ef86dfda202b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -143,7 +143,7 @@ public class Consumer { private long negtiveUnackedMsgsTimestamp; - private SchemaType schemaType; + private final SchemaType schemaType; public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId, int priorityLevel, String consumerName, @@ -236,7 +236,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.clientAddress = null; this.startMessageId = null; this.isAcknowledgmentAtBatchIndexLevelEnabled = false; - this.schemaType = SchemaType.AUTO_CONSUME; + this.schemaType = SchemaType.BYTES; MESSAGE_PERMITS_UPDATER.set(this, availablePermits); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index a962bb5488751..dee94e5c6dd3a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1130,7 +1130,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { .subscriptionProperties(subscriptionProperties) .consumerEpoch(consumerEpoch) .build(); - if (schema != null) { + if (schema != null && schema.getType() != SchemaType.AUTO) { return topic.addSchemaIfIdleOrCheckCompatible(schema) .thenCompose(v -> topic.subscribe(option, schema)); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 3db8b447407ef..92d750f6f009a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -325,7 +325,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest, - DEFAULT_CONSUMER_EPOCH, schemaData == null ? SchemaType.AUTO_CONSUME : schemaData.getType()); + DEFAULT_CONSUMER_EPOCH, schemaData == null ? SchemaType.BYTES : schemaData.getType()); if (isMigrated()) { consumer.topicMigrated(getClusterMigrationUrl()); } @@ -1167,7 +1167,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem return hasSchema().thenCompose((hasSchema) -> { int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() .mapToInt(subscription -> subscription.getConsumers().stream() - .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) + .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO) .toList().size()) .sum(); if (hasSchema diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 1d97e69cde281..3174a3c44306f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -812,7 +812,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, isDurable, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, startMessageId, consumerEpoch, - schemaData == null ? SchemaType.AUTO_CONSUME : schemaData.getType()); + schemaData == null ? SchemaType.BYTES : schemaData.getType()); return addConsumerToSubscription(subscription, consumer).thenCompose(v -> { checkBackloggedCursors(); @@ -3061,7 +3061,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem return hasSchema().thenCompose((hasSchema) -> { int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() .mapToInt(subscription -> subscription.getConsumers().stream() - .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) + .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO) .toList().size()) .sum(); if (hasSchema diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 2680e70703022..1646b92baf170 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -83,6 +83,7 @@ import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; +import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.client.util.RetryMessageUtil; @@ -811,6 +812,13 @@ public void connectionOpened(final ClientCnx cnx) { } SchemaInfo si = schema.getSchemaInfo(); + if (si == null && schema instanceof AutoConsumeSchema) { + si = SchemaInfo.builder() + .name("") + .type(SchemaType.AUTO_CONSUME) + .schema(new byte[0]) + .build(); + } if (si != null && (SchemaType.BYTES == si.getType() || SchemaType.NONE == si.getType())) { // don't set schema for Schema.BYTES si = null; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index fdb94c177959c..9860d56781ead 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -772,7 +772,7 @@ public static ByteBuf newProducer(String topic, long producerId, long requestId, private static Schema.Type getSchemaType(SchemaType type) { if (type.getValue() < 0) { - return Schema.Type.None; + return Schema.Type.Auto; } else { return Schema.Type.valueOf(type.getValue()); } @@ -781,7 +781,7 @@ private static Schema.Type getSchemaType(SchemaType type) { public static SchemaType getSchemaType(Schema.Type type) { if (type.getValue() < 0) { // this is unexpected - return SchemaType.NONE; + return SchemaType.AUTO; } else { return SchemaType.valueOf(type.getValue()); } diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto index acf75eab85826..63540d0329ddb 100644 --- a/pulsar-common/src/main/proto/PulsarApi.proto +++ b/pulsar-common/src/main/proto/PulsarApi.proto @@ -45,6 +45,7 @@ message Schema { LocalTime = 18; LocalDateTime = 19; ProtobufNative = 20; + Auto = -2; } required string name = 1; From 3bf681be690728eefd797359512b47f5110d843f Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Sun, 25 Dec 2022 18:13:59 +0800 Subject: [PATCH 11/20] upload AUTO_CONSUME schema --- .../org/apache/pulsar/broker/service/ServerCnx.java | 2 +- .../service/nonpersistent/NonPersistentTopic.java | 2 +- .../broker/service/persistent/PersistentTopic.java | 2 +- .../org/apache/pulsar/common/protocol/Commands.java | 13 ++----------- pulsar-common/src/main/proto/PulsarApi.proto | 2 ++ 5 files changed, 7 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index dee94e5c6dd3a..769ee12a3c31b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1130,7 +1130,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { .subscriptionProperties(subscriptionProperties) .consumerEpoch(consumerEpoch) .build(); - if (schema != null && schema.getType() != SchemaType.AUTO) { + if (schema != null && schema.getType() != SchemaType.AUTO_CONSUME) { return topic.addSchemaIfIdleOrCheckCompatible(schema) .thenCompose(v -> topic.subscribe(option, schema)); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 92d750f6f009a..841ecd227676e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -1167,7 +1167,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem return hasSchema().thenCompose((hasSchema) -> { int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() .mapToInt(subscription -> subscription.getConsumers().stream() - .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO) + .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) .toList().size()) .sum(); if (hasSchema diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 3174a3c44306f..ee8b99c5bb056 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3061,7 +3061,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem return hasSchema().thenCompose((hasSchema) -> { int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() .mapToInt(subscription -> subscription.getConsumers().stream() - .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO) + .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) .toList().size()) .sum(); if (hasSchema diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index 9860d56781ead..5c06967fccbe6 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -771,20 +771,11 @@ public static ByteBuf newProducer(String topic, long producerId, long requestId, } private static Schema.Type getSchemaType(SchemaType type) { - if (type.getValue() < 0) { - return Schema.Type.Auto; - } else { - return Schema.Type.valueOf(type.getValue()); - } + return Schema.Type.valueOf(type.getValue()); } public static SchemaType getSchemaType(Schema.Type type) { - if (type.getValue() < 0) { - // this is unexpected - return SchemaType.AUTO; - } else { - return SchemaType.valueOf(type.getValue()); - } + return SchemaType.valueOf(type.getValue()); } private static void convertSchema(SchemaInfo schemaInfo, Schema schema) { diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto index 63540d0329ddb..7fe112e7d66f1 100644 --- a/pulsar-common/src/main/proto/PulsarApi.proto +++ b/pulsar-common/src/main/proto/PulsarApi.proto @@ -46,6 +46,8 @@ message Schema { LocalDateTime = 19; ProtobufNative = 20; Auto = -2; + AutoConsume = -3; + AutoPublish = -4; } required string name = 1; From d04d70c690bf671672e76f9c30b4c89e4132e382 Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Wed, 28 Dec 2022 21:24:56 +0800 Subject: [PATCH 12/20] SchemaType add in SubscriptionOption; only add AutoConsume; add a singleton in AutoConsumeSchema --- .../pulsar/broker/service/ServerCnx.java | 5 ++-- .../broker/service/SubscriptionOption.java | 2 ++ .../apache/pulsar/broker/service/Topic.java | 5 ++-- .../nonpersistent/NonPersistentTopic.java | 12 ++++---- .../service/persistent/PersistentTopic.java | 12 ++++---- .../PersistentTopicConcurrentTest.java | 8 +++--- .../broker/service/PersistentTopicTest.java | 28 +++++++++---------- .../pulsar/client/impl/ConsumerImpl.java | 8 ------ .../client/impl/schema/AutoConsumeSchema.java | 12 +++++++- .../pulsar/common/protocol/Commands.java | 13 +++++++-- pulsar-common/src/main/proto/PulsarApi.proto | 2 -- 11 files changed, 60 insertions(+), 47 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 769ee12a3c31b..b5e68893596f2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1129,12 +1129,13 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { .replicatedSubscriptionStateArg(isReplicated).keySharedMeta(keySharedMeta) .subscriptionProperties(subscriptionProperties) .consumerEpoch(consumerEpoch) + .schemaType(schema == null ? null : schema.getType()) .build(); if (schema != null && schema.getType() != SchemaType.AUTO_CONSUME) { return topic.addSchemaIfIdleOrCheckCompatible(schema) - .thenCompose(v -> topic.subscribe(option, schema)); + .thenCompose(v -> topic.subscribe(option)); } else { - return topic.subscribe(option, schema); + return topic.subscribe(option); } }) .thenAccept(consumer -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java index d375c539e550e..af56d023616b4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java @@ -29,6 +29,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.KeyValue; +import org.apache.pulsar.common.schema.SchemaType; @Getter @Builder @@ -49,6 +50,7 @@ public class SubscriptionOption { private KeySharedMeta keySharedMeta; private Optional> subscriptionProperties; private long consumerEpoch; + private SchemaType schemaType; public static Optional> getPropertiesMap(List list) { if (list == null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java index e08c4f7906c21..11e2d04b581b0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java @@ -43,6 +43,7 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.protocol.schema.SchemaVersion; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.utils.StatsOutputStream; @@ -170,14 +171,14 @@ CompletableFuture subscribe(TransportCnx cnx, String subscriptionName, InitialPosition initialPosition, long startMessageRollbackDurationSec, boolean replicateSubscriptionState, KeySharedMeta keySharedMeta, - SchemaData schemaData); + SchemaType schemaType); /** * Subscribe a topic. * @param option * @return */ - CompletableFuture subscribe(SubscriptionOption option, SchemaData schemaData); + CompletableFuture subscribe(SubscriptionOption option); CompletableFuture createSubscription(String subscriptionName, InitialPosition initialPosition, boolean replicateSubscriptionState, Map properties); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 841ecd227676e..38c282254f85e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -250,13 +250,13 @@ public CompletableFuture checkIfTransactionBufferRecoverCompletely(boolean } @Override - public CompletableFuture subscribe(SubscriptionOption option, SchemaData schemaData) { + public CompletableFuture subscribe(SubscriptionOption option) { return internalSubscribe(option.getCnx(), option.getSubscriptionName(), option.getConsumerId(), option.getSubType(), option.getPriorityLevel(), option.getConsumerName(), option.isDurable(), option.getStartMessageId(), option.getMetadata(), option.isReadCompacted(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), - option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(null), schemaData); + option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(null), option.getSchemaType()); } @Override @@ -267,10 +267,10 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs InitialPosition initialPosition, long resetStartMessageBackInSec, boolean replicateSubscriptionState, KeySharedMeta keySharedMeta, - SchemaData schemaData) { + SchemaType schemaType) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, resetStartMessageBackInSec, - replicateSubscriptionState, keySharedMeta, null, schemaData); + replicateSubscriptionState, keySharedMeta, null, schemaType); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -282,7 +282,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St boolean replicateSubscriptionState, KeySharedMeta keySharedMeta, Map subscriptionProperties, - SchemaData schemaData) { + SchemaType schemaType) { return brokerService.checkTopicNsOwnership(getName()).thenCompose(__ -> { final CompletableFuture future = new CompletableFuture<>(); @@ -325,7 +325,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest, - DEFAULT_CONSUMER_EPOCH, schemaData == null ? SchemaType.BYTES : schemaData.getType()); + DEFAULT_CONSUMER_EPOCH, schemaType == null ? SchemaType.BYTES : schemaType); if (isMigrated()) { consumer.topicMigrated(getClusterMigrationUrl()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index ee8b99c5bb056..c3231f24a4aac 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -702,14 +702,14 @@ protected void handleProducerRemoved(Producer producer) { } @Override - public CompletableFuture subscribe(SubscriptionOption option, SchemaData schemaData) { + public CompletableFuture subscribe(SubscriptionOption option) { return internalSubscribe(option.getCnx(), option.getSubscriptionName(), option.getConsumerId(), option.getSubType(), option.getPriorityLevel(), option.getConsumerName(), option.isDurable(), option.getStartMessageId(), option.getMetadata(), option.isReadCompacted(), option.getInitialPosition(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(Collections.emptyMap()), - option.getConsumerEpoch(), schemaData); + option.getConsumerEpoch(), option.getSchemaType()); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -723,7 +723,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St KeySharedMeta keySharedMeta, Map subscriptionProperties, long consumerEpoch, - SchemaData schemaData) { + SchemaType schemaType) { if (readCompacted && !(subType == SubType.Failover || subType == SubType.Exclusive)) { return FutureUtil.failedFuture(new NotAllowedException( "readCompacted only allowed on failover or exclusive subscriptions")); @@ -812,7 +812,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, isDurable, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, startMessageId, consumerEpoch, - schemaData == null ? SchemaType.BYTES : schemaData.getType()); + schemaType == null ? SchemaType.BYTES : schemaType); return addConsumerToSubscription(subscription, consumer).thenCompose(v -> { checkBackloggedCursors(); @@ -887,10 +887,10 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs long startMessageRollbackDurationSec, boolean replicatedSubscriptionStateArg, KeySharedMeta keySharedMeta, - SchemaData schemaData) { + SchemaType schemaType) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, initialPosition, startMessageRollbackDurationSec, - replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH, schemaData); + replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH, schemaType); } private CompletableFuture getDurableSubscription(String subscriptionName, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java index 6aa37c3cd49af..4bf0481e0d084 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicConcurrentTest.java @@ -129,7 +129,7 @@ public void testConcurrentTopicAndSubscriptionDelete() throws Exception { SubscriptionOption subscriptionOption = getSubscriptionOption(cmd); - Future f1 = topic.subscribe(subscriptionOption, null); + Future f1 = topic.subscribe(subscriptionOption); f1.get(); final CyclicBarrier barrier = new CyclicBarrier(2); @@ -187,7 +187,7 @@ public void testConcurrentTopicGCAndSubscriptionDelete() throws Exception { SubscriptionOption subscriptionOption = getSubscriptionOption(cmd); - Future f1 = topic.subscribe(subscriptionOption, null); + Future f1 = topic.subscribe(subscriptionOption); f1.get(); final CyclicBarrier barrier = new CyclicBarrier(2); @@ -251,7 +251,7 @@ public void testConcurrentTopicDeleteAndUnsubscribe() throws Exception { SubscriptionOption subscriptionOption = getSubscriptionOption(cmd); - Future f1 = topic.subscribe(subscriptionOption, null); + Future f1 = topic.subscribe(subscriptionOption); f1.get(); final CyclicBarrier barrier = new CyclicBarrier(2); @@ -309,7 +309,7 @@ public void testConcurrentTopicDeleteAndSubsUnsubscribe() throws Exception { SubscriptionOption subscriptionOption = getSubscriptionOption(cmd); - Future f1 = topic.subscribe(subscriptionOption, null); + Future f1 = topic.subscribe(subscriptionOption); f1.get(); final CyclicBarrier barrier = new CyclicBarrier(2); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 9d4f35edaa010..bb0d957fc02e3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -712,7 +712,7 @@ public void testSubscribeFail() throws Exception { SubscriptionOption subscriptionOption = getSubscriptionOption(cmd); - Future f1 = topic.subscribe(subscriptionOption, null); + Future f1 = topic.subscribe(subscriptionOption); try { f1.get(); fail("should fail with exception"); @@ -747,11 +747,11 @@ public void testSubscribeUnsubscribe() throws Exception { .setSubType(SubType.Exclusive); // 1. simple subscribe - Future f1 = topic.subscribe(getSubscriptionOption(cmd), null); + Future f1 = topic.subscribe(getSubscriptionOption(cmd)); f1.get(); // 2. duplicate subscribe - Future f2 = topic.subscribe(getSubscriptionOption(cmd), null); + Future f2 = topic.subscribe(getSubscriptionOption(cmd)); try { f2.get(); fail("should fail with exception"); @@ -1318,7 +1318,7 @@ public void testDeleteTopic() throws Exception { .setRequestId(1) .setSubType(SubType.Exclusive); - Future f1 = topic.subscribe(getSubscriptionOption(cmd), null); + Future f1 = topic.subscribe(getSubscriptionOption(cmd)); f1.get(); CompletableFuture cf2 = topic.delete(); @@ -1342,7 +1342,7 @@ public void testDeleteAndUnsubscribeTopic() throws Exception { .setReadCompacted(false) .setSubType(SubType.Exclusive); - Future f1 = topic.subscribe(getSubscriptionOption(cmd), null); + Future f1 = topic.subscribe(getSubscriptionOption(cmd)); f1.get(); final CyclicBarrier barrier = new CyclicBarrier(2); @@ -1399,7 +1399,7 @@ public void testConcurrentTopicAndSubscriptionDelete() throws Exception { .setRequestId(1) .setSubType(SubType.Exclusive); - Future f1 = topic.subscribe(getSubscriptionOption(cmd), null); + Future f1 = topic.subscribe(getSubscriptionOption(cmd)); f1.get(); @@ -1493,7 +1493,7 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { .setRequestId(1) .setSubType(SubType.Exclusive); - Future f = topic.subscribe(getSubscriptionOption(cmd), null); + Future f = topic.subscribe(getSubscriptionOption(cmd)); try { f.get(); fail("should have failed"); @@ -1634,7 +1634,7 @@ public void testFailoverSubscription() throws Exception { .setSubType(SubType.Failover); // 1. Subscribe with non partition topic - Future f1 = topic1.subscribe(getSubscriptionOption(cmd1), null); + Future f1 = topic1.subscribe(getSubscriptionOption(cmd1)); f1.get(); // 2. Subscribe with partition topic @@ -1649,7 +1649,7 @@ public void testFailoverSubscription() throws Exception { .setRequestId(1) .setSubType(SubType.Failover); - Future f2 = topic2.subscribe(getSubscriptionOption(cmd2), null); + Future f2 = topic2.subscribe(getSubscriptionOption(cmd2)); f2.get(); // 3. Subscribe and create second consumer @@ -1662,7 +1662,7 @@ public void testFailoverSubscription() throws Exception { .setRequestId(1) .setSubType(SubType.Failover); - Future f3 = topic2.subscribe(getSubscriptionOption(cmd3), null); + Future f3 = topic2.subscribe(getSubscriptionOption(cmd3)); f3.get(); assertEquals( @@ -1686,7 +1686,7 @@ public void testFailoverSubscription() throws Exception { .setRequestId(1) .setSubType(SubType.Failover); - Future f4 = topic2.subscribe(getSubscriptionOption(cmd4), null); + Future f4 = topic2.subscribe(getSubscriptionOption(cmd4)); f4.get(); assertEquals( @@ -1715,7 +1715,7 @@ public void testFailoverSubscription() throws Exception { .setRequestId(1) .setSubType(SubType.Exclusive); - Future f5 = topic2.subscribe(getSubscriptionOption(cmd5), null); + Future f5 = topic2.subscribe(getSubscriptionOption(cmd5)); try { f5.get(); fail("should fail with exception"); @@ -1734,7 +1734,7 @@ public void testFailoverSubscription() throws Exception { .setRequestId(1) .setSubType(SubType.Exclusive); - Future f6 = topic2.subscribe(getSubscriptionOption(cmd6), null); + Future f6 = topic2.subscribe(getSubscriptionOption(cmd6)); f6.get(); // 7. unsubscribe exclusive sub @@ -2255,7 +2255,7 @@ public void testGetDurableSubscription() throws Exception { .setRequestId(1) .setSubType(SubType.Exclusive); - Future f1 = topic.subscribe(getSubscriptionOption(cmd), null); + Future f1 = topic.subscribe(getSubscriptionOption(cmd)); f1.get(); Future f2 = topic.unsubscribe(successSubName); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 1646b92baf170..2680e70703022 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -83,7 +83,6 @@ import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; -import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.client.util.RetryMessageUtil; @@ -812,13 +811,6 @@ public void connectionOpened(final ClientCnx cnx) { } SchemaInfo si = schema.getSchemaInfo(); - if (si == null && schema instanceof AutoConsumeSchema) { - si = SchemaInfo.builder() - .name("") - .type(SchemaType.AUTO_CONSUME) - .schema(new byte[0]) - .build(); - } if (si != null && (SchemaType.BYTES == si.getType() || SchemaType.NONE == si.getType())) { // don't set schema for Schema.BYTES si = null; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java index 33fcd18876be6..750f24de41bb0 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java @@ -57,6 +57,16 @@ public class AutoConsumeSchema implements Schema { private SchemaInfoProvider schemaInfoProvider; + private static final SchemaInfo SCHEMA_INFO; + + static { + SCHEMA_INFO = SchemaInfoImpl.builder() + .name("AutoConsume") + .type(SchemaType.AUTO_CONSUME) + .schema(new byte[0]) + .build(); + } + private ConcurrentMap> initSchemaMap() { ConcurrentMap> schemaMap = new ConcurrentHashMap<>(); // The Schema.BYTES will not be uploaded to the broker and store in the schema storage, @@ -140,7 +150,7 @@ public void setSchemaInfoProvider(SchemaInfoProvider schemaInfoProvider) { @Override public SchemaInfo getSchemaInfo() { if (!schemaMap.containsKey(SchemaVersion.Latest)) { - return null; + return SCHEMA_INFO; } return schemaMap.get(SchemaVersion.Latest).getSchemaInfo(); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index 5c06967fccbe6..deb9606942799 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -771,11 +771,20 @@ public static ByteBuf newProducer(String topic, long producerId, long requestId, } private static Schema.Type getSchemaType(SchemaType type) { - return Schema.Type.valueOf(type.getValue()); + if (type.getValue() < 0 && type.getValue() != -3) { + return Schema.Type.None; + } else { + return Schema.Type.valueOf(type.getValue()); + } } public static SchemaType getSchemaType(Schema.Type type) { - return SchemaType.valueOf(type.getValue()); + if (type.getValue() < 0 && type.getValue() != -3) { + // this is unexpected + return SchemaType.NONE; + } else { + return SchemaType.valueOf(type.getValue()); + } } private static void convertSchema(SchemaInfo schemaInfo, Schema schema) { diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto index 7fe112e7d66f1..9f5d405583c0b 100644 --- a/pulsar-common/src/main/proto/PulsarApi.proto +++ b/pulsar-common/src/main/proto/PulsarApi.proto @@ -45,9 +45,7 @@ message Schema { LocalTime = 18; LocalDateTime = 19; ProtobufNative = 20; - Auto = -2; AutoConsume = -3; - AutoPublish = -4; } required string name = 1; From 9b2f5ff6c5bb5a891d855c470e921619d1f2ae4d Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Fri, 30 Dec 2022 23:46:40 +0800 Subject: [PATCH 13/20] Add a new method in AutoConsumeSchema getAutoConsumeSchemaInfo() instead of getSchemaInfo when the schema type is AUTO_CONSUME, and modify the code for test. --- .../pulsar/client/api/SimpleSchemaTest.java | 29 +++++-------------- .../pulsar/client/impl/ConsumerImpl.java | 5 ++++ .../client/impl/schema/AutoConsumeSchema.java | 6 +++- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java index a3f45e77de62d..c8c7c3b2ccc38 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java @@ -1243,27 +1243,28 @@ public void testAutoCreatedSchema(String domain) throws Exception { public void testSubscribeWithSchemaAfterAutoConsumeNewTopic(String domain) throws Exception { final String topic = domain + "my-property/my-ns/testSubscribeWithSchemaAfterAutoConsume-1"; + @Cleanup Consumer autoConsumer1 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) .topic(topic) .subscriptionType(SubscriptionType.Shared) .subscriptionName("sub0") .consumerName("autoConsumer1") .subscribe(); - + @Cleanup Consumer autoConsumer2 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) .topic(topic) .subscriptionType(SubscriptionType.Shared) .subscriptionName("sub0") .consumerName("autoConsumer2") .subscribe(); - + @Cleanup Consumer autoConsumer3 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) .topic(topic) .subscriptionType(SubscriptionType.Shared) .subscriptionName("sub1") .consumerName("autoConsumer3") .subscribe(); - + @Cleanup Consumer autoConsumer4 = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) .topic(topic) .subscriptionType(SubscriptionType.Shared) @@ -1281,48 +1282,34 @@ public void testSubscribeWithSchemaAfterAutoConsumeNewTopic(String domain) throw assertEquals(e.getStatusCode(), 404); } + @Cleanup Consumer consumerWithSchema1 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) .topic(topic) .subscriptionType(SubscriptionType.Shared) .subscriptionName("sub0") .consumerName("consumerWithSchema-1") .subscribe(); + @Cleanup Consumer consumerWithSchema2 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) .topic(topic) .subscriptionType(SubscriptionType.Shared) .subscriptionName("sub0") .consumerName("consumerWithSchema-2") .subscribe(); + @Cleanup Consumer consumerWithSchema3 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) .topic(topic) .subscriptionType(SubscriptionType.Shared) .subscriptionName("sub1") .consumerName("consumerWithSchema-3") .subscribe(); + @Cleanup Consumer consumerWithSchema4 = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) .topic(topic) .subscriptionType(SubscriptionType.Shared) .subscriptionName("sub1") .consumerName("consumerWithSchema-4") .subscribe(); - try { - log.info(admin.schemas().getSchemaInfo(topic).toString()); - log.info("The autoConsumer1 isConnected: " + autoConsumer1.isConnected()); - log.info("The autoConsumer2 isConnected: " + autoConsumer2.isConnected()); - log.info("The autoConsumer3 isConnected: " + autoConsumer3.isConnected()); - log.info("The autoConsumer4 isConnected: " + autoConsumer4.isConnected()); - log.info("The consumerWithSchema-1 isConnected: " + consumerWithSchema1.isConnected()); - log.info("The consumerWithSchema-2 isConnected: " + consumerWithSchema2.isConnected()); - log.info("The consumerWithSchema-3 isConnected: " + consumerWithSchema3.isConnected()); - log.info("The consumerWithSchema-4 isConnected: " + consumerWithSchema4.isConnected()); - } catch (PulsarAdminException e) { - assertEquals(e.getStatusCode(), 404); - } - - autoConsumer1.close(); - autoConsumer2.close(); - consumerWithSchema1.close(); - consumerWithSchema2.close(); } @DataProvider(name = "keyEncodingType") diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 2680e70703022..16814f8615f02 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -83,6 +83,7 @@ import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; +import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.client.impl.transaction.TransactionImpl; import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.client.util.RetryMessageUtil; @@ -814,6 +815,10 @@ public void connectionOpened(final ClientCnx cnx) { if (si != null && (SchemaType.BYTES == si.getType() || SchemaType.NONE == si.getType())) { // don't set schema for Schema.BYTES si = null; + } else { + if (schema instanceof AutoConsumeSchema) { + si = ((AutoConsumeSchema) schema).getAutoConsumeSchemaInfo(); + } } // startMessageRollbackDurationInSec should be consider only once when consumer connects to first time long startMessageRollbackDuration = (startMessageRollbackDurationInSec > 0 diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java index 750f24de41bb0..de61d7fa3f49e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java @@ -150,7 +150,7 @@ public void setSchemaInfoProvider(SchemaInfoProvider schemaInfoProvider) { @Override public SchemaInfo getSchemaInfo() { if (!schemaMap.containsKey(SchemaVersion.Latest)) { - return SCHEMA_INFO; + return null; } return schemaMap.get(SchemaVersion.Latest).getSchemaInfo(); } @@ -163,6 +163,10 @@ public SchemaInfo getSchemaInfo(byte[] schemaVersion) { return null; } + public SchemaInfo getAutoConsumeSchemaInfo() { + return SCHEMA_INFO; + } + @Override public void configureSchemaInfo(String topicName, String componentName, From f0c2b25c7463caefdcead33eabaa43291f337604 Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Mon, 2 Jan 2023 22:58:30 +0800 Subject: [PATCH 14/20] Backward compatibility, if the old pulsar-client uses AUTO_CONSUME consumer subscribe topic, the schemaType is null. Then the BYTE is wrong. --- .../main/java/org/apache/pulsar/broker/service/Consumer.java | 2 +- .../broker/service/nonpersistent/NonPersistentTopic.java | 2 +- .../pulsar/broker/service/persistent/PersistentTopic.java | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 2ef86dfda202b..6858b84172613 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -236,7 +236,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.clientAddress = null; this.startMessageId = null; this.isAcknowledgmentAtBatchIndexLevelEnabled = false; - this.schemaType = SchemaType.BYTES; + this.schemaType = null; MESSAGE_PERMITS_UPDATER.set(this, availablePermits); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 38c282254f85e..3ec2abc9402a7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -325,7 +325,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest, - DEFAULT_CONSUMER_EPOCH, schemaType == null ? SchemaType.BYTES : schemaType); + DEFAULT_CONSUMER_EPOCH, schemaType); if (isMigrated()) { consumer.topicMigrated(getClusterMigrationUrl()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index c3231f24a4aac..b1fad866956c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -811,8 +811,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St CompletableFuture future = subscriptionFuture.thenCompose(subscription -> { Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, isDurable, cnx, cnx.getAuthRole(), metadata, - readCompacted, keySharedMeta, startMessageId, consumerEpoch, - schemaType == null ? SchemaType.BYTES : schemaType); + readCompacted, keySharedMeta, startMessageId, consumerEpoch, schemaType); return addConsumerToSubscription(subscription, consumer).thenCompose(v -> { checkBackloggedCursors(); From 530e37ca4849bd2f8541a83dba957fc97bad7ebc Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Wed, 4 Jan 2023 22:26:50 +0800 Subject: [PATCH 15/20] Add the is_auto_consume_schema field in PulsarApi.proto to record whether the schema is AUTO_CONSUME. --- .../pulsar/broker/service/Consumer.java | 23 +++++--- .../pulsar/broker/service/ServerCnx.java | 5 +- .../broker/service/SubscriptionOption.java | 3 +- .../apache/pulsar/broker/service/Topic.java | 4 +- .../nonpersistent/NonPersistentTopic.java | 15 +++-- .../service/persistent/PersistentTopic.java | 14 ++--- .../pulsar/broker/service/ConsumerTest.java | 2 +- .../service/MessageCumulativeAckTest.java | 6 +- ...sistentDispatcherFailoverConsumerTest.java | 18 +++--- .../broker/service/PersistentTopicTest.java | 57 ++++++++----------- .../pulsar/common/protocol/Commands.java | 7 ++- pulsar-common/src/main/proto/PulsarApi.proto | 2 +- 12 files changed, 75 insertions(+), 81 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 6858b84172613..38a175797f79f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -58,7 +58,6 @@ import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.protocol.Commands; -import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.stats.Rate; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; @@ -143,14 +142,24 @@ public class Consumer { private long negtiveUnackedMsgsTimestamp; - private final SchemaType schemaType; + @Getter + private final boolean isAutoConsumeSchema; public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId, int priorityLevel, String consumerName, boolean isDurable, TransportCnx cnx, String appId, Map metadata, boolean readCompacted, - KeySharedMeta keySharedMeta, MessageId startMessageId, long consumerEpoch, SchemaType schemaType) { + KeySharedMeta keySharedMeta, MessageId startMessageId, long consumerEpoch) { + this(subscription, subType, topicName, consumerId, priorityLevel, consumerName, isDurable, cnx, appId, + metadata, readCompacted, keySharedMeta, startMessageId, consumerEpoch, false); + } + public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId, + int priorityLevel, String consumerName, + boolean isDurable, TransportCnx cnx, String appId, + Map metadata, boolean readCompacted, + KeySharedMeta keySharedMeta, MessageId startMessageId, + long consumerEpoch, boolean isAutoConsumeSchema) { this.subscription = subscription; this.subType = subType; this.topicName = topicName; @@ -208,7 +217,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.isAcknowledgmentAtBatchIndexLevelEnabled = subscription.getTopic().getBrokerService() .getPulsar().getConfiguration().isAcknowledgmentAtBatchIndexLevelEnabled(); - this.schemaType = schemaType; + this.isAutoConsumeSchema = isAutoConsumeSchema; } @VisibleForTesting @@ -236,7 +245,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.clientAddress = null; this.startMessageId = null; this.isAcknowledgmentAtBatchIndexLevelEnabled = false; - this.schemaType = null; + this.isAutoConsumeSchema = false; MESSAGE_PERMITS_UPDATER.set(this, availablePermits); } @@ -1104,10 +1113,6 @@ public Map getMetadata() { return metadata; } - public SchemaType getSchemaType() { - return schemaType; - } - private int getStickyKeyHash(Entry entry) { final byte[] stickyKey; if (entry instanceof EntryAndMetadata) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index b5e68893596f2..2be5e46277b62 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1021,6 +1021,7 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { ? subscribe.getStartMessageRollbackDurationSec() : -1; final SchemaData schema = subscribe.hasSchema() ? getSchema(subscribe.getSchema()) : null; + final boolean isAutoConsumeSchema = subscribe.hasSchema() && subscribe.getSchema().isIsAutoConsumeSchema(); final boolean isReplicated = subscribe.hasReplicateSubscriptionState() && subscribe.isReplicateSubscriptionState(); final boolean forceTopicCreation = subscribe.isForceTopicCreation(); @@ -1129,9 +1130,9 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { .replicatedSubscriptionStateArg(isReplicated).keySharedMeta(keySharedMeta) .subscriptionProperties(subscriptionProperties) .consumerEpoch(consumerEpoch) - .schemaType(schema == null ? null : schema.getType()) + .isAutoConsumeSchema(isAutoConsumeSchema) .build(); - if (schema != null && schema.getType() != SchemaType.AUTO_CONSUME) { + if (schema != null && !isAutoConsumeSchema) { return topic.addSchemaIfIdleOrCheckCompatible(schema) .thenCompose(v -> topic.subscribe(option)); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java index af56d023616b4..9c04909019099 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java @@ -29,7 +29,6 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.KeyValue; -import org.apache.pulsar.common.schema.SchemaType; @Getter @Builder @@ -50,7 +49,7 @@ public class SubscriptionOption { private KeySharedMeta keySharedMeta; private Optional> subscriptionProperties; private long consumerEpoch; - private SchemaType schemaType; + private boolean isAutoConsumeSchema; public static Optional> getPropertiesMap(List list) { if (list == null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java index 11e2d04b581b0..3949df92ceca5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Topic.java @@ -43,7 +43,6 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.schema.SchemaData; import org.apache.pulsar.common.protocol.schema.SchemaVersion; -import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.utils.StatsOutputStream; @@ -170,8 +169,7 @@ CompletableFuture subscribe(TransportCnx cnx, String subscriptionName, Map metadata, boolean readCompacted, InitialPosition initialPosition, long startMessageRollbackDurationSec, boolean replicateSubscriptionState, - KeySharedMeta keySharedMeta, - SchemaType schemaType); + KeySharedMeta keySharedMeta); /** * Subscribe a topic. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 3ec2abc9402a7..bbef09845343e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -87,7 +87,6 @@ import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; import org.apache.pulsar.common.protocol.schema.SchemaData; -import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -256,7 +255,8 @@ public CompletableFuture subscribe(SubscriptionOption option) { option.isDurable(), option.getStartMessageId(), option.getMetadata(), option.isReadCompacted(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), - option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(null), option.getSchemaType()); + option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(null), + option.isAutoConsumeSchema()); } @Override @@ -266,11 +266,10 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs Map metadata, boolean readCompacted, InitialPosition initialPosition, long resetStartMessageBackInSec, boolean replicateSubscriptionState, - KeySharedMeta keySharedMeta, - SchemaType schemaType) { + KeySharedMeta keySharedMeta) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, resetStartMessageBackInSec, - replicateSubscriptionState, keySharedMeta, null, schemaType); + replicateSubscriptionState, keySharedMeta, null, false); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -282,7 +281,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St boolean replicateSubscriptionState, KeySharedMeta keySharedMeta, Map subscriptionProperties, - SchemaType schemaType) { + boolean isAutoConsumeSchema) { return brokerService.checkTopicNsOwnership(getName()).thenCompose(__ -> { final CompletableFuture future = new CompletableFuture<>(); @@ -325,7 +324,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest, - DEFAULT_CONSUMER_EPOCH, schemaType); + DEFAULT_CONSUMER_EPOCH, isAutoConsumeSchema); if (isMigrated()) { consumer.topicMigrated(getClusterMigrationUrl()); } @@ -1167,7 +1166,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem return hasSchema().thenCompose((hasSchema) -> { int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() .mapToInt(subscription -> subscription.getConsumers().stream() - .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) + .filter(consumer -> !consumer.isAutoConsumeSchema()) .toList().size()) .sum(); if (hasSchema diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index b1fad866956c8..2c1f9a83f9627 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -151,7 +151,6 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.SchemaData; -import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; @@ -709,7 +708,7 @@ public CompletableFuture subscribe(SubscriptionOption option) { option.getInitialPosition(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(Collections.emptyMap()), - option.getConsumerEpoch(), option.getSchemaType()); + option.getConsumerEpoch(), option.isAutoConsumeSchema()); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -723,7 +722,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St KeySharedMeta keySharedMeta, Map subscriptionProperties, long consumerEpoch, - SchemaType schemaType) { + boolean isAutoConsumeSchema) { if (readCompacted && !(subType == SubType.Failover || subType == SubType.Exclusive)) { return FutureUtil.failedFuture(new NotAllowedException( "readCompacted only allowed on failover or exclusive subscriptions")); @@ -811,7 +810,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St CompletableFuture future = subscriptionFuture.thenCompose(subscription -> { Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, isDurable, cnx, cnx.getAuthRole(), metadata, - readCompacted, keySharedMeta, startMessageId, consumerEpoch, schemaType); + readCompacted, keySharedMeta, startMessageId, consumerEpoch, isAutoConsumeSchema); return addConsumerToSubscription(subscription, consumer).thenCompose(v -> { checkBackloggedCursors(); @@ -885,11 +884,10 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs InitialPosition initialPosition, long startMessageRollbackDurationSec, boolean replicatedSubscriptionStateArg, - KeySharedMeta keySharedMeta, - SchemaType schemaType) { + KeySharedMeta keySharedMeta) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, initialPosition, startMessageRollbackDurationSec, - replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH, schemaType); + replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH, false); } private CompletableFuture getDurableSubscription(String subscriptionName, @@ -3060,7 +3058,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem return hasSchema().thenCompose((hasSchema) -> { int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() .mapToInt(subscription -> subscription.getConsumers().stream() - .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) + .filter(consumer -> !consumer.isAutoConsumeSchema()) .toList().size()) .sum(); if (hasSchema diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java index a7b31ee8c6f1e..a974655b06c27 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsumerTest.java @@ -57,7 +57,7 @@ public void beforeMethod() { consumer = new Consumer(subscription, Exclusive, "topic", 1, 0, "Cons1", true, cnx, "myrole-1", emptyMap(), false, - new KeySharedMeta().setKeySharedMode(AUTO_SPLIT), latest, DEFAULT_CONSUMER_EPOCH, null); + new KeySharedMeta().setKeySharedMode(AUTO_SPLIT), latest, DEFAULT_CONSUMER_EPOCH); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java index 7e777f0a2758e..6b106bfd47de7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessageCumulativeAckTest.java @@ -160,7 +160,7 @@ public static Object[][] notIndividualAckModes() { public void testAckWithIndividualAckMode(CommandSubscribe.SubType subType) throws Exception { Consumer consumer = new Consumer(sub, subType, "topic-1", consumerId, 0, "Cons1", true, serverCnx, "myrole-1", emptyMap(), false, null, - MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + MessageId.latest, DEFAULT_CONSUMER_EPOCH); CommandAck commandAck = new CommandAck(); commandAck.setAckType(Cumulative); @@ -175,7 +175,7 @@ public void testAckWithIndividualAckMode(CommandSubscribe.SubType subType) throw public void testAckWithNotIndividualAckMode(CommandSubscribe.SubType subType) throws Exception { Consumer consumer = new Consumer(sub, subType, "topic-1", consumerId, 0, "Cons1", true, serverCnx, "myrole-1", emptyMap(), false, null, - MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + MessageId.latest, DEFAULT_CONSUMER_EPOCH); CommandAck commandAck = new CommandAck(); commandAck.setAckType(Cumulative); @@ -190,7 +190,7 @@ public void testAckWithNotIndividualAckMode(CommandSubscribe.SubType subType) th public void testAckWithMoreThanNoneMessageIds() throws Exception { Consumer consumer = new Consumer(sub, Failover, "topic-1", consumerId, 0, "Cons1", true, serverCnx, "myrole-1", emptyMap(), false, null, - MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + MessageId.latest, DEFAULT_CONSUMER_EPOCH); CommandAck commandAck = new CommandAck(); commandAck.setAckType(Cumulative); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java index 4a348c1341076..5381a367afd00 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentDispatcherFailoverConsumerTest.java @@ -315,7 +315,7 @@ public void testConsumerGroupChangesWithOldNewConsumers() throws Exception { // 2. Add old consumer Consumer consumer1 = new Consumer(sub, SubType.Exclusive, topic.getName(), 1 /* consumer id */, 0, - "Cons1"/* consumer name */, true, serverCnxWithOldVersion, "myrole-1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + "Cons1"/* consumer name */, true, serverCnxWithOldVersion, "myrole-1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); pdfc.addConsumer(consumer1); List consumers = pdfc.getConsumers(); assertSame(consumers.get(0).consumerName(), consumer1.consumerName()); @@ -326,7 +326,7 @@ public void testConsumerGroupChangesWithOldNewConsumers() throws Exception { // 3. Add new consumer Consumer consumer2 = new Consumer(sub, SubType.Exclusive, topic.getName(), 2 /* consumer id */, 0, - "Cons2"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + "Cons2"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); pdfc.addConsumer(consumer2); consumers = pdfc.getConsumers(); assertSame(consumers.get(0).consumerName(), consumer1.consumerName()); @@ -355,7 +355,7 @@ public void testAddRemoveConsumer() throws Exception { // 2. Add consumer Consumer consumer1 = spy(new Consumer(sub, SubType.Exclusive, topic.getName(), 1 /* consumer id */, 0, "Cons1"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null)); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); pdfc.addConsumer(consumer1); List consumers = pdfc.getConsumers(); assertSame(consumers.get(0).consumerName(), consumer1.consumerName()); @@ -380,7 +380,7 @@ public void testAddRemoveConsumer() throws Exception { // 5. Add another consumer which does not change active consumer Consumer consumer2 = spy(new Consumer(sub, SubType.Exclusive, topic.getName(), 2 /* consumer id */, 0, "Cons2"/* consumer name */, - true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null)); + true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); pdfc.addConsumer(consumer2); consumers = pdfc.getConsumers(); assertSame(pdfc.getActiveConsumer().consumerName(), consumer1.consumerName()); @@ -395,7 +395,7 @@ public void testAddRemoveConsumer() throws Exception { // 6. Add a consumer which changes active consumer Consumer consumer0 = spy(new Consumer(sub, SubType.Exclusive, topic.getName(), 0 /* consumer id */, 0, "Cons0"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null)); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); pdfc.addConsumer(consumer0); consumers = pdfc.getConsumers(); assertSame(pdfc.getActiveConsumer().consumerName(), consumer0.consumerName()); @@ -487,7 +487,7 @@ public void testAddRemoveConsumerNonPartitionedTopic() throws Exception { // 2. Add a consumer Consumer consumer1 = spy(new Consumer(sub, SubType.Failover, topic.getName(), 1 /* consumer id */, 1, "Cons1"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null)); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); pdfc.addConsumer(consumer1); List consumers = pdfc.getConsumers(); assertEquals(1, consumers.size()); @@ -496,7 +496,7 @@ public void testAddRemoveConsumerNonPartitionedTopic() throws Exception { // 3. Add a consumer with same priority level and consumer name is smaller in lexicographic order. Consumer consumer2 = spy(new Consumer(sub, SubType.Failover, topic.getName(), 2 /* consumer id */, 1, "Cons2"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null)); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); pdfc.addConsumer(consumer2); // 4. Verify active consumer doesn't change @@ -509,7 +509,7 @@ public void testAddRemoveConsumerNonPartitionedTopic() throws Exception { // 5. Add another consumer which has higher priority level Consumer consumer3 = spy(new Consumer(sub, SubType.Failover, topic.getName(), 3 /* consumer id */, 0, "Cons3"/* consumer name */, - true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null)); + true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)); pdfc.addConsumer(consumer3); consumers = pdfc.getConsumers(); assertEquals(3, consumers.size()); @@ -700,7 +700,7 @@ private Consumer createConsumer(PersistentTopic topic, int priority, int permit, PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock, false); Consumer consumer = new Consumer(sub, SubType.Shared, "test-topic", id, priority, ""+id, true, - serverCnx, "appId", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest,DEFAULT_CONSUMER_EPOCH, null); + serverCnx, "appId", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest,DEFAULT_CONSUMER_EPOCH); try { consumer.flowPermits(permit); } catch (Exception e) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index bb0d957fc02e3..adb31f2d1a593 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -774,8 +774,7 @@ public void testChangeSubscriptionType() throws Exception { Consumer consumer = new Consumer(sub, SubType.Exclusive, topic.getName(), 1, 0, "Cons1", true, serverCnx, "myrole-1", Collections.emptyMap(), false, - new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT), MessageId.latest, - DEFAULT_CONSUMER_EPOCH, null); + new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT), MessageId.latest, DEFAULT_CONSUMER_EPOCH); sub.addConsumer(consumer); consumer.close(); @@ -786,8 +785,7 @@ public void testChangeSubscriptionType() throws Exception { consumer = new Consumer(sub, subType, topic.getName(), 1, 0, "Cons1", true, serverCnx, "myrole-1", Collections.emptyMap(), false, - new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT), MessageId.latest, - DEFAULT_CONSUMER_EPOCH, null); + new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT), MessageId.latest, DEFAULT_CONSUMER_EPOCH); sub.addConsumer(consumer); assertTrue(sub.getDispatcher().isConsumerConnected()); @@ -810,8 +808,7 @@ public void testAddRemoveConsumer() throws Exception { // 1. simple add consumer Consumer consumer = new Consumer(sub, SubType.Exclusive, topic.getName(), 1 /* consumer id */, 0, "Cons1"/* consumer name */, - true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, - DEFAULT_CONSUMER_EPOCH, null); + true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); sub.addConsumer(consumer); assertTrue(sub.getDispatcher().isConsumerConnected()); @@ -844,7 +841,7 @@ public void testAddRemoveConsumerDurableCursor() throws Exception { PersistentSubscription sub = new PersistentSubscription(topic, "non-durable-sub", cursorMock, false); Consumer consumer = new Consumer(sub, SubType.Exclusive, topic.getName(), 1, 0, "Cons1", true, serverCnx, - "myrole-1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + "myrole-1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); sub.addConsumer(consumer); assertFalse(sub.getDispatcher().isClosed()); @@ -886,14 +883,14 @@ private void testMaxConsumersShared() throws Exception { // 1. add consumer1 Consumer consumer = new Consumer(sub, SubType.Shared, topic.getName(), 1 /* consumer id */, 0, "Cons1"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); addConsumerToSubscription.invoke(topic, sub, consumer); assertEquals(sub.getConsumers().size(), 1); // 2. add consumer2 Consumer consumer2 = new Consumer(sub, SubType.Shared, topic.getName(), 2 /* consumer id */, 0, "Cons2"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); addConsumerToSubscription.invoke(topic, sub, consumer2); assertEquals(sub.getConsumers().size(), 2); @@ -901,7 +898,7 @@ private void testMaxConsumersShared() throws Exception { try { Consumer consumer3 = new Consumer(sub, SubType.Shared, topic.getName(), 3 /* consumer id */, 0, "Cons3"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub, consumer3)).get(); fail("should have failed"); } catch (ExecutionException e) { @@ -914,7 +911,7 @@ private void testMaxConsumersShared() throws Exception { // 4. add consumer4 to sub2 Consumer consumer4 = new Consumer(sub2, SubType.Shared, topic.getName(), 4 /* consumer id */, 0, "Cons4"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); addConsumerToSubscription.invoke(topic, sub2, consumer4); assertEquals(sub2.getConsumers().size(), 1); @@ -925,7 +922,7 @@ private void testMaxConsumersShared() throws Exception { try { Consumer consumer5 = new Consumer(sub2, SubType.Shared, topic.getName(), 5 /* consumer id */, 0, "Cons5"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer5)).get(); fail("should have failed"); } catch (ExecutionException e) { @@ -994,14 +991,14 @@ private void testMaxConsumersFailover() throws Exception { // 1. add consumer1 Consumer consumer = new Consumer(sub, SubType.Failover, topic.getName(), 1 /* consumer id */, 0, "Cons1"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); addConsumerToSubscription.invoke(topic, sub, consumer); assertEquals(sub.getConsumers().size(), 1); // 2. add consumer2 Consumer consumer2 = new Consumer(sub, SubType.Failover, topic.getName(), 2 /* consumer id */, 0, "Cons2"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); addConsumerToSubscription.invoke(topic, sub, consumer2); assertEquals(sub.getConsumers().size(), 2); @@ -1009,7 +1006,7 @@ private void testMaxConsumersFailover() throws Exception { try { Consumer consumer3 = new Consumer(sub, SubType.Failover, topic.getName(), 3 /* consumer id */, 0, "Cons3"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub, consumer3)).get(); fail("should have failed"); } catch (ExecutionException e) { @@ -1022,7 +1019,7 @@ private void testMaxConsumersFailover() throws Exception { // 4. add consumer4 to sub2 Consumer consumer4 = new Consumer(sub2, SubType.Failover, topic.getName(), 4 /* consumer id */, 0, "Cons4"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); addConsumerToSubscription.invoke(topic, sub2, consumer4); assertEquals(sub2.getConsumers().size(), 1); @@ -1033,7 +1030,7 @@ private void testMaxConsumersFailover() throws Exception { try { Consumer consumer5 = new Consumer(sub2, SubType.Failover, topic.getName(), 5 /* consumer id */, 0, "Cons5"/* consumer name */, true, serverCnx, "myrole-1", Collections.emptyMap(), - false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); ((CompletableFuture) addConsumerToSubscription.invoke(topic, sub2, consumer5)).get(); fail("should have failed"); } catch (ExecutionException e) { @@ -1087,7 +1084,7 @@ private Consumer getMockedConsumerWithSpecificAddress(Topic topic, Subscription doReturn(new PulsarCommandSenderImpl(null, cnx)).when(cnx).getCommandSender(); return new Consumer(sub, SubType.Shared, topic.getName(), consumerId, 0, consumerNameBase + consumerId, true, - cnx, role, Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + cnx, role, Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); } @Test @@ -1199,8 +1196,7 @@ public void testUbsubscribeRaceConditions() throws Exception { PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService); PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock, false); Consumer consumer1 = new Consumer(sub, SubType.Exclusive, topic.getName(), 1 /* consumer id */, 0, "Cons1"/* consumer name */, - true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, - DEFAULT_CONSUMER_EPOCH, null); + true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); sub.addConsumer(consumer1); doAnswer(new Answer() { @@ -1224,8 +1220,7 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { Thread.sleep(10); /* delay to ensure that the ubsubscribe gets executed first */ sub.addConsumer(new Consumer(sub, SubType.Exclusive, topic.getName(), 2 /* consumer id */, 0, "Cons2"/* consumer name */, true, serverCnx, - "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, - DEFAULT_CONSUMER_EPOCH, null)).get(); + "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH)).get(); fail(); } catch (Exception e) { assertTrue(e.getCause() instanceof BrokerServiceException.SubscriptionFencedException); @@ -2009,24 +2004,21 @@ public void testBacklogCursor() throws Exception { ManagedCursor cursor1 = ledger.openCursor("c1"); PersistentSubscription sub1 = new PersistentSubscription(topic, "sub-1", cursor1, false); Consumer consumer1 = new Consumer(sub1, SubType.Exclusive, topic.getName(), 1 /* consumer id */, 0, "Cons1"/* consumer name */, - true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, - DEFAULT_CONSUMER_EPOCH, null); + true, serverCnx, "myrole-1", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); topic.getSubscriptions().put(Codec.decode(cursor1.getName()), sub1); sub1.addConsumer(consumer1); // Open cursor2, add it into activeCursor-container and add it into subscription consumer list ManagedCursor cursor2 = ledger.openCursor("c2"); PersistentSubscription sub2 = new PersistentSubscription(topic, "sub-2", cursor2, false); Consumer consumer2 = new Consumer(sub2, SubType.Exclusive, topic.getName(), 2 /* consumer id */, 0, "Cons2"/* consumer name */, - true, serverCnx, "myrole-2", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, - DEFAULT_CONSUMER_EPOCH, null); + true, serverCnx, "myrole-2", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); topic.getSubscriptions().put(Codec.decode(cursor2.getName()), sub2); sub2.addConsumer(consumer2); // Open cursor3, add it into activeCursor-container and do not add it into subscription consumer list ManagedCursor cursor3 = ledger.openCursor("c3"); PersistentSubscription sub3 = new PersistentSubscription(topic, "sub-3", cursor3, false); Consumer consumer3 = new Consumer(sub2, SubType.Exclusive, topic.getName(), 3 /* consumer id */, 0, "Cons2"/* consumer name */, - true, serverCnx, "myrole-3", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, - DEFAULT_CONSUMER_EPOCH, null); + true, serverCnx, "myrole-3", Collections.emptyMap(), false /* read compacted */, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); topic.getSubscriptions().put(Codec.decode(cursor3.getName()), sub3); // Case1: cursors are active as haven't started deactivateBacklogCursor scan @@ -2140,8 +2132,7 @@ public void testCheckInactiveSubscriptions() throws Exception { addConsumerToSubscription.setAccessible(true); Consumer consumer = new Consumer(nonDeletableSubscription1, SubType.Shared, topic.getName(), 1, 0, "consumer1", - true, serverCnx, "app1", Collections.emptyMap(), false, null, MessageId.latest, - DEFAULT_CONSUMER_EPOCH, null); + true, serverCnx, "app1", Collections.emptyMap(), false, null, MessageId.latest, DEFAULT_CONSUMER_EPOCH); addConsumerToSubscription.invoke(topic, nonDeletableSubscription1, consumer); NamespaceResources nsr = pulsar.getPulsarResources().getNamespaceResources(); @@ -2286,7 +2277,7 @@ public void testKeySharedMetadataExposedToStats() throws Exception { Consumer consumer1 = new Consumer(sub1, SubType.Key_Shared, topic.getName(), 1, 0, "Cons1", true, serverCnx, "myrole-1", Collections.emptyMap(), false, new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT).setAllowOutOfOrderDelivery(false), - MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + MessageId.latest, DEFAULT_CONSUMER_EPOCH); sub1.addConsumer(consumer1); consumer1.close(); @@ -2297,7 +2288,7 @@ public void testKeySharedMetadataExposedToStats() throws Exception { Consumer consumer2 = new Consumer(sub2, SubType.Key_Shared, topic.getName(), 2, 0, "Cons2", true, serverCnx, "myrole-1", Collections.emptyMap(), false, new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT).setAllowOutOfOrderDelivery(true), - MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + MessageId.latest, DEFAULT_CONSUMER_EPOCH); sub2.addConsumer(consumer2); consumer2.close(); @@ -2309,7 +2300,7 @@ public void testKeySharedMetadataExposedToStats() throws Exception { .setAllowOutOfOrderDelivery(false); ksm.addHashRange().setStart(0).setEnd(65535); Consumer consumer3 = new Consumer(sub3, SubType.Key_Shared, topic.getName(), 3, 0, "Cons3", true, serverCnx, - "myrole-1", Collections.emptyMap(), false, ksm, MessageId.latest, DEFAULT_CONSUMER_EPOCH, null); + "myrole-1", Collections.emptyMap(), false, ksm, MessageId.latest, DEFAULT_CONSUMER_EPOCH); sub3.addConsumer(consumer3); consumer3.close(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index deb9606942799..b5565dabb034b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -771,7 +771,7 @@ public static ByteBuf newProducer(String topic, long producerId, long requestId, } private static Schema.Type getSchemaType(SchemaType type) { - if (type.getValue() < 0 && type.getValue() != -3) { + if (type.getValue() < 0) { return Schema.Type.None; } else { return Schema.Type.valueOf(type.getValue()); @@ -779,7 +779,7 @@ private static Schema.Type getSchemaType(SchemaType type) { } public static SchemaType getSchemaType(Schema.Type type) { - if (type.getValue() < 0 && type.getValue() != -3) { + if (type.getValue() < 0) { // this is unexpected return SchemaType.NONE; } else { @@ -791,6 +791,9 @@ private static void convertSchema(SchemaInfo schemaInfo, Schema schema) { schema.setName(schemaInfo.getName()) .setSchemaData(schemaInfo.getSchema()) .setType(getSchemaType(schemaInfo.getType())); + if (schemaInfo.getType() == SchemaType.AUTO_CONSUME) { + schema.setIsAutoConsumeSchema(true); + } schemaInfo.getProperties().entrySet().stream().forEach(entry -> { if (entry.getKey() != null && entry.getValue() != null) { diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto index 9f5d405583c0b..9324f47653178 100644 --- a/pulsar-common/src/main/proto/PulsarApi.proto +++ b/pulsar-common/src/main/proto/PulsarApi.proto @@ -45,13 +45,13 @@ message Schema { LocalTime = 18; LocalDateTime = 19; ProtobufNative = 20; - AutoConsume = -3; } required string name = 1; required bytes schema_data = 3; required Type type = 4; repeated KeyValue properties = 5; + optional bool is_auto_consume_schema = 6 [default = false]; } From 560c51d550b1a5143113958c4f16fc2a9f2c2f28 Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Mon, 16 Jan 2023 22:24:02 +0800 Subject: [PATCH 16/20] Add new proto version in ProtocolVersion and carry AUTO_CONSUME schema to broker. --- .../pulsar/broker/service/Consumer.java | 11 ++++---- .../pulsar/broker/service/ServerCnx.java | 8 +++--- .../broker/service/SubscriptionOption.java | 3 ++- .../nonpersistent/NonPersistentTopic.java | 11 ++++---- .../service/persistent/PersistentTopic.java | 11 ++++---- .../pulsar/client/impl/ConsumerImpl.java | 3 ++- .../pulsar/common/protocol/Commands.java | 27 ++++++++++++++++--- pulsar-common/src/main/proto/PulsarApi.proto | 3 ++- 8 files changed, 51 insertions(+), 26 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 38a175797f79f..1ee3f513ef288 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -58,6 +58,7 @@ import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.stats.Rate; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; @@ -143,7 +144,7 @@ public class Consumer { private long negtiveUnackedMsgsTimestamp; @Getter - private final boolean isAutoConsumeSchema; + private final SchemaType schemaType; public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId, int priorityLevel, String consumerName, @@ -151,7 +152,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo Map metadata, boolean readCompacted, KeySharedMeta keySharedMeta, MessageId startMessageId, long consumerEpoch) { this(subscription, subType, topicName, consumerId, priorityLevel, consumerName, isDurable, cnx, appId, - metadata, readCompacted, keySharedMeta, startMessageId, consumerEpoch, false); + metadata, readCompacted, keySharedMeta, startMessageId, consumerEpoch, null); } public Consumer(Subscription subscription, SubType subType, String topicName, long consumerId, @@ -159,7 +160,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo boolean isDurable, TransportCnx cnx, String appId, Map metadata, boolean readCompacted, KeySharedMeta keySharedMeta, MessageId startMessageId, - long consumerEpoch, boolean isAutoConsumeSchema) { + long consumerEpoch, SchemaType schemaType) { this.subscription = subscription; this.subType = subType; this.topicName = topicName; @@ -217,7 +218,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.isAcknowledgmentAtBatchIndexLevelEnabled = subscription.getTopic().getBrokerService() .getPulsar().getConfiguration().isAcknowledgmentAtBatchIndexLevelEnabled(); - this.isAutoConsumeSchema = isAutoConsumeSchema; + this.schemaType = schemaType; } @VisibleForTesting @@ -245,7 +246,7 @@ public Consumer(Subscription subscription, SubType subType, String topicName, lo this.clientAddress = null; this.startMessageId = null; this.isAcknowledgmentAtBatchIndexLevelEnabled = false; - this.isAutoConsumeSchema = false; + this.schemaType = null; MESSAGE_PERMITS_UPDATER.set(this, availablePermits); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 2be5e46277b62..ad1e72978ab75 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1021,7 +1021,6 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { ? subscribe.getStartMessageRollbackDurationSec() : -1; final SchemaData schema = subscribe.hasSchema() ? getSchema(subscribe.getSchema()) : null; - final boolean isAutoConsumeSchema = subscribe.hasSchema() && subscribe.getSchema().isIsAutoConsumeSchema(); final boolean isReplicated = subscribe.hasReplicateSubscriptionState() && subscribe.isReplicateSubscriptionState(); final boolean forceTopicCreation = subscribe.isForceTopicCreation(); @@ -1130,9 +1129,9 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { .replicatedSubscriptionStateArg(isReplicated).keySharedMeta(keySharedMeta) .subscriptionProperties(subscriptionProperties) .consumerEpoch(consumerEpoch) - .isAutoConsumeSchema(isAutoConsumeSchema) + .schemaType(schema == null ? null : schema.getType()) .build(); - if (schema != null && !isAutoConsumeSchema) { + if (schema != null && schema.getType() != SchemaType.AUTO_CONSUME) { return topic.addSchemaIfIdleOrCheckCompatible(schema) .thenCompose(v -> topic.subscribe(option)); } else { @@ -1214,7 +1213,8 @@ private SchemaData getSchema(Schema protocolSchema) { .isDeleted(false) .timestamp(System.currentTimeMillis()) .user(Strings.nullToEmpty(originalPrincipal)) - .type(Commands.getSchemaType(protocolSchema.getType())) + .type(protocolSchema.getType() == Schema.Type.AutoConsume + ? SchemaType.AUTO_CONSUME : Commands.getSchemaType(protocolSchema.getType())) .props(protocolSchema.getPropertiesList().stream().collect( Collectors.toMap( KeyValue::getKey, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java index 9c04909019099..af56d023616b4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SubscriptionOption.java @@ -29,6 +29,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.KeyValue; +import org.apache.pulsar.common.schema.SchemaType; @Getter @Builder @@ -49,7 +50,7 @@ public class SubscriptionOption { private KeySharedMeta keySharedMeta; private Optional> subscriptionProperties; private long consumerEpoch; - private boolean isAutoConsumeSchema; + private SchemaType schemaType; public static Optional> getPropertiesMap(List list) { if (list == null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index bbef09845343e..b56ec30d7b873 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -87,6 +87,7 @@ import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; import org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl; import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -256,7 +257,7 @@ public CompletableFuture subscribe(SubscriptionOption option) { option.isReadCompacted(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(null), - option.isAutoConsumeSchema()); + option.getSchemaType()); } @Override @@ -269,7 +270,7 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs KeySharedMeta keySharedMeta) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, resetStartMessageBackInSec, - replicateSubscriptionState, keySharedMeta, null, false); + replicateSubscriptionState, keySharedMeta, null, null); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -281,7 +282,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St boolean replicateSubscriptionState, KeySharedMeta keySharedMeta, Map subscriptionProperties, - boolean isAutoConsumeSchema) { + SchemaType schemaType) { return brokerService.checkTopicNsOwnership(getName()).thenCompose(__ -> { final CompletableFuture future = new CompletableFuture<>(); @@ -324,7 +325,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, false, cnx, cnx.getAuthRole(), metadata, readCompacted, keySharedMeta, MessageId.latest, - DEFAULT_CONSUMER_EPOCH, isAutoConsumeSchema); + DEFAULT_CONSUMER_EPOCH, schemaType); if (isMigrated()) { consumer.topicMigrated(getClusterMigrationUrl()); } @@ -1166,7 +1167,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem return hasSchema().thenCompose((hasSchema) -> { int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() .mapToInt(subscription -> subscription.getConsumers().stream() - .filter(consumer -> !consumer.isAutoConsumeSchema()) + .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) .toList().size()) .sum(); if (hasSchema diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 2c1f9a83f9627..ced407ea2eb68 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -151,6 +151,7 @@ import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.schema.SchemaData; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; @@ -708,7 +709,7 @@ public CompletableFuture subscribe(SubscriptionOption option) { option.getInitialPosition(), option.getStartMessageRollbackDurationSec(), option.isReplicatedSubscriptionStateArg(), option.getKeySharedMeta(), option.getSubscriptionProperties().orElse(Collections.emptyMap()), - option.getConsumerEpoch(), option.isAutoConsumeSchema()); + option.getConsumerEpoch(), option.getSchemaType()); } private CompletableFuture internalSubscribe(final TransportCnx cnx, String subscriptionName, @@ -722,7 +723,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St KeySharedMeta keySharedMeta, Map subscriptionProperties, long consumerEpoch, - boolean isAutoConsumeSchema) { + SchemaType schemaType) { if (readCompacted && !(subType == SubType.Failover || subType == SubType.Exclusive)) { return FutureUtil.failedFuture(new NotAllowedException( "readCompacted only allowed on failover or exclusive subscriptions")); @@ -810,7 +811,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St CompletableFuture future = subscriptionFuture.thenCompose(subscription -> { Consumer consumer = new Consumer(subscription, subType, topic, consumerId, priorityLevel, consumerName, isDurable, cnx, cnx.getAuthRole(), metadata, - readCompacted, keySharedMeta, startMessageId, consumerEpoch, isAutoConsumeSchema); + readCompacted, keySharedMeta, startMessageId, consumerEpoch, schemaType); return addConsumerToSubscription(subscription, consumer).thenCompose(v -> { checkBackloggedCursors(); @@ -887,7 +888,7 @@ public CompletableFuture subscribe(final TransportCnx cnx, String subs KeySharedMeta keySharedMeta) { return internalSubscribe(cnx, subscriptionName, consumerId, subType, priorityLevel, consumerName, isDurable, startMessageId, metadata, readCompacted, initialPosition, startMessageRollbackDurationSec, - replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH, false); + replicatedSubscriptionStateArg, keySharedMeta, null, DEFAULT_CONSUMER_EPOCH, null); } private CompletableFuture getDurableSubscription(String subscriptionName, @@ -3058,7 +3059,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem return hasSchema().thenCompose((hasSchema) -> { int numActiveConsumersWithoutAutoSchema = subscriptions.values().stream() .mapToInt(subscription -> subscription.getConsumers().stream() - .filter(consumer -> !consumer.isAutoConsumeSchema()) + .filter(consumer -> consumer.getSchemaType() != SchemaType.AUTO_CONSUME) .toList().size()) .sum(); if (hasSchema diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 16814f8615f02..4473d86edbd87 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -816,7 +816,8 @@ public void connectionOpened(final ClientCnx cnx) { // don't set schema for Schema.BYTES si = null; } else { - if (schema instanceof AutoConsumeSchema) { + if (schema instanceof AutoConsumeSchema + && Commands.peerSupportsCarryAutoConsumeSchemaToBroker(cnx.getRemoteEndpointProtocolVersion())) { si = ((AutoConsumeSchema) schema).getAutoConsumeSchemaInfo(); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index b5565dabb034b..d6aa11eec4fcc 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -647,7 +647,11 @@ public static ByteBuf newSubscribe(String topic, String subscription, long consu throw new IllegalStateException(); } - convertSchema(schemaInfo, subscribe.setSchema()); + if (schemaInfo.getType() == SchemaType.AUTO_CONSUME) { + convertAutoConsumeSchema(schemaInfo, subscribe.setSchema()); + } else { + convertSchema(schemaInfo, subscribe.setSchema()); + } } return serializeWithSize(cmd); @@ -791,9 +795,20 @@ private static void convertSchema(SchemaInfo schemaInfo, Schema schema) { schema.setName(schemaInfo.getName()) .setSchemaData(schemaInfo.getSchema()) .setType(getSchemaType(schemaInfo.getType())); - if (schemaInfo.getType() == SchemaType.AUTO_CONSUME) { - schema.setIsAutoConsumeSchema(true); - } + + schemaInfo.getProperties().entrySet().stream().forEach(entry -> { + if (entry.getKey() != null && entry.getValue() != null) { + schema.addProperty() + .setKey(entry.getKey()) + .setValue(entry.getValue()); + } + }); + } + + private static void convertAutoConsumeSchema(SchemaInfo schemaInfo, Schema schema) { + schema.setName(schemaInfo.getName()) + .setSchemaData(schemaInfo.getSchema()) + .setType(Schema.Type.AutoConsume); schemaInfo.getProperties().entrySet().stream().forEach(entry -> { if (entry.getKey() != null && entry.getValue() != null) { @@ -1962,6 +1977,10 @@ public static boolean peerSupportsAckReceipt(int peerVersion) { return peerVersion >= ProtocolVersion.v17.getValue(); } + public static boolean peerSupportsCarryAutoConsumeSchemaToBroker(int peerVersion) { + return peerVersion >= ProtocolVersion.v21.getValue(); + } + private static org.apache.pulsar.common.api.proto.ProducerAccessMode convertProducerAccessMode( ProducerAccessMode accessMode) { switch (accessMode) { diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto index 9324f47653178..16fa99e34f022 100644 --- a/pulsar-common/src/main/proto/PulsarApi.proto +++ b/pulsar-common/src/main/proto/PulsarApi.proto @@ -45,13 +45,13 @@ message Schema { LocalTime = 18; LocalDateTime = 19; ProtobufNative = 20; + AutoConsume = 100; } required string name = 1; required bytes schema_data = 3; required Type type = 4; repeated KeyValue properties = 5; - optional bool is_auto_consume_schema = 6 [default = false]; } @@ -264,6 +264,7 @@ enum ProtocolVersion { v18 = 18; // Add client support for broker entry metadata v19 = 19; // Add CommandTcClientConnectRequest and CommandTcClientConnectResponse v20 = 20; // Add client support for topic migration redirection CommandTopicMigrated + v21 = 21; // Carry the AUTO_CONSUME schema to the Broker after this version } message CommandConnect { From 59efefc213ca8389d0f78e6f58518c22da1ca4d8 Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Tue, 17 Jan 2023 21:49:53 +0800 Subject: [PATCH 17/20] change new enum value --- pulsar-common/src/main/proto/PulsarApi.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-common/src/main/proto/PulsarApi.proto b/pulsar-common/src/main/proto/PulsarApi.proto index 16fa99e34f022..d9c41eeec9740 100644 --- a/pulsar-common/src/main/proto/PulsarApi.proto +++ b/pulsar-common/src/main/proto/PulsarApi.proto @@ -45,7 +45,7 @@ message Schema { LocalTime = 18; LocalDateTime = 19; ProtobufNative = 20; - AutoConsume = 100; + AutoConsume = 21; } required string name = 1; From 276b462d3be4ca00a53b50d57115b48a30de172e Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Wed, 1 Feb 2023 16:55:11 +0800 Subject: [PATCH 18/20] change SCHEMA_INFO public and reference AutoConsumerSchema.SCHEMA_INFO. Reduce the duplication of convertAutoConsumeSchema code and reuse convertSchema. --- .../pulsar/client/impl/ConsumerImpl.java | 2 +- .../client/impl/schema/AutoConsumeSchema.java | 18 +++++------------- .../pulsar/common/protocol/Commands.java | 13 ++----------- 3 files changed, 8 insertions(+), 25 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index dd1568f328170..2775050050f85 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -815,7 +815,7 @@ public void connectionOpened(final ClientCnx cnx) { } else { if (schema instanceof AutoConsumeSchema && Commands.peerSupportsCarryAutoConsumeSchemaToBroker(cnx.getRemoteEndpointProtocolVersion())) { - si = ((AutoConsumeSchema) schema).getAutoConsumeSchemaInfo(); + si = AutoConsumeSchema.SCHEMA_INFO; } } // startMessageRollbackDurationInSec should be consider only once when consumer connects to first time diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java index de61d7fa3f49e..82a3b69da20b6 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AutoConsumeSchema.java @@ -57,15 +57,11 @@ public class AutoConsumeSchema implements Schema { private SchemaInfoProvider schemaInfoProvider; - private static final SchemaInfo SCHEMA_INFO; - - static { - SCHEMA_INFO = SchemaInfoImpl.builder() - .name("AutoConsume") - .type(SchemaType.AUTO_CONSUME) - .schema(new byte[0]) - .build(); - } + public static final SchemaInfo SCHEMA_INFO = SchemaInfoImpl.builder() + .name("AutoConsume") + .type(SchemaType.AUTO_CONSUME) + .schema(new byte[0]) + .build(); private ConcurrentMap> initSchemaMap() { ConcurrentMap> schemaMap = new ConcurrentHashMap<>(); @@ -163,10 +159,6 @@ public SchemaInfo getSchemaInfo(byte[] schemaVersion) { return null; } - public SchemaInfo getAutoConsumeSchemaInfo() { - return SCHEMA_INFO; - } - @Override public void configureSchemaInfo(String topicName, String componentName, diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index 084ae39d01829..cf4c1e2b118e9 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -806,17 +806,8 @@ private static void convertSchema(SchemaInfo schemaInfo, Schema schema) { } private static void convertAutoConsumeSchema(SchemaInfo schemaInfo, Schema schema) { - schema.setName(schemaInfo.getName()) - .setSchemaData(schemaInfo.getSchema()) - .setType(Schema.Type.AutoConsume); - - schemaInfo.getProperties().entrySet().stream().forEach(entry -> { - if (entry.getKey() != null && entry.getValue() != null) { - schema.addProperty() - .setKey(entry.getKey()) - .setValue(entry.getValue()); - } - }); + convertSchema(schemaInfo, schema); + schema.setType(Schema.Type.AutoConsume); } public static ByteBuf newProducer(String topic, long producerId, long requestId, String producerName, From 4cc9c46b3c342d5c9a293fe8014992b495cb9b94 Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Wed, 1 Feb 2023 17:18:55 +0800 Subject: [PATCH 19/20] delete convertAutoConsumeSchema and set AutoConsume in getSchemaType. --- .../apache/pulsar/common/protocol/Commands.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index cf4c1e2b118e9..ad49757efb209 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -647,11 +647,7 @@ public static ByteBuf newSubscribe(String topic, String subscription, long consu throw new IllegalStateException(); } - if (schemaInfo.getType() == SchemaType.AUTO_CONSUME) { - convertAutoConsumeSchema(schemaInfo, subscribe.setSchema()); - } else { - convertSchema(schemaInfo, subscribe.setSchema()); - } + convertSchema(schemaInfo, subscribe.setSchema()); } return serializeWithSize(cmd); @@ -775,7 +771,9 @@ public static ByteBuf newProducer(String topic, long producerId, long requestId, } private static Schema.Type getSchemaType(SchemaType type) { - if (type.getValue() < 0) { + if (type == SchemaType.AUTO_CONSUME) { + return Schema.Type.AutoConsume; + } else if (type.getValue() < 0) { return Schema.Type.None; } else { return Schema.Type.valueOf(type.getValue()); @@ -805,11 +803,6 @@ private static void convertSchema(SchemaInfo schemaInfo, Schema schema) { }); } - private static void convertAutoConsumeSchema(SchemaInfo schemaInfo, Schema schema) { - convertSchema(schemaInfo, schema); - schema.setType(Schema.Type.AutoConsume); - } - public static ByteBuf newProducer(String topic, long producerId, long requestId, String producerName, boolean encrypted, Map metadata, SchemaInfo schemaInfo, long epoch, boolean userProvidedProducerName, From 0b9286341383864aea5e66b4d437c765b7377917 Mon Sep 17 00:00:00 2001 From: Denovo1998 Date: Wed, 1 Feb 2023 17:31:52 +0800 Subject: [PATCH 20/20] fix SchemaType getSchemaType(Schema.Type type) like Schema.Type getSchemaType(SchemaType type), and delete the judgment of schema as AutoConsume in ServerCnx. --- .../main/java/org/apache/pulsar/broker/service/ServerCnx.java | 3 +-- .../main/java/org/apache/pulsar/common/protocol/Commands.java | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 84470193def84..9778c3c0635b3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1241,8 +1241,7 @@ private SchemaData getSchema(Schema protocolSchema) { .isDeleted(false) .timestamp(System.currentTimeMillis()) .user(Strings.nullToEmpty(originalPrincipal)) - .type(protocolSchema.getType() == Schema.Type.AutoConsume - ? SchemaType.AUTO_CONSUME : Commands.getSchemaType(protocolSchema.getType())) + .type(Commands.getSchemaType(protocolSchema.getType())) .props(protocolSchema.getPropertiesList().stream().collect( Collectors.toMap( KeyValue::getKey, diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index ad49757efb209..8a5684cf676b0 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -781,7 +781,9 @@ private static Schema.Type getSchemaType(SchemaType type) { } public static SchemaType getSchemaType(Schema.Type type) { - if (type.getValue() < 0) { + if (type == Schema.Type.AutoConsume) { + return SchemaType.AUTO_CONSUME; + } else if (type.getValue() < 0) { // this is unexpected return SchemaType.NONE; } else {