From 304d15a74a8f43031c0910b649ba7ff54fcfa103 Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 16 Nov 2021 00:21:01 +0800 Subject: [PATCH 01/25] PIP 110: Support Topic metadata (#12629). --- .../mledger/ManagedLedgerConfig.java | 11 +++ .../mledger/impl/ManagedLedgerImpl.java | 3 +- .../bookkeeper/mledger/impl/MetaStore.java | 20 +++++- .../mledger/impl/MetaStoreImpl.java | 16 ++++- .../mledger/impl/ManagedLedgerTest.java | 30 ++++++++ .../pulsar/broker/admin/AdminResource.java | 18 +++-- .../admin/impl/PersistentTopicsBase.java | 12 ++-- .../broker/admin/v1/PersistentTopics.java | 58 ++++++++++++++- .../broker/admin/v2/PersistentTopics.java | 72 ++++++++++++++++++- .../pulsar/broker/service/BrokerService.java | 35 +++++---- .../broker/admin/PersistentTopicsTest.java | 30 +++++++- .../broker/service/BrokerServiceTest.java | 2 +- .../apache/pulsar/client/admin/Topics.java | 70 ++++++++++++++++-- .../partition/PartitionedTopicMetadata.java | 10 +++ .../client/admin/internal/TopicsImpl.java | 27 +++---- .../apache/pulsar/admin/cli/CmdTopics.java | 22 +++++- 16 files changed, 386 insertions(+), 50 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java index 2f4e098d677e5..6a898dd53aa93 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java @@ -77,6 +77,7 @@ public class ManagedLedgerConfig { private int newEntriesCheckDelayInMillis = 10; private Clock clock = Clock.systemUTC(); private ManagedLedgerInterceptor managedLedgerInterceptor; + private Map topicMetadata; public boolean isCreateIfMissing() { return createIfMissing; @@ -623,6 +624,16 @@ public void setBookKeeperEnsemblePlacementPolicyProperties( this.bookKeeperEnsemblePlacementPolicyProperties = bookKeeperEnsemblePlacementPolicyProperties; } + + public Map getTopicMetadata() { + return topicMetadata; + } + + + public void setTopicMetadata(Map topicMetadata) { + this.topicMetadata = topicMetadata; + } + public boolean isDeletionAtBatchIndexLevelEnabled() { return deletionAtBatchIndexLevelEnabled; } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 629d10d48d64a..3dec904c9a561 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -325,7 +325,8 @@ synchronized void initialize(final ManagedLedgerInitializeLedgerCallback callbac log.info("Opening managed ledger {}", name); // Fetch the list of existing ledgers in the managed ledger - store.getManagedLedgerInfo(name, config.isCreateIfMissing(), new MetaStoreCallback() { + store.getManagedLedgerInfo(name, config.isCreateIfMissing(), config.getTopicMetadata(), + new MetaStoreCallback() { @Override public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) { ledgersStat = stat; diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStore.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStore.java index 9f1563be61863..4991e453cc4e2 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStore.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStore.java @@ -19,6 +19,7 @@ package org.apache.bookkeeper.mledger.impl; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException; @@ -52,7 +53,24 @@ interface MetaStoreCallback { * whether the managed ledger metadata should be created if it doesn't exist already * @throws MetaStoreException */ - void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, MetaStoreCallback callback); + default void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, + MetaStoreCallback callback) { + getManagedLedgerInfo(ledgerName, createIfMissing, null, callback); + } + + /** + * Get the metadata used by the ManagedLedger. + * + * @param ledgerName + * the name of the ManagedLedger + * @param createIfMissing + * whether the managed ledger metadata should be created if it doesn't exist already + * @param topicMetadata + * topic metadata + * @throws MetaStoreException + */ + void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map topicMetadata, + MetaStoreCallback callback); /** * diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java index 5ad62b228bce0..4b9eb94681004 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -84,7 +85,7 @@ public MetaStoreImpl(MetadataStore store, OrderedExecutor executor, String compr } @Override - public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, + public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map topicMetadata, MetaStoreCallback callback) { // Try to get the content or create an empty node String path = PREFIX + ledgerName; @@ -106,8 +107,17 @@ public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, store.put(path, new byte[0], Optional.of(-1L)) .thenAccept(stat -> { - ManagedLedgerInfo info = ManagedLedgerInfo.getDefaultInstance(); - callback.operationComplete(info, stat); + ManagedLedgerInfo.Builder ledgerBuilder = ManagedLedgerInfo.newBuilder(); + if (topicMetadata != null) { + topicMetadata.forEach((k, v) -> { + ledgerBuilder.addProperties( + MLDataFormats.KeyValue.newBuilder() + .setKey(k) + .setValue(v) + .build()); + }); + } + callback.operationComplete(ledgerBuilder.build(), stat); }).exceptionally(ex -> { callback.operationFailed(getException(ex)); return null; diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index d837651f1fac5..cde27284cb6c6 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -1632,6 +1632,36 @@ public void cursorReadsWithDiscardedEmptyLedgers() throws Exception { assertEquals(c1.readEntries(1).size(), 0); } + @Test + public void testSetTopicMetadata() throws Exception { + Map properties = new HashMap<>(); + properties.put("key1", "value1"); + properties.put("key2", "value2"); + final MetaStore store = factory.getMetaStore(); + final CountDownLatch latch = new CountDownLatch(1); + final ManagedLedgerInfo[] storedMLInfo = new ManagedLedgerInfo[1]; + store.getManagedLedgerInfo("my_test_ledger", true, properties, new MetaStoreCallback() { + @Override + public void operationComplete(ManagedLedgerInfo result, Stat version) { + storedMLInfo[0] = result; + latch.countDown(); + } + + @Override + public void operationFailed(MetaStoreException e) { + latch.countDown(); + fail("Should have failed here"); + } + }); + latch.await(); + + assertEquals(storedMLInfo[0].getPropertiesCount(), 2); + assertEquals(storedMLInfo[0].getPropertiesList().get(0).getKey(), "key1"); + assertEquals(storedMLInfo[0].getPropertiesList().get(0).getValue(), "value1"); + assertEquals(storedMLInfo[0].getPropertiesList().get(1).getKey(), "key2"); + assertEquals(storedMLInfo[0].getPropertiesList().get(1).getValue(), "value2"); + } + @Test public void cursorReadsWithDiscardedEmptyLedgersStillListed() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index f3a94d222fdca..86574aa22d73c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -22,6 +22,7 @@ import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -574,6 +575,11 @@ protected List getTopicPartitionList(TopicDomain topicDomain) { protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int numPartitions, boolean createLocalTopicOnly) { + internalCreatePartitionedTopic(asyncResponse, numPartitions, createLocalTopicOnly, null); + } + + protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int numPartitions, + boolean createLocalTopicOnly, Map topicMetadata) { Integer maxTopicsPerNamespace = null; try { @@ -640,7 +646,7 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n return; } - provisionPartitionedTopicPath(asyncResponse, numPartitions, createLocalTopicOnly) + provisionPartitionedTopicPath(asyncResponse, numPartitions, createLocalTopicOnly, topicMetadata) .thenCompose(ignored -> tryCreatePartitionsAsync(numPartitions)) .whenComplete((ignored, ex) -> { if (ex != null) { @@ -679,7 +685,7 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n ((TopicsImpl) pulsar().getBrokerService() .getClusterPulsarAdmin(cluster, clusterDataOp).topics()) .createPartitionedTopicAsync( - topicName.getPartitionedTopicName(), numPartitions, true); + topicName.getPartitionedTopicName(), numPartitions, true, null); }) .exceptionally(throwable -> { log.error("Failed to create partition topic in cluster {}.", cluster, throwable); @@ -718,13 +724,13 @@ protected CompletableFuture checkTopicExistsAsync(TopicName topicName) }); } - private CompletableFuture provisionPartitionedTopicPath(AsyncResponse asyncResponse, - int numPartitions, - boolean createLocalTopicOnly) { + private CompletableFuture provisionPartitionedTopicPath(AsyncResponse asyncResponse, int numPartitions, + boolean createLocalTopicOnly, + Map topicMetadata) { CompletableFuture future = new CompletableFuture<>(); namespaceResources() .getPartitionedTopicResources() - .createPartitionedTopicAsync(topicName, new PartitionedTopicMetadata(numPartitions)) + .createPartitionedTopicAsync(topicName, new PartitionedTopicMetadata(numPartitions, topicMetadata)) .whenComplete((ignored, ex) -> { if (ex != null) { if (ex instanceof AlreadyExistsException) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index fc33ad4ea5673..30e8f4f4dfa88 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -357,7 +357,7 @@ protected void internalRevokePermissionsOnTopic(String role) { revokePermissions(topicName.toString(), role); } - protected void internalCreateNonPartitionedTopic(boolean authoritative) { + protected void internalCreateNonPartitionedTopic(boolean authoritative, Map topicMetadata) { validateNonPartitionTopicName(topicName.getLocalName()); if (topicName.isGlobal()) { validateGlobalNamespaceOwnership(namespaceName); @@ -378,7 +378,7 @@ protected void internalCreateNonPartitionedTopic(boolean authoritative) { throw new RestException(Status.CONFLICT, "This topic already exists"); } - Topic createdTopic = getOrCreateTopic(topicName); + Topic createdTopic = getOrCreateTopic(topicName, topicMetadata); log.info("[{}] Successfully created non-partitioned topic {}", clientAppId(), createdTopic); } catch (Exception e) { if (e instanceof RestException) { @@ -3750,8 +3750,12 @@ private CompletableFuture topicNotFoundReasonAsync(TopicName topicName) { } private Topic getOrCreateTopic(TopicName topicName) { - return pulsar().getBrokerService().getTopic( - topicName.toString(), true).thenApply(Optional::get).join(); + return getOrCreateTopic(topicName, null); + } + + private Topic getOrCreateTopic(TopicName topicName, Map topicMetadata) { + return pulsar().getBrokerService().getTopic(topicName.toString(), true, topicMetadata) + .thenApply(Optional::get).join(); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index 2917482857363..d12cb64700f88 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -171,6 +171,33 @@ public void createPartitionedTopic( } } + @PUT + @Path("/{property}/{cluster}/{namespace}/{topic}/partitions/topicMetadata") + @ApiOperation(hidden = true, value = "Create a partitioned topic.", + notes = "It needs to be called before creating a producer on a partitioned topic.") + @ApiResponses(value = { + @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 406, message = "The number of partitions should be " + + "more than 0 and less than or equal to maxNumPartitionsPerPartitionedTopic"), + @ApiResponse(code = 409, message = "Partitioned topic already exist")}) + public void createPartitionedTopic( + @Suspended final AsyncResponse asyncResponse, + @PathParam("property") String property, + @PathParam("cluster") String cluster, + @PathParam("namespace") String namespace, + @PathParam("topic") @Encoded String encodedTopic, + @QueryParam("createLocalTopicOnly") @DefaultValue("false") boolean createLocalTopicOnly, + PartitionedTopicMetadata metadata) { + try { + validateTopicName(property, cluster, namespace, encodedTopic); + internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly, metadata.topicMetadata); + } catch (Exception e) { + log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); + resumeAsyncResponseExceptionally(asyncResponse, e); + } + } + @PUT @Path("/{tenant}/{cluster}/{namespace}/{topic}") @ApiOperation(value = "Create a non-partitioned topic.", @@ -196,10 +223,39 @@ public void createNonPartitionedTopic( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Is authentication required to perform this operation") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + createNonPartitionedTopic(tenant, cluster, namespace, encodedTopic, authoritative, null); + } + + @PUT + @Path("/{tenant}/{cluster}/{namespace}/{topic}/topicMetadata") + @ApiOperation(value = "Create a non-partitioned topic.", + notes = "This is the only REST endpoint from which non-partitioned topics could be created.") + @ApiResponses(value = { + @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), + @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 409, message = "Partitioned topic already exist"), + @ApiResponse(code = 412, + message = "Failed Reason : Name is invalid or Namespace does not have any clusters configured"), + @ApiResponse(code = 500, message = "Internal server error"), + @ApiResponse(code = 503, message = "Failed to validate global cluster configuration") + }) + public void createNonPartitionedTopic( + @ApiParam(value = "Specify the tenant", required = true) + @PathParam("tenant") String tenant, + @ApiParam(value = "Specify the cluster", required = true) + @PathParam("cluster") String cluster, + @ApiParam(value = "Specify the namespace", required = true) + @PathParam("namespace") String namespace, + @ApiParam(value = "Specify topic name", required = true) + @PathParam("topic") @Encoded String encodedTopic, + @ApiParam(value = "Is authentication required to perform this operation") + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @ApiParam(value = "Key value pair properties for the topic metadata") Map topicMetadata) { validateNamespaceName(tenant, cluster, namespace); validateTopicName(tenant, cluster, namespace, encodedTopic); validateGlobalNamespaceOwnership(); - internalCreateNonPartitionedTopic(authoritative); + internalCreateNonPartitionedTopic(authoritative, topicMetadata); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index bf6c64712c851..015e4dd4b61b0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -235,7 +235,47 @@ public void createPartitionedTopic( validatePartitionedTopicName(tenant, namespace, encodedTopic); validateTopicPolicyOperation(topicName, PolicyName.PARTITION, PolicyOperation.WRITE); validateCreateTopic(topicName); - internalCreatePartitionedTopic(asyncResponse, numPartitions, createLocalTopicOnly); + internalCreatePartitionedTopic(asyncResponse, numPartitions, createLocalTopicOnly, null); + } catch (Exception e) { + log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); + resumeAsyncResponseExceptionally(asyncResponse, e); + } + } + + @PUT + @Path("/{tenant}/{namespace}/{topic}/partitions/topicMetadata") + @ApiOperation(value = "Create a partitioned topic.", + notes = "It needs to be called before creating a producer on a partitioned topic.") + @ApiResponses(value = { + @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), + @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant does not exist"), + @ApiResponse(code = 406, message = "The number of partitions should be more than 0 and" + + " less than or equal to maxNumPartitionsPerPartitionedTopic"), + @ApiResponse(code = 409, message = "Partitioned topic already exist"), + @ApiResponse(code = 412, + message = "Failed Reason : Name is invalid or Namespace does not have any clusters configured"), + @ApiResponse(code = 500, message = "Internal server error"), + @ApiResponse(code = 503, message = "Failed to validate global cluster configuration") + }) + public void createPartitionedTopic( + @Suspended final AsyncResponse asyncResponse, + @ApiParam(value = "Specify the tenant", required = true) + @PathParam("tenant") String tenant, + @ApiParam(value = "Specify the namespace", required = true) + @PathParam("namespace") String namespace, + @ApiParam(value = "Specify topic name", required = true) + @PathParam("topic") @Encoded String encodedTopic, + @QueryParam("createLocalTopicOnly") @DefaultValue("false") boolean createLocalTopicOnly, + PartitionedTopicMetadata metadata) { + try { + validateNamespaceName(tenant, namespace); + validateGlobalNamespaceOwnership(); + validatePartitionedTopicName(tenant, namespace, encodedTopic); + validateTopicPolicyOperation(topicName, PolicyName.PARTITION, PolicyOperation.WRITE); + validateCreateTopic(topicName); + internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly, metadata.topicMetadata); } catch (Exception e) { log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); resumeAsyncResponseExceptionally(asyncResponse, e); @@ -265,11 +305,39 @@ public void createNonPartitionedTopic( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Is authentication required to perform this operation") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + createNonPartitionedTopic(tenant, namespace, encodedTopic, authoritative, null); + } + + @PUT + @Path("/{tenant}/{namespace}/{topic}/topicMetadata") + @ApiOperation(value = "Create a non-partitioned topic.", + notes = "This is the only REST endpoint from which non-partitioned topics could be created.") + @ApiResponses(value = { + @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), + @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 409, message = "Partitioned topic already exist"), + @ApiResponse(code = 412, + message = "Failed Reason : Name is invalid or Namespace does not have any clusters configured"), + @ApiResponse(code = 500, message = "Internal server error"), + @ApiResponse(code = 503, message = "Failed to validate global cluster configuration") + }) + public void createNonPartitionedTopic( + @ApiParam(value = "Specify the tenant", required = true) + @PathParam("tenant") String tenant, + @ApiParam(value = "Specify the namespace", required = true) + @PathParam("namespace") String namespace, + @ApiParam(value = "Specify topic name", required = true) + @PathParam("topic") @Encoded String encodedTopic, + @ApiParam(value = "Is authentication required to perform this operation") + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @ApiParam(value = "Key value pair properties for the topic metadata") + Map topicMetadata) { validateNamespaceName(tenant, namespace); validateGlobalNamespaceOwnership(); validateTopicName(tenant, namespace, encodedTopic); validateCreateTopic(topicName); - internalCreateNonPartitionedTopic(authoritative); + internalCreateNonPartitionedTopic(authoritative, topicMetadata); } @GET diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index ba3a74d4e6cb0..ac51176ad6f4e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -88,8 +88,8 @@ import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.bookkeeper.mledger.util.Futures; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.lang3.tuple.ImmutableTriple; +import org.apache.commons.lang3.tuple.Triple; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -211,7 +211,8 @@ public class BrokerService implements Closeable { prepareDynamicConfigurationMap(); private final ConcurrentOpenHashMap> configRegisteredListeners; - private final ConcurrentLinkedQueue>>> pendingTopicLoadingQueue; + private final ConcurrentLinkedQueue>, Map>> + pendingTopicLoadingQueue; private AuthorizationService authorizationService = null; private final ScheduledExecutorService statsUpdater; @@ -858,6 +859,11 @@ public CompletableFuture getOrCreateTopic(final String topic) { } public CompletableFuture> getTopic(final String topic, boolean createIfMissing) { + return getTopic(topic, createIfMissing, null); + } + + public CompletableFuture> getTopic(final String topic, boolean createIfMissing, + Map topicMetadata) { try { CompletableFuture> topicFuture = topics.get(topic); if (topicFuture != null) { @@ -874,7 +880,7 @@ public CompletableFuture> getTopic(final String topic, boolean c return topicFuture.thenCompose(value -> { if (!value.isPresent()) { // retry and create topic - return getTopic(topic, createIfMissing); + return getTopic(topic, createIfMissing, topicMetadata); } else { // in-progress future completed successfully return CompletableFuture.completedFuture(value); @@ -889,7 +895,7 @@ public CompletableFuture> getTopic(final String topic, boolean c final boolean isPersistentTopic = TopicName.get(topic).getDomain().equals(TopicDomain.persistent); if (isPersistentTopic) { return topics.computeIfAbsent(topic, (topicName) -> { - return this.loadOrCreatePersistentTopic(topicName, createIfMissing); + return this.loadOrCreatePersistentTopic(topicName, createIfMissing, topicMetadata); }); } else { return topics.computeIfAbsent(topic, (name) -> { @@ -1232,7 +1238,7 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c * @throws RuntimeException */ protected CompletableFuture> loadOrCreatePersistentTopic(final String topic, - boolean createIfMissing) throws RuntimeException { + boolean createIfMissing, Map topicMetadata) throws RuntimeException { final CompletableFuture> topicFuture = FutureUtil.createFutureWithTimeout( Duration.ofSeconds(pulsar.getConfiguration().getTopicLoadTimeoutSeconds()), executor(), () -> FAILED_TO_LOAD_TOPIC_TIMEOUT_EXCEPTION); @@ -1250,7 +1256,7 @@ protected CompletableFuture> loadOrCreatePersistentTopic(final S final Semaphore topicLoadSemaphore = topicLoadRequestSemaphore.get(); if (topicLoadSemaphore.tryAcquire()) { - createPersistentTopic(topic, createIfMissing, topicFuture); + createPersistentTopic(topic, createIfMissing, topicFuture, topicMetadata); topicFuture.handle((persistentTopic, ex) -> { // release permit and process pending topic topicLoadSemaphore.release(); @@ -1258,7 +1264,7 @@ protected CompletableFuture> loadOrCreatePersistentTopic(final S return null; }); } else { - pendingTopicLoadingQueue.add(new ImmutablePair<>(topic, topicFuture)); + pendingTopicLoadingQueue.add(new ImmutableTriple<>(topic, topicFuture, topicMetadata)); if (log.isDebugEnabled()) { log.debug("topic-loading for {} added into pending queue", topic); } @@ -1272,7 +1278,8 @@ protected CompletableFuture> loadOrCreatePersistentTopic(final S } private void createPersistentTopic(final String topic, boolean createIfMissing, - CompletableFuture> topicFuture) { + CompletableFuture> topicFuture, + Map topicMetadata) { final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); TopicName topicName = TopicName.get(topic); @@ -1313,6 +1320,7 @@ private void createPersistentTopic(final String topic, boolean createIfMissing, } managedLedgerConfig.setCreateIfMissing(createIfMissing); + managedLedgerConfig.setTopicMetadata(topicMetadata); // Once we have the configuration, we can proceed with the async open operation managedLedgerFactory.asyncOpen(topicName.getPersistenceNamingEncoding(), managedLedgerConfig, @@ -2330,17 +2338,18 @@ private ConcurrentOpenHashMap getRuntimeConfigurationMap() { * permit if it was successful to acquire it. */ private void createPendingLoadTopic() { - Pair>> pendingTopic = pendingTopicLoadingQueue.poll(); + Triple>, Map> pendingTopic = + pendingTopicLoadingQueue.poll(); if (pendingTopic == null) { return; } final String topic = pendingTopic.getLeft(); checkTopicNsOwnership(topic).thenRun(() -> { - CompletableFuture> pendingFuture = pendingTopic.getRight(); + CompletableFuture> pendingFuture = pendingTopic.getMiddle(); final Semaphore topicLoadSemaphore = topicLoadRequestSemaphore.get(); final boolean acquiredPermit = topicLoadSemaphore.tryAcquire(); - createPersistentTopic(topic, true, pendingFuture); + createPersistentTopic(topic, true, pendingFuture, pendingTopic.getRight()); pendingFuture.handle((persistentTopic, ex) -> { // release permit and process next pending topic if (acquiredPermit) { @@ -2351,7 +2360,7 @@ private void createPendingLoadTopic() { }); }).exceptionally(e -> { log.error("Failed to create pending topic {}", topic, e); - pendingTopic.getRight() + pendingTopic.getMiddle() .completeExceptionally((e instanceof RuntimeException && e.getCause() != null) ? e.getCause() : e); // schedule to process next pending topic inactivityMonitor.schedule(this::createPendingLoadTopic, 100, TimeUnit.MILLISECONDS); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 3836e4f73cd0c..bcf0a65cd4f32 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -88,6 +88,7 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import org.testng.collections.Maps; @Slf4j @Test(groups = "broker") @@ -404,6 +405,33 @@ public void testCreateNonPartitionedTopic() { PartitionedTopicMetadata metadata = persistentTopics.getPartitionedMetadata( testTenant, testNamespace, topicName, true, true); Assert.assertEquals(metadata.partitions, 0); + final String topicName2 = "standard-topic-partition-b"; + Map topicMetadata = Maps.newHashMap(); + topicMetadata.put("key1", "value1"); + persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName2, true, topicMetadata); + PartitionedTopicMetadata pMetadata2 = persistentTopics.getPartitionedMetadata( + testTenant, testNamespace, topicName2, true, false); + Assert.assertNull(pMetadata2.topicMetadata); + } + + @Test + public void testCreatePartitionedTopic() { + AsyncResponse response = mock(AsyncResponse.class); + final String topicName = "standard-partitioned-topic-a"; + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2, true); + PartitionedTopicMetadata pMetadata = persistentTopics.getPartitionedMetadata( + testTenant, testNamespace, topicName, true, false); + Assert.assertNull(pMetadata.topicMetadata); + + final String topicName2 = "standard-partitioned-topic-b"; + Map topicMetadata = Maps.newHashMap(); + topicMetadata.put("key1", "value1"); + PartitionedTopicMetadata metadata = new PartitionedTopicMetadata(2, topicMetadata); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName2, true, metadata); + PartitionedTopicMetadata pMetadata2 = persistentTopics.getPartitionedMetadata( + testTenant, testNamespace, topicName2, true, false); + Assert.assertEquals(pMetadata2.topicMetadata.size(), 1); + Assert.assertEquals(pMetadata2.topicMetadata, topicMetadata); } @Test(expectedExceptions = RestException.class) @@ -671,7 +699,7 @@ public void testPeekWithSubscriptionNameNotExist() throws Exception { admin.namespaces().setRetention("tenant-xyz/ns-abc", retention); final String topic = "persistent://tenant-xyz/ns-abc/topic-testPeekWithSubscriptionNameNotExist"; final String subscriptionName = "sub"; - ((TopicsImpl) admin.topics()).createPartitionedTopicAsync(topic, 3, true).get(); + ((TopicsImpl) admin.topics()).createPartitionedTopicAsync(topic, 3, true, null).get(); final String partitionedTopic = topic + "-partition-0"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index af8ddb583a58f..48a7ccd8a256c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -1051,7 +1051,7 @@ public void testTopicLoadingOnDisableNamespaceBundle() throws Exception { // try to create topic which should fail as bundle is disable CompletableFuture> futureResult = pulsar.getBrokerService() - .loadOrCreatePersistentTopic(topicName, true); + .loadOrCreatePersistentTopic(topicName, true, null); try { futureResult.get(); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java index 03bca5cfcb2ef..927950a72285e 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java @@ -344,7 +344,43 @@ List getListInBundle(String namespace, String bundleRange) * Number of partitions to create of the topic * @throws PulsarAdminException */ - void createPartitionedTopic(String topic, int numPartitions) throws PulsarAdminException; + default void createPartitionedTopic(String topic, int numPartitions) throws PulsarAdminException { + createPartitionedTopic(topic, numPartitions, null); + } + + /** + * Create a partitioned topic. + *

+ * Create a partitioned topic. It needs to be called before creating a producer for a partitioned topic. + *

+ * + * @param topic + * Topic name + * @param numPartitions + * Number of partitions to create of the topic + * @param metadata + * Metadata of the topic + * @throws PulsarAdminException + */ + void createPartitionedTopic(String topic, int numPartitions, Map metadata) + throws PulsarAdminException; + + /** + * Create a partitioned topic asynchronously. + *

+ * Create a partitioned topic asynchronously. It needs to be called before creating a producer for a partitioned + * topic. + *

+ * + * @param topic + * Topic name + * @param numPartitions + * Number of partitions to create of the topic + * @return a future that can be used to track when the partitioned topic is created + */ + default CompletableFuture createPartitionedTopicAsync(String topic, int numPartitions) { + return createPartitionedTopicAsync(topic, numPartitions, null); + } /** * Create a partitioned topic asynchronously. @@ -357,9 +393,11 @@ List getListInBundle(String namespace, String bundleRange) * Topic name * @param numPartitions * Number of partitions to create of the topic + * @param metadata + * Metadata of the topic * @return a future that can be used to track when the partitioned topic is created */ - CompletableFuture createPartitionedTopicAsync(String topic, int numPartitions); + CompletableFuture createPartitionedTopicAsync(String topic, int numPartitions, Map metadata); /** * Create a non-partitioned topic. @@ -370,14 +408,38 @@ List getListInBundle(String namespace, String bundleRange) * @param topic Topic name * @throws PulsarAdminException */ - void createNonPartitionedTopic(String topic) throws PulsarAdminException; + default void createNonPartitionedTopic(String topic) throws PulsarAdminException { + createNonPartitionedTopic(topic, null); + } + + /** + * Create a non-partitioned topic. + *

+ * Create a non-partitioned topic. + *

+ * + * @param topic Topic name + * @param metadata Topic metadata + * @throws PulsarAdminException + */ + void createNonPartitionedTopic(String topic, Map metadata) throws PulsarAdminException; + + /** + * Create a non-partitioned topic asynchronously. + * + * @param topic Topic name + */ + default CompletableFuture createNonPartitionedTopicAsync(String topic) { + return createNonPartitionedTopicAsync(topic, null); + } /** * Create a non-partitioned topic asynchronously. * * @param topic Topic name + * @param metadata Topic metadata */ - CompletableFuture createNonPartitionedTopicAsync(String topic); + CompletableFuture createNonPartitionedTopicAsync(String topic, Map metadata); /** * Create missed partitions for partitioned topic. diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/partition/PartitionedTopicMetadata.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/partition/PartitionedTopicMetadata.java index 024ca0a32cef0..1c32ff5731376 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/partition/PartitionedTopicMetadata.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/partition/PartitionedTopicMetadata.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.common.partition; +import java.util.Map; + /** * Metadata of a partitioned topic. */ @@ -26,6 +28,9 @@ public class PartitionedTopicMetadata { /* Number of partitions for the topic */ public int partitions; + /* Topic metadata */ + public Map topicMetadata; + public PartitionedTopicMetadata() { this.partitions = 0; } @@ -34,6 +39,11 @@ public PartitionedTopicMetadata(int partitions) { this.partitions = partitions; } + public PartitionedTopicMetadata(int partitions, Map topicMetadata) { + this.partitions = partitions; + this.topicMetadata = topicMetadata; + } + /** * A topic with '0' partitions is treated like non-partitioned topic. */ diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java index 4758b9aef4744..9d7a765b417de 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java @@ -368,9 +368,10 @@ public CompletableFuture revokePermissionsAsync(String topic, String role) } @Override - public void createPartitionedTopic(String topic, int numPartitions) throws PulsarAdminException { + public void createPartitionedTopic(String topic, int numPartitions, Map metadata) + throws PulsarAdminException { try { - createPartitionedTopicAsync(topic, numPartitions).get(this.readTimeoutMs, TimeUnit.MILLISECONDS); + createPartitionedTopicAsync(topic, numPartitions, metadata).get(this.readTimeoutMs, TimeUnit.MILLISECONDS); } catch (ExecutionException e) { throw (PulsarAdminException) e.getCause(); } catch (InterruptedException e) { @@ -382,9 +383,9 @@ public void createPartitionedTopic(String topic, int numPartitions) throws Pulsa } @Override - public void createNonPartitionedTopic(String topic) throws PulsarAdminException { + public void createNonPartitionedTopic(String topic, Map metadata) throws PulsarAdminException { try { - createNonPartitionedTopicAsync(topic).get(this.readTimeoutMs, TimeUnit.MILLISECONDS); + createNonPartitionedTopicAsync(topic, metadata).get(this.readTimeoutMs, TimeUnit.MILLISECONDS); } catch (ExecutionException e) { throw (PulsarAdminException) e.getCause(); } catch (InterruptedException e) { @@ -410,24 +411,26 @@ public void createMissedPartitions(String topic) throws PulsarAdminException { } @Override - public CompletableFuture createNonPartitionedTopicAsync(String topic){ + public CompletableFuture createNonPartitionedTopicAsync(String topic, Map metadata){ TopicName tn = validateTopic(topic); - WebTarget path = topicPath(tn); - return asyncPutRequest(path, Entity.entity("", MediaType.APPLICATION_JSON)); + WebTarget path = topicPath(tn, "topicMetadata"); + return asyncPutRequest(path, Entity.entity(metadata, MediaType.APPLICATION_JSON)); } @Override - public CompletableFuture createPartitionedTopicAsync(String topic, int numPartitions) { - return createPartitionedTopicAsync(topic, numPartitions, false); + public CompletableFuture createPartitionedTopicAsync(String topic, int numPartitions, + Map metadata) { + return createPartitionedTopicAsync(topic, numPartitions, false, metadata); } public CompletableFuture createPartitionedTopicAsync( - String topic, int numPartitions, boolean createLocalTopicOnly) { + String topic, int numPartitions, boolean createLocalTopicOnly, Map topicMetadata) { checkArgument(numPartitions > 0, "Number of partitions should be more than 0"); TopicName tn = validateTopic(topic); - WebTarget path = topicPath(tn, "partitions") + WebTarget path = topicPath(tn, "partitions/topicMetadata") .queryParam("createLocalTopicOnly", Boolean.toString(createLocalTopicOnly)); - return asyncPutRequest(path, Entity.entity(numPartitions, MediaType.APPLICATION_JSON)); + PartitionedTopicMetadata metadata = new PartitionedTopicMetadata(numPartitions, topicMetadata); + return asyncPutRequest(path, Entity.entity(metadata, MediaType.APPLICATION_JSON)); } @Override diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index ece4d3ea256f5..42a2f6d77b590 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -34,9 +34,11 @@ import io.netty.buffer.Unpooled; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -383,10 +385,28 @@ private class CreatePartitionedCmd extends CliCommand { "--partitions" }, description = "Number of partitions for the topic", required = true) private int numPartitions; + @Parameter(names = {"--topic-metadata", "-tm"}, description = "key value pair properties(a=a,b=b,c=c)", + required = true) + private java.util.List topicMetadata; + @Override void run() throws Exception { String topic = validateTopicName(params); - getTopics().createPartitionedTopic(topic, numPartitions); + Map map = new HashMap<>(); + for (String property : topicMetadata) { + if (!property.contains("=")) { + throw new ParameterException(String.format("Invalid key value pair '%s', " + + "valid format like 'a=a,b=b,c=c'.", property)); + } else { + String[] keyValue = property.split("="); + if (keyValue.length != 2) { + throw new ParameterException(String.format("Invalid key value pair '%s', " + + "valid format like 'a=a,b=b,c=c'.", property)); + } + map.put(keyValue[0], keyValue[1]); + } + } + getTopics().createPartitionedTopic(topic, numPartitions, map); } } From d2234c42c2a05771c7c7cd6445470dcf47f2c671 Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 16 Nov 2021 01:16:25 +0800 Subject: [PATCH 02/25] fix checkstyle --- .../org/apache/pulsar/broker/admin/v1/PersistentTopics.java | 3 ++- .../org/apache/pulsar/broker/admin/v2/PersistentTopics.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index d12cb64700f88..d58f810be9b56 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -191,7 +191,8 @@ public void createPartitionedTopic( PartitionedTopicMetadata metadata) { try { validateTopicName(property, cluster, namespace, encodedTopic); - internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly, metadata.topicMetadata); + internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly, + metadata.topicMetadata); } catch (Exception e) { log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); resumeAsyncResponseExceptionally(asyncResponse, e); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 015e4dd4b61b0..d234c0ab817b3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -275,7 +275,8 @@ public void createPartitionedTopic( validatePartitionedTopicName(tenant, namespace, encodedTopic); validateTopicPolicyOperation(topicName, PolicyName.PARTITION, PolicyOperation.WRITE); validateCreateTopic(topicName); - internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly, metadata.topicMetadata); + internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly, + metadata.topicMetadata); } catch (Exception e) { log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); resumeAsyncResponseExceptionally(asyncResponse, e); From 58eecf19becec0e2f48d29780705df57d6a4c501 Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 16 Nov 2021 11:49:10 +0800 Subject: [PATCH 03/25] fix path --- .../org/apache/pulsar/client/admin/internal/TopicsImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java index 9d7a765b417de..25cbe4d3a92ca 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java @@ -427,7 +427,7 @@ public CompletableFuture createPartitionedTopicAsync( String topic, int numPartitions, boolean createLocalTopicOnly, Map topicMetadata) { checkArgument(numPartitions > 0, "Number of partitions should be more than 0"); TopicName tn = validateTopic(topic); - WebTarget path = topicPath(tn, "partitions/topicMetadata") + WebTarget path = topicPath(tn, "partitions", "topicMetadata") .queryParam("createLocalTopicOnly", Boolean.toString(createLocalTopicOnly)); PartitionedTopicMetadata metadata = new PartitionedTopicMetadata(numPartitions, topicMetadata); return asyncPutRequest(path, Entity.entity(metadata, MediaType.APPLICATION_JSON)); From 18efd4dd20ce021d82f8222081fb0cfa2b595269 Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 16 Nov 2021 12:03:21 +0800 Subject: [PATCH 04/25] remove required=true --- .../apache/pulsar/admin/cli/CmdTopics.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index 42a2f6d77b590..7168bce96e297 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -385,8 +385,7 @@ private class CreatePartitionedCmd extends CliCommand { "--partitions" }, description = "Number of partitions for the topic", required = true) private int numPartitions; - @Parameter(names = {"--topic-metadata", "-tm"}, description = "key value pair properties(a=a,b=b,c=c)", - required = true) + @Parameter(names = {"--topic-metadata", "-tm"}, description = "key value pair properties(a=a,b=b,c=c)") private java.util.List topicMetadata; @Override @@ -431,10 +430,27 @@ private class CreateNonPartitionedCmd extends CliCommand { @Parameter(description = "persistent://tenant/namespace/topic", required = true) private java.util.List params; + @Parameter(names = {"--topic-metadata", "-tm"}, description = "key value pair properties(a=a,b=b,c=c)") + private java.util.List topicMetadata; + @Override void run() throws Exception { String topic = validateTopicName(params); - getTopics().createNonPartitionedTopic(topic); + Map map = new HashMap<>(); + for (String property : topicMetadata) { + if (!property.contains("=")) { + throw new ParameterException(String.format("Invalid key value pair '%s', " + + "valid format like 'a=a,b=b,c=c'.", property)); + } else { + String[] keyValue = property.split("="); + if (keyValue.length != 2) { + throw new ParameterException(String.format("Invalid key value pair '%s', " + + "valid format like 'a=a,b=b,c=c'.", property)); + } + map.put(keyValue[0], keyValue[1]); + } + } + getTopics().createNonPartitionedTopic(topic, map); } } From 978f04fe7412ba0c5f550e9a3dc02d67b42114d4 Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 16 Nov 2021 14:33:32 +0800 Subject: [PATCH 05/25] change the field name from topicMetadata to properties --- .../apache/bookkeeper/mledger/ManagedLedgerConfig.java | 10 +++++----- .../bookkeeper/mledger/impl/ManagedLedgerImpl.java | 2 +- .../org/apache/bookkeeper/mledger/impl/MetaStore.java | 6 +++--- .../apache/bookkeeper/mledger/impl/MetaStoreImpl.java | 6 +++--- .../apache/pulsar/broker/service/BrokerService.java | 3 +-- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java index 6a898dd53aa93..f33ac944c4ff4 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java @@ -77,7 +77,7 @@ public class ManagedLedgerConfig { private int newEntriesCheckDelayInMillis = 10; private Clock clock = Clock.systemUTC(); private ManagedLedgerInterceptor managedLedgerInterceptor; - private Map topicMetadata; + private Map properties; public boolean isCreateIfMissing() { return createIfMissing; @@ -625,13 +625,13 @@ public void setBookKeeperEnsemblePlacementPolicyProperties( } - public Map getTopicMetadata() { - return topicMetadata; + public Map getProperties() { + return properties; } - public void setTopicMetadata(Map topicMetadata) { - this.topicMetadata = topicMetadata; + public void setProperties(Map properties) { + this.properties = properties; } public boolean isDeletionAtBatchIndexLevelEnabled() { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 3dec904c9a561..a1b64f52b70a9 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -325,7 +325,7 @@ synchronized void initialize(final ManagedLedgerInitializeLedgerCallback callbac log.info("Opening managed ledger {}", name); // Fetch the list of existing ledgers in the managed ledger - store.getManagedLedgerInfo(name, config.isCreateIfMissing(), config.getTopicMetadata(), + store.getManagedLedgerInfo(name, config.isCreateIfMissing(), config.getProperties(), new MetaStoreCallback() { @Override public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStore.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStore.java index 4991e453cc4e2..3d1901d78cedd 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStore.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStore.java @@ -65,11 +65,11 @@ default void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, * the name of the ManagedLedger * @param createIfMissing * whether the managed ledger metadata should be created if it doesn't exist already - * @param topicMetadata - * topic metadata + * @param properties + * ledger properties * @throws MetaStoreException */ - void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map topicMetadata, + void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map properties, MetaStoreCallback callback); /** diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java index 4b9eb94681004..4d1e075c5e47c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java @@ -85,7 +85,7 @@ public MetaStoreImpl(MetadataStore store, OrderedExecutor executor, String compr } @Override - public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map topicMetadata, + public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map properties, MetaStoreCallback callback) { // Try to get the content or create an empty node String path = PREFIX + ledgerName; @@ -108,8 +108,8 @@ public void getManagedLedgerInfo(String ledgerName, boolean createIfMissing, Map store.put(path, new byte[0], Optional.of(-1L)) .thenAccept(stat -> { ManagedLedgerInfo.Builder ledgerBuilder = ManagedLedgerInfo.newBuilder(); - if (topicMetadata != null) { - topicMetadata.forEach((k, v) -> { + if (properties != null) { + properties.forEach((k, v) -> { ledgerBuilder.addProperties( MLDataFormats.KeyValue.newBuilder() .setKey(k) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index ac51176ad6f4e..61db1399d67eb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1320,7 +1320,7 @@ private void createPersistentTopic(final String topic, boolean createIfMissing, } managedLedgerConfig.setCreateIfMissing(createIfMissing); - managedLedgerConfig.setTopicMetadata(topicMetadata); + managedLedgerConfig.setProperties(topicMetadata); // Once we have the configuration, we can proceed with the async open operation managedLedgerFactory.asyncOpen(topicName.getPersistenceNamingEncoding(), managedLedgerConfig, @@ -1337,7 +1337,6 @@ public void openLedgerComplete(ManagedLedger ledger, Object ctx) { .initialize() .thenCompose(__ -> persistentTopic.checkReplication()); - CompletableFuture.allOf(preCreateSubForCompaction, replicationFuture) .thenCompose(v -> { // Also check dedup status From c62643fe7e4fb74109190024868811d397e08abc Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 16 Nov 2021 15:19:51 +0800 Subject: [PATCH 06/25] change the field name from topicMetadata to properties --- .../pulsar/broker/admin/AdminResource.java | 8 +++--- .../admin/impl/PersistentTopicsBase.java | 8 +++--- .../broker/admin/v1/PersistentTopics.java | 6 ++--- .../broker/admin/v2/PersistentTopics.java | 10 +++---- .../pulsar/broker/service/BrokerService.java | 16 +++++------ .../broker/admin/PersistentTopicsTest.java | 8 +++--- .../apache/pulsar/client/admin/Topics.java | 21 ++++++++------- .../partition/PartitionedTopicMetadata.java | 8 +++--- .../client/admin/internal/TopicsImpl.java | 27 +++++++------------ .../apache/pulsar/admin/cli/CmdTopics.java | 12 ++++----- 10 files changed, 58 insertions(+), 66 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index 86574aa22d73c..a57f2599dcf6e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -579,7 +579,7 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n } protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int numPartitions, - boolean createLocalTopicOnly, Map topicMetadata) { + boolean createLocalTopicOnly, Map properties) { Integer maxTopicsPerNamespace = null; try { @@ -646,7 +646,7 @@ protected void internalCreatePartitionedTopic(AsyncResponse asyncResponse, int n return; } - provisionPartitionedTopicPath(asyncResponse, numPartitions, createLocalTopicOnly, topicMetadata) + provisionPartitionedTopicPath(asyncResponse, numPartitions, createLocalTopicOnly, properties) .thenCompose(ignored -> tryCreatePartitionsAsync(numPartitions)) .whenComplete((ignored, ex) -> { if (ex != null) { @@ -726,11 +726,11 @@ protected CompletableFuture checkTopicExistsAsync(TopicName topicName) private CompletableFuture provisionPartitionedTopicPath(AsyncResponse asyncResponse, int numPartitions, boolean createLocalTopicOnly, - Map topicMetadata) { + Map properties) { CompletableFuture future = new CompletableFuture<>(); namespaceResources() .getPartitionedTopicResources() - .createPartitionedTopicAsync(topicName, new PartitionedTopicMetadata(numPartitions, topicMetadata)) + .createPartitionedTopicAsync(topicName, new PartitionedTopicMetadata(numPartitions, properties)) .whenComplete((ignored, ex) -> { if (ex != null) { if (ex instanceof AlreadyExistsException) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 30e8f4f4dfa88..75386deeed8cd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -357,7 +357,7 @@ protected void internalRevokePermissionsOnTopic(String role) { revokePermissions(topicName.toString(), role); } - protected void internalCreateNonPartitionedTopic(boolean authoritative, Map topicMetadata) { + protected void internalCreateNonPartitionedTopic(boolean authoritative, Map properties) { validateNonPartitionTopicName(topicName.getLocalName()); if (topicName.isGlobal()) { validateGlobalNamespaceOwnership(namespaceName); @@ -378,7 +378,7 @@ protected void internalCreateNonPartitionedTopic(boolean authoritative, Map topicMetadata) { - return pulsar().getBrokerService().getTopic(topicName.toString(), true, topicMetadata) + private Topic getOrCreateTopic(TopicName topicName, Map properties) { + return pulsar().getBrokerService().getTopic(topicName.toString(), true, properties) .thenApply(Optional::get).join(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index d58f810be9b56..95aa96a953361 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -172,7 +172,7 @@ public void createPartitionedTopic( } @PUT - @Path("/{property}/{cluster}/{namespace}/{topic}/partitions/topicMetadata") + @Path("/{property}/{cluster}/{namespace}/{topic}/partitions/properties") @ApiOperation(hidden = true, value = "Create a partitioned topic.", notes = "It needs to be called before creating a producer on a partitioned topic.") @ApiResponses(value = { @@ -192,7 +192,7 @@ public void createPartitionedTopic( try { validateTopicName(property, cluster, namespace, encodedTopic); internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly, - metadata.topicMetadata); + metadata.properties); } catch (Exception e) { log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); resumeAsyncResponseExceptionally(asyncResponse, e); @@ -228,7 +228,7 @@ public void createNonPartitionedTopic( } @PUT - @Path("/{tenant}/{cluster}/{namespace}/{topic}/topicMetadata") + @Path("/{tenant}/{cluster}/{namespace}/{topic}/properties") @ApiOperation(value = "Create a non-partitioned topic.", notes = "This is the only REST endpoint from which non-partitioned topics could be created.") @ApiResponses(value = { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index d234c0ab817b3..2cca9c2e781e4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -243,7 +243,7 @@ public void createPartitionedTopic( } @PUT - @Path("/{tenant}/{namespace}/{topic}/partitions/topicMetadata") + @Path("/{tenant}/{namespace}/{topic}/partitions/properties") @ApiOperation(value = "Create a partitioned topic.", notes = "It needs to be called before creating a producer on a partitioned topic.") @ApiResponses(value = { @@ -276,7 +276,7 @@ public void createPartitionedTopic( validateTopicPolicyOperation(topicName, PolicyName.PARTITION, PolicyOperation.WRITE); validateCreateTopic(topicName); internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly, - metadata.topicMetadata); + metadata.properties); } catch (Exception e) { log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); resumeAsyncResponseExceptionally(asyncResponse, e); @@ -310,7 +310,7 @@ public void createNonPartitionedTopic( } @PUT - @Path("/{tenant}/{namespace}/{topic}/topicMetadata") + @Path("/{tenant}/{namespace}/{topic}/properties") @ApiOperation(value = "Create a non-partitioned topic.", notes = "This is the only REST endpoint from which non-partitioned topics could be created.") @ApiResponses(value = { @@ -333,12 +333,12 @@ public void createNonPartitionedTopic( @ApiParam(value = "Is authentication required to perform this operation") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @ApiParam(value = "Key value pair properties for the topic metadata") - Map topicMetadata) { + Map properties) { validateNamespaceName(tenant, namespace); validateGlobalNamespaceOwnership(); validateTopicName(tenant, namespace, encodedTopic); validateCreateTopic(topicName); - internalCreateNonPartitionedTopic(authoritative, topicMetadata); + internalCreateNonPartitionedTopic(authoritative, properties); } @GET diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 61db1399d67eb..696f7baa7bb2b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -863,7 +863,7 @@ public CompletableFuture> getTopic(final String topic, boolean c } public CompletableFuture> getTopic(final String topic, boolean createIfMissing, - Map topicMetadata) { + Map properties) { try { CompletableFuture> topicFuture = topics.get(topic); if (topicFuture != null) { @@ -880,7 +880,7 @@ public CompletableFuture> getTopic(final String topic, boolean c return topicFuture.thenCompose(value -> { if (!value.isPresent()) { // retry and create topic - return getTopic(topic, createIfMissing, topicMetadata); + return getTopic(topic, createIfMissing, properties); } else { // in-progress future completed successfully return CompletableFuture.completedFuture(value); @@ -895,7 +895,7 @@ public CompletableFuture> getTopic(final String topic, boolean c final boolean isPersistentTopic = TopicName.get(topic).getDomain().equals(TopicDomain.persistent); if (isPersistentTopic) { return topics.computeIfAbsent(topic, (topicName) -> { - return this.loadOrCreatePersistentTopic(topicName, createIfMissing, topicMetadata); + return this.loadOrCreatePersistentTopic(topicName, createIfMissing, properties); }); } else { return topics.computeIfAbsent(topic, (name) -> { @@ -1238,7 +1238,7 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c * @throws RuntimeException */ protected CompletableFuture> loadOrCreatePersistentTopic(final String topic, - boolean createIfMissing, Map topicMetadata) throws RuntimeException { + boolean createIfMissing, Map properties) throws RuntimeException { final CompletableFuture> topicFuture = FutureUtil.createFutureWithTimeout( Duration.ofSeconds(pulsar.getConfiguration().getTopicLoadTimeoutSeconds()), executor(), () -> FAILED_TO_LOAD_TOPIC_TIMEOUT_EXCEPTION); @@ -1256,7 +1256,7 @@ protected CompletableFuture> loadOrCreatePersistentTopic(final S final Semaphore topicLoadSemaphore = topicLoadRequestSemaphore.get(); if (topicLoadSemaphore.tryAcquire()) { - createPersistentTopic(topic, createIfMissing, topicFuture, topicMetadata); + createPersistentTopic(topic, createIfMissing, topicFuture, properties); topicFuture.handle((persistentTopic, ex) -> { // release permit and process pending topic topicLoadSemaphore.release(); @@ -1264,7 +1264,7 @@ protected CompletableFuture> loadOrCreatePersistentTopic(final S return null; }); } else { - pendingTopicLoadingQueue.add(new ImmutableTriple<>(topic, topicFuture, topicMetadata)); + pendingTopicLoadingQueue.add(new ImmutableTriple<>(topic, topicFuture, properties)); if (log.isDebugEnabled()) { log.debug("topic-loading for {} added into pending queue", topic); } @@ -1279,7 +1279,7 @@ protected CompletableFuture> loadOrCreatePersistentTopic(final S private void createPersistentTopic(final String topic, boolean createIfMissing, CompletableFuture> topicFuture, - Map topicMetadata) { + Map properties) { final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); TopicName topicName = TopicName.get(topic); @@ -1320,7 +1320,7 @@ private void createPersistentTopic(final String topic, boolean createIfMissing, } managedLedgerConfig.setCreateIfMissing(createIfMissing); - managedLedgerConfig.setProperties(topicMetadata); + managedLedgerConfig.setProperties(properties); // Once we have the configuration, we can proceed with the async open operation managedLedgerFactory.asyncOpen(topicName.getPersistenceNamingEncoding(), managedLedgerConfig, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index bcf0a65cd4f32..768697b23f9b6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -411,7 +411,7 @@ public void testCreateNonPartitionedTopic() { persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName2, true, topicMetadata); PartitionedTopicMetadata pMetadata2 = persistentTopics.getPartitionedMetadata( testTenant, testNamespace, topicName2, true, false); - Assert.assertNull(pMetadata2.topicMetadata); + Assert.assertNull(pMetadata2.properties); } @Test @@ -421,7 +421,7 @@ public void testCreatePartitionedTopic() { persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2, true); PartitionedTopicMetadata pMetadata = persistentTopics.getPartitionedMetadata( testTenant, testNamespace, topicName, true, false); - Assert.assertNull(pMetadata.topicMetadata); + Assert.assertNull(pMetadata.partitions); final String topicName2 = "standard-partitioned-topic-b"; Map topicMetadata = Maps.newHashMap(); @@ -430,8 +430,8 @@ public void testCreatePartitionedTopic() { persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName2, true, metadata); PartitionedTopicMetadata pMetadata2 = persistentTopics.getPartitionedMetadata( testTenant, testNamespace, topicName2, true, false); - Assert.assertEquals(pMetadata2.topicMetadata.size(), 1); - Assert.assertEquals(pMetadata2.topicMetadata, topicMetadata); + Assert.assertEquals(pMetadata2.properties.size(), 1); + Assert.assertEquals(pMetadata2.properties, topicMetadata); } @Test(expectedExceptions = RestException.class) diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java index 927950a72285e..7c0f8a5fbd20e 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java @@ -358,11 +358,11 @@ default void createPartitionedTopic(String topic, int numPartitions) throws Puls * Topic name * @param numPartitions * Number of partitions to create of the topic - * @param metadata - * Metadata of the topic + * @param properties + * topic properties * @throws PulsarAdminException */ - void createPartitionedTopic(String topic, int numPartitions, Map metadata) + void createPartitionedTopic(String topic, int numPartitions, Map properties) throws PulsarAdminException; /** @@ -393,11 +393,12 @@ default CompletableFuture createPartitionedTopicAsync(String topic, int nu * Topic name * @param numPartitions * Number of partitions to create of the topic - * @param metadata - * Metadata of the topic + * @param properties + * Topic properties * @return a future that can be used to track when the partitioned topic is created */ - CompletableFuture createPartitionedTopicAsync(String topic, int numPartitions, Map metadata); + CompletableFuture createPartitionedTopicAsync(String topic, int numPartitions, + Map properties); /** * Create a non-partitioned topic. @@ -419,10 +420,10 @@ default void createNonPartitionedTopic(String topic) throws PulsarAdminException *

* * @param topic Topic name - * @param metadata Topic metadata + * @param properties Topic properties * @throws PulsarAdminException */ - void createNonPartitionedTopic(String topic, Map metadata) throws PulsarAdminException; + void createNonPartitionedTopic(String topic, Map properties) throws PulsarAdminException; /** * Create a non-partitioned topic asynchronously. @@ -437,9 +438,9 @@ default CompletableFuture createNonPartitionedTopicAsync(String topic) { * Create a non-partitioned topic asynchronously. * * @param topic Topic name - * @param metadata Topic metadata + * @param properties Topic properties */ - CompletableFuture createNonPartitionedTopicAsync(String topic, Map metadata); + CompletableFuture createNonPartitionedTopicAsync(String topic, Map properties); /** * Create missed partitions for partitioned topic. diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/partition/PartitionedTopicMetadata.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/partition/PartitionedTopicMetadata.java index 1c32ff5731376..4d0f6b8e313d9 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/partition/PartitionedTopicMetadata.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/partition/PartitionedTopicMetadata.java @@ -28,8 +28,8 @@ public class PartitionedTopicMetadata { /* Number of partitions for the topic */ public int partitions; - /* Topic metadata */ - public Map topicMetadata; + /* Topic properties */ + public Map properties; public PartitionedTopicMetadata() { this.partitions = 0; @@ -39,9 +39,9 @@ public PartitionedTopicMetadata(int partitions) { this.partitions = partitions; } - public PartitionedTopicMetadata(int partitions, Map topicMetadata) { + public PartitionedTopicMetadata(int partitions, Map properties) { this.partitions = partitions; - this.topicMetadata = topicMetadata; + this.properties = properties; } /** diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java index 25cbe4d3a92ca..0c00ab8ced5dc 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java @@ -370,16 +370,7 @@ public CompletableFuture revokePermissionsAsync(String topic, String role) @Override public void createPartitionedTopic(String topic, int numPartitions, Map metadata) throws PulsarAdminException { - try { - createPartitionedTopicAsync(topic, numPartitions, metadata).get(this.readTimeoutMs, TimeUnit.MILLISECONDS); - } catch (ExecutionException e) { - throw (PulsarAdminException) e.getCause(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new PulsarAdminException(e); - } catch (TimeoutException e) { - throw new PulsarAdminException.TimeoutException(e); - } + sync(() -> createPartitionedTopicAsync(topic, numPartitions, metadata)); } @Override @@ -411,25 +402,25 @@ public void createMissedPartitions(String topic) throws PulsarAdminException { } @Override - public CompletableFuture createNonPartitionedTopicAsync(String topic, Map metadata){ + public CompletableFuture createNonPartitionedTopicAsync(String topic, Map properties){ TopicName tn = validateTopic(topic); - WebTarget path = topicPath(tn, "topicMetadata"); - return asyncPutRequest(path, Entity.entity(metadata, MediaType.APPLICATION_JSON)); + WebTarget path = topicPath(tn, "properties"); + return asyncPutRequest(path, Entity.entity(properties, MediaType.APPLICATION_JSON)); } @Override public CompletableFuture createPartitionedTopicAsync(String topic, int numPartitions, - Map metadata) { - return createPartitionedTopicAsync(topic, numPartitions, false, metadata); + Map properties) { + return createPartitionedTopicAsync(topic, numPartitions, false, properties); } public CompletableFuture createPartitionedTopicAsync( - String topic, int numPartitions, boolean createLocalTopicOnly, Map topicMetadata) { + String topic, int numPartitions, boolean createLocalTopicOnly, Map properties) { checkArgument(numPartitions > 0, "Number of partitions should be more than 0"); TopicName tn = validateTopic(topic); - WebTarget path = topicPath(tn, "partitions", "topicMetadata") + WebTarget path = topicPath(tn, "partitions", "properties") .queryParam("createLocalTopicOnly", Boolean.toString(createLocalTopicOnly)); - PartitionedTopicMetadata metadata = new PartitionedTopicMetadata(numPartitions, topicMetadata); + PartitionedTopicMetadata metadata = new PartitionedTopicMetadata(numPartitions, properties); return asyncPutRequest(path, Entity.entity(metadata, MediaType.APPLICATION_JSON)); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index 7168bce96e297..ae009f22b4db1 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -385,14 +385,14 @@ private class CreatePartitionedCmd extends CliCommand { "--partitions" }, description = "Number of partitions for the topic", required = true) private int numPartitions; - @Parameter(names = {"--topic-metadata", "-tm"}, description = "key value pair properties(a=a,b=b,c=c)") - private java.util.List topicMetadata; + @Parameter(names = {"--metadata", "-m"}, description = "key value pair properties(a=a,b=b,c=c)") + private java.util.List metadata; @Override void run() throws Exception { String topic = validateTopicName(params); Map map = new HashMap<>(); - for (String property : topicMetadata) { + for (String property : metadata) { if (!property.contains("=")) { throw new ParameterException(String.format("Invalid key value pair '%s', " + "valid format like 'a=a,b=b,c=c'.", property)); @@ -430,14 +430,14 @@ private class CreateNonPartitionedCmd extends CliCommand { @Parameter(description = "persistent://tenant/namespace/topic", required = true) private java.util.List params; - @Parameter(names = {"--topic-metadata", "-tm"}, description = "key value pair properties(a=a,b=b,c=c)") - private java.util.List topicMetadata; + @Parameter(names = {"--metadata", "-m"}, description = "key value pair properties(a=a,b=b,c=c)") + private java.util.List metadata; @Override void run() throws Exception { String topic = validateTopicName(params); Map map = new HashMap<>(); - for (String property : topicMetadata) { + for (String property : metadata) { if (!property.contains("=")) { throw new ParameterException(String.format("Invalid key value pair '%s', " + "valid format like 'a=a,b=b,c=c'.", property)); From 42b2f4253bdf4f188b5e14baa1bc2f55c437f570 Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 16 Nov 2021 16:43:26 +0800 Subject: [PATCH 07/25] fix test. --- .../pulsar/admin/cli/PulsarAdminToolTest.java | 3 +- .../apache/pulsar/admin/cli/CmdTopics.java | 36 ++++++++++--------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 529c1ea19a4b6..58587909e6c68 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.concurrent.CompletableFuture; @@ -1434,7 +1435,7 @@ public void nonPersistentTopics() throws Exception { verify(mockTopics).getInternalStats("non-persistent://myprop/ns1/ds1", false); topics.run(split("create-partitioned-topic non-persistent://myprop/ns1/ds1 --partitions 32")); - verify(mockTopics).createPartitionedTopic("non-persistent://myprop/ns1/ds1", 32); + verify(mockTopics).createPartitionedTopic("non-persistent://myprop/ns1/ds1", 32, new HashMap<>()); topics.run(split("list myprop/ns1")); verify(mockTopics).getList("myprop/ns1", null); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index ae009f22b4db1..011f25bf443ab 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -392,17 +392,19 @@ private class CreatePartitionedCmd extends CliCommand { void run() throws Exception { String topic = validateTopicName(params); Map map = new HashMap<>(); - for (String property : metadata) { - if (!property.contains("=")) { - throw new ParameterException(String.format("Invalid key value pair '%s', " + - "valid format like 'a=a,b=b,c=c'.", property)); - } else { - String[] keyValue = property.split("="); - if (keyValue.length != 2) { + if (metadata != null) { + for (String property : metadata) { + if (!property.contains("=")) { throw new ParameterException(String.format("Invalid key value pair '%s', " + "valid format like 'a=a,b=b,c=c'.", property)); + } else { + String[] keyValue = property.split("="); + if (keyValue.length != 2) { + throw new ParameterException(String.format("Invalid key value pair '%s', " + + "valid format like 'a=a,b=b,c=c'.", property)); + } + map.put(keyValue[0], keyValue[1]); } - map.put(keyValue[0], keyValue[1]); } } getTopics().createPartitionedTopic(topic, numPartitions, map); @@ -437,17 +439,19 @@ private class CreateNonPartitionedCmd extends CliCommand { void run() throws Exception { String topic = validateTopicName(params); Map map = new HashMap<>(); - for (String property : metadata) { - if (!property.contains("=")) { - throw new ParameterException(String.format("Invalid key value pair '%s', " + - "valid format like 'a=a,b=b,c=c'.", property)); - } else { - String[] keyValue = property.split("="); - if (keyValue.length != 2) { + if (metadata != null) { + for (String property : metadata) { + if (!property.contains("=")) { throw new ParameterException(String.format("Invalid key value pair '%s', " + "valid format like 'a=a,b=b,c=c'.", property)); + } else { + String[] keyValue = property.split("="); + if (keyValue.length != 2) { + throw new ParameterException(String.format("Invalid key value pair '%s', " + + "valid format like 'a=a,b=b,c=c'.", property)); + } + map.put(keyValue[0], keyValue[1]); } - map.put(keyValue[0], keyValue[1]); } } getTopics().createNonPartitionedTopic(topic, map); From fa420e573114c578e234f7d8a1c11159397a39cc Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 16 Nov 2021 17:11:37 +0800 Subject: [PATCH 08/25] fix test. --- .../pulsar/client/admin/internal/TopicsImpl.java | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java index 0c00ab8ced5dc..d321f6420a359 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.Base64; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -375,16 +376,7 @@ public void createPartitionedTopic(String topic, int numPartitions, Map metadata) throws PulsarAdminException { - try { - createNonPartitionedTopicAsync(topic, metadata).get(this.readTimeoutMs, TimeUnit.MILLISECONDS); - } catch (ExecutionException e) { - throw (PulsarAdminException) e.getCause(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new PulsarAdminException(e); - } catch (TimeoutException e) { - throw new PulsarAdminException.TimeoutException(e); - } + sync(() -> createNonPartitionedTopicAsync(topic, metadata)); } @Override @@ -405,6 +397,7 @@ public void createMissedPartitions(String topic) throws PulsarAdminException { public CompletableFuture createNonPartitionedTopicAsync(String topic, Map properties){ TopicName tn = validateTopic(topic); WebTarget path = topicPath(tn, "properties"); + properties = properties == null ? new HashMap<>() : properties; return asyncPutRequest(path, Entity.entity(properties, MediaType.APPLICATION_JSON)); } From 96fc5dc9e7429f3914defdad359ec0a5952e0854 Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 16 Nov 2021 20:04:59 +0800 Subject: [PATCH 09/25] fix test. --- .../org/apache/pulsar/broker/admin/PersistentTopicsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 768697b23f9b6..1eb1a28b28f1c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -421,7 +421,7 @@ public void testCreatePartitionedTopic() { persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2, true); PartitionedTopicMetadata pMetadata = persistentTopics.getPartitionedMetadata( testTenant, testNamespace, topicName, true, false); - Assert.assertNull(pMetadata.partitions); + Assert.assertNull(pMetadata.properties); final String topicName2 = "standard-partitioned-topic-b"; Map topicMetadata = Maps.newHashMap(); From 3972fb587ebf87eab97dc83ab84f0918ce3147af Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 16 Nov 2021 21:34:51 +0800 Subject: [PATCH 10/25] fix test. --- .../java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index 58587909e6c68..650209ccaaa1e 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -1000,13 +1000,13 @@ public void topics() throws Exception { verify(mockTopics).createSubscription("persistent://myprop/clust/ns1/ds1", "sub1", MessageId.earliest); cmdTopics.run(split("create-partitioned-topic persistent://myprop/clust/ns1/ds1 --partitions 32")); - verify(mockTopics).createPartitionedTopic("persistent://myprop/clust/ns1/ds1", 32); + verify(mockTopics).createPartitionedTopic("persistent://myprop/clust/ns1/ds1", 32, new HashMap<>()); cmdTopics.run(split("create-missed-partitions persistent://myprop/clust/ns1/ds1")); verify(mockTopics).createMissedPartitions("persistent://myprop/clust/ns1/ds1"); cmdTopics.run(split("create persistent://myprop/clust/ns1/ds1")); - verify(mockTopics).createNonPartitionedTopic("persistent://myprop/clust/ns1/ds1"); + verify(mockTopics).createNonPartitionedTopic("persistent://myprop/clust/ns1/ds1", new HashMap<>()); cmdTopics.run(split("list-partitioned-topics myprop/clust/ns1")); verify(mockTopics).getPartitionedTopicList("myprop/clust/ns1"); From 524911dbf0c40f931c3d651e67de286cf123a347 Mon Sep 17 00:00:00 2001 From: technoboy Date: Thu, 30 Dec 2021 19:39:40 +0800 Subject: [PATCH 11/25] fix compile failure --- .../java/org/apache/pulsar/broker/service/BrokerService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 6872eff4fb473..f0e860c93b1b9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -89,6 +89,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.bookkeeper.mledger.util.Futures; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.ImmutableTriple; import org.apache.commons.lang3.tuple.Triple; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; From 13e5a28f80dd789051deec5456c85de07f9d525b Mon Sep 17 00:00:00 2001 From: technoboy Date: Thu, 30 Dec 2021 20:24:55 +0800 Subject: [PATCH 12/25] fix checkstyle --- .../java/org/apache/pulsar/broker/service/BrokerService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index f0e860c93b1b9..3f7ac101eeb6e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -91,8 +91,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutableTriple; import org.apache.commons.lang3.tuple.Triple; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.bookie.rackawareness.IsolatedBookieEnsemblePlacementPolicy; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; From 5cdef7f130ff7cc10dde75423e27c1d2c2d47a23 Mon Sep 17 00:00:00 2001 From: technoboy Date: Thu, 30 Dec 2021 21:52:55 +0800 Subject: [PATCH 13/25] fix test. --- .../org/apache/pulsar/broker/admin/PersistentTopicsTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 8528285da22a3..591753328ed45 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -39,6 +39,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -422,13 +423,13 @@ public void testCreateNonPartitionedTopic() { @Test public void testCreatePartitionedTopic() { AsyncResponse response = mock(AsyncResponse.class); - final String topicName = "standard-partitioned-topic-a"; + final String topicName = "standard-partitioned-topic-a" + UUID.randomUUID().toString(); persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2, true); PartitionedTopicMetadata pMetadata = persistentTopics.getPartitionedMetadata( testTenant, testNamespace, topicName, true, false); Assert.assertNull(pMetadata.properties); - final String topicName2 = "standard-partitioned-topic-b"; + final String topicName2 = "standard-partitioned-topic-b" + UUID.randomUUID().toString(); Map topicMetadata = Maps.newHashMap(); topicMetadata.put("key1", "value1"); PartitionedTopicMetadata metadata = new PartitionedTopicMetadata(2, topicMetadata); From 8f4b015ad7715278a0f762b3861891e125e43379 Mon Sep 17 00:00:00 2001 From: technoboy Date: Thu, 30 Dec 2021 22:33:46 +0800 Subject: [PATCH 14/25] fix test. --- .../broker/admin/PersistentTopicsTest.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 591753328ed45..5523bca18b6d6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -85,6 +85,7 @@ import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.zookeeper.KeeperException; +import org.awaitility.Awaitility; import org.mockito.ArgumentCaptor; import org.powermock.reflect.Whitebox; import org.testng.Assert; @@ -423,21 +424,24 @@ public void testCreateNonPartitionedTopic() { @Test public void testCreatePartitionedTopic() { AsyncResponse response = mock(AsyncResponse.class); - final String topicName = "standard-partitioned-topic-a" + UUID.randomUUID().toString(); + final String topicName = "standard-partitioned-topic-a"; persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2, true); - PartitionedTopicMetadata pMetadata = persistentTopics.getPartitionedMetadata( - testTenant, testNamespace, topicName, true, false); - Assert.assertNull(pMetadata.properties); - - final String topicName2 = "standard-partitioned-topic-b" + UUID.randomUUID().toString(); + Awaitility.await().untilAsserted(() -> { + PartitionedTopicMetadata pMetadata = persistentTopics.getPartitionedMetadata( + testTenant, testNamespace, topicName, true, false); + Assert.assertNull(pMetadata.properties); + }); + final String topicName2 = "standard-partitioned-topic-b"; Map topicMetadata = Maps.newHashMap(); topicMetadata.put("key1", "value1"); PartitionedTopicMetadata metadata = new PartitionedTopicMetadata(2, topicMetadata); persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName2, true, metadata); - PartitionedTopicMetadata pMetadata2 = persistentTopics.getPartitionedMetadata( - testTenant, testNamespace, topicName2, true, false); - Assert.assertEquals(pMetadata2.properties.size(), 1); - Assert.assertEquals(pMetadata2.properties, topicMetadata); + Awaitility.await().untilAsserted(() -> { + PartitionedTopicMetadata pMetadata2 = persistentTopics.getPartitionedMetadata( + testTenant, testNamespace, topicName2, true, false); + Assert.assertEquals(pMetadata2.properties.size(), 1); + Assert.assertEquals(pMetadata2.properties, topicMetadata); + }); } @Test(expectedExceptions = RestException.class) From d435669a8081edbca1a0df8069b75124d44eb8f1 Mon Sep 17 00:00:00 2001 From: technoboy Date: Fri, 31 Dec 2021 09:26:04 +0800 Subject: [PATCH 15/25] fix checkstyle. --- .../org/apache/pulsar/broker/admin/PersistentTopicsTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 5523bca18b6d6..670375db4788a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -39,7 +39,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; From b17c7de49c2f8552f5b8c33474cf0304131013ce Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 11 Jan 2022 10:08:18 +0800 Subject: [PATCH 16/25] fix checkstyle --- .../apache/pulsar/admin/cli/CmdTopics.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index da96eb0f5b064..8d0a2d8beba41 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -497,13 +497,13 @@ void run() throws Exception { if (metadata != null) { for (String property : metadata) { if (!property.contains("=")) { - throw new ParameterException(String.format("Invalid key value pair '%s', " + - "valid format like 'a=a,b=b,c=c'.", property)); + throw new ParameterException(String.format("Invalid key value pair '%s', " + + "valid format like 'a=a,b=b,c=c'.", property)); } else { String[] keyValue = property.split("="); if (keyValue.length != 2) { - throw new ParameterException(String.format("Invalid key value pair '%s', " + - "valid format like 'a=a,b=b,c=c'.", property)); + throw new ParameterException(String.format("Invalid key value pair '%s', " + + "valid format like 'a=a,b=b,c=c'.", property)); } map.put(keyValue[0], keyValue[1]); } @@ -537,27 +537,27 @@ private class CreateNonPartitionedCmd extends CliCommand { @Parameter(names = {"--metadata", "-m"}, description = "key value pair properties(a=a,b=b,c=c)") private java.util.List metadata; - @Override - void run() throws Exception { - String topic = validateTopicName(params); + @Override + void run() throws Exception { + String topic = validateTopicName(params); Map map = new HashMap<>(); if (metadata != null) { for (String property : metadata) { if (!property.contains("=")) { - throw new ParameterException(String.format("Invalid key value pair '%s', " + - "valid format like 'a=a,b=b,c=c'.", property)); + throw new ParameterException(String.format("Invalid key value pair '%s', " + + "valid format like 'a=a,b=b,c=c'.", property)); } else { String[] keyValue = property.split("="); if (keyValue.length != 2) { - throw new ParameterException(String.format("Invalid key value pair '%s', " + - "valid format like 'a=a,b=b,c=c'.", property)); + throw new ParameterException(String.format("Invalid key value pair '%s', " + + "valid format like 'a=a,b=b,c=c'.", property)); } map.put(keyValue[0], keyValue[1]); } } } - getTopics().createNonPartitionedTopic(topic, map); - } + getTopics().createNonPartitionedTopic(topic, map); + } } @Parameters(commandDescription = "Update existing non-global partitioned topic. " From ab8adba54d7d1778d942db80a4997c1b7720fd26 Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 18 Jan 2022 20:40:40 +0800 Subject: [PATCH 17/25] Remove createPartitionedTopic from v1. --- .../broker/admin/v1/PersistentTopics.java | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index 16622bd257f55..8905064e7e30e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -173,34 +173,6 @@ public void createPartitionedTopic( } } - @PUT - @Path("/{property}/{cluster}/{namespace}/{topic}/partitions/properties") - @ApiOperation(hidden = true, value = "Create a partitioned topic.", - notes = "It needs to be called before creating a producer on a partitioned topic.") - @ApiResponses(value = { - @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), - @ApiResponse(code = 403, message = "Don't have admin permission"), - @ApiResponse(code = 406, message = "The number of partitions should be " - + "more than 0 and less than or equal to maxNumPartitionsPerPartitionedTopic"), - @ApiResponse(code = 409, message = "Partitioned topic already exist")}) - public void createPartitionedTopic( - @Suspended final AsyncResponse asyncResponse, - @PathParam("property") String property, - @PathParam("cluster") String cluster, - @PathParam("namespace") String namespace, - @PathParam("topic") @Encoded String encodedTopic, - @QueryParam("createLocalTopicOnly") @DefaultValue("false") boolean createLocalTopicOnly, - PartitionedTopicMetadata metadata) { - try { - validateTopicName(property, cluster, namespace, encodedTopic); - internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly, - metadata.properties); - } catch (Exception e) { - log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); - resumeAsyncResponseExceptionally(asyncResponse, e); - } - } - @PUT @Path("/{tenant}/{cluster}/{namespace}/{topic}") @ApiOperation(value = "Create a non-partitioned topic.", From 2131936b15bae70e14dc642a01913ce01c7e9c78 Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 18 Jan 2022 21:55:35 +0800 Subject: [PATCH 18/25] remove createNonPartitionedTopic in v1 --- .../broker/admin/v1/PersistentTopics.java | 34 +------------------ 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index 8905064e7e30e..8c0b16e82b254 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -198,39 +198,7 @@ public void createNonPartitionedTopic( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Is authentication required to perform this operation") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { - createNonPartitionedTopic(tenant, cluster, namespace, encodedTopic, authoritative, null); - } - - @PUT - @Path("/{tenant}/{cluster}/{namespace}/{topic}/properties") - @ApiOperation(value = "Create a non-partitioned topic.", - notes = "This is the only REST endpoint from which non-partitioned topics could be created.") - @ApiResponses(value = { - @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), - @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), - @ApiResponse(code = 403, message = "Don't have admin permission"), - @ApiResponse(code = 409, message = "Partitioned topic already exist"), - @ApiResponse(code = 412, - message = "Failed Reason : Name is invalid or Namespace does not have any clusters configured"), - @ApiResponse(code = 500, message = "Internal server error"), - @ApiResponse(code = 503, message = "Failed to validate global cluster configuration") - }) - public void createNonPartitionedTopic( - @ApiParam(value = "Specify the tenant", required = true) - @PathParam("tenant") String tenant, - @ApiParam(value = "Specify the cluster", required = true) - @PathParam("cluster") String cluster, - @ApiParam(value = "Specify the namespace", required = true) - @PathParam("namespace") String namespace, - @ApiParam(value = "Specify topic name", required = true) - @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") - @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, - @ApiParam(value = "Key value pair properties for the topic metadata") Map topicMetadata) { - validateNamespaceName(tenant, cluster, namespace); - validateTopicName(tenant, cluster, namespace, encodedTopic); - validateGlobalNamespaceOwnership(); - internalCreateNonPartitionedTopic(authoritative, topicMetadata); + createNonPartitionedTopic(tenant, cluster, namespace, encodedTopic, authoritative); } /** From 629517c6e56989be1165f8527e48548969382959 Mon Sep 17 00:00:00 2001 From: technoboy Date: Tue, 18 Jan 2022 21:58:42 +0800 Subject: [PATCH 19/25] updates --- .../org/apache/pulsar/broker/admin/v1/PersistentTopics.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index 8c0b16e82b254..92c791da4bf96 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -198,7 +198,10 @@ public void createNonPartitionedTopic( @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Is authentication required to perform this operation") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { - createNonPartitionedTopic(tenant, cluster, namespace, encodedTopic, authoritative); + validateNamespaceName(tenant, cluster, namespace); + validateTopicName(tenant, cluster, namespace, encodedTopic); + validateGlobalNamespaceOwnership(); + internalCreateNonPartitionedTopic(authoritative, null); } /** From a4121fe15f4299a0eda37e4528378e05f59c8a9e Mon Sep 17 00:00:00 2001 From: technoboy Date: Wed, 19 Jan 2022 13:09:39 +0800 Subject: [PATCH 20/25] updates --- .../broker/admin/v1/PersistentTopics.java | 11 +-- .../broker/admin/v2/NonPersistentTopics.java | 7 +- .../broker/admin/v2/PersistentTopics.java | 71 +------------------ .../apache/pulsar/broker/admin/AdminTest.java | 7 +- .../broker/admin/PersistentTopicsTest.java | 28 ++++---- .../partition/PartitionedTopicMetadata.java | 3 +- .../client/admin/internal/TopicsImpl.java | 4 +- 7 files changed, 34 insertions(+), 97 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index 92c791da4bf96..4f2e3e377c872 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -162,11 +162,13 @@ public void createPartitionedTopic( @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - int numPartitions, + @ApiParam(value = "The metadata for the topic", required = true, type = "PartitionedTopicMetadata") + PartitionedTopicMetadata metadata, @QueryParam("createLocalTopicOnly") @DefaultValue("false") boolean createLocalTopicOnly) { try { validateTopicName(property, cluster, namespace, encodedTopic); - internalCreatePartitionedTopic(asyncResponse, numPartitions, createLocalTopicOnly); + internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly, + metadata.properties); } catch (Exception e) { log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); resumeAsyncResponseExceptionally(asyncResponse, e); @@ -197,11 +199,12 @@ public void createNonPartitionedTopic( @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Is authentication required to perform this operation") - @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, + @ApiParam(value = "Key value pair properties for the topic metadata") Map topicMetadata) { validateNamespaceName(tenant, cluster, namespace); validateTopicName(tenant, cluster, namespace, encodedTopic); validateGlobalNamespaceOwnership(); - internalCreateNonPartitionedTopic(authoritative, null); + internalCreateNonPartitionedTopic(authoritative, topicMetadata); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java index 5eb33ab542040..aaf204632336c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java @@ -196,15 +196,14 @@ public void createPartitionedTopic( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "The number of partitions for the topic", - required = true, type = "int", defaultValue = "0") - int numPartitions, + @ApiParam(value = "The metadata for the topic", + required = true, type = "PartitionedTopicMetadata") PartitionedTopicMetadata metadata, @QueryParam("createLocalTopicOnly") @DefaultValue("false") boolean createLocalTopicOnly) { try { validateNamespaceName(tenant, namespace); validateGlobalNamespaceOwnership(); validateTopicName(tenant, namespace, encodedTopic); - internalCreatePartitionedTopic(asyncResponse, numPartitions, createLocalTopicOnly); + internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly); } catch (Exception e) { log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); resumeAsyncResponseExceptionally(asyncResponse, e); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 3aeb5142b957f..af027ab46d859 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -225,50 +225,9 @@ public void createPartitionedTopic( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "The number of partitions for the topic", - required = true, type = "int", defaultValue = "0") - int numPartitions, + @ApiParam(value = "The metadata for the topic", + required = true, type = "PartitionedTopicMetadata") PartitionedTopicMetadata metadata, @QueryParam("createLocalTopicOnly") @DefaultValue("false") boolean createLocalTopicOnly) { - try { - validateNamespaceName(tenant, namespace); - validateGlobalNamespaceOwnership(); - validatePartitionedTopicName(tenant, namespace, encodedTopic); - validateTopicPolicyOperation(topicName, PolicyName.PARTITION, PolicyOperation.WRITE); - validateCreateTopic(topicName); - internalCreatePartitionedTopic(asyncResponse, numPartitions, createLocalTopicOnly, null); - } catch (Exception e) { - log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); - resumeAsyncResponseExceptionally(asyncResponse, e); - } - } - - @PUT - @Path("/{tenant}/{namespace}/{topic}/partitions/properties") - @ApiOperation(value = "Create a partitioned topic.", - notes = "It needs to be called before creating a producer on a partitioned topic.") - @ApiResponses(value = { - @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), - @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), - @ApiResponse(code = 403, message = "Don't have admin permission"), - @ApiResponse(code = 404, message = "Tenant does not exist"), - @ApiResponse(code = 406, message = "The number of partitions should be more than 0 and" - + " less than or equal to maxNumPartitionsPerPartitionedTopic"), - @ApiResponse(code = 409, message = "Partitioned topic already exist"), - @ApiResponse(code = 412, - message = "Failed Reason : Name is invalid or Namespace does not have any clusters configured"), - @ApiResponse(code = 500, message = "Internal server error"), - @ApiResponse(code = 503, message = "Failed to validate global cluster configuration") - }) - public void createPartitionedTopic( - @Suspended final AsyncResponse asyncResponse, - @ApiParam(value = "Specify the tenant", required = true) - @PathParam("tenant") String tenant, - @ApiParam(value = "Specify the namespace", required = true) - @PathParam("namespace") String namespace, - @ApiParam(value = "Specify topic name", required = true) - @PathParam("topic") @Encoded String encodedTopic, - @QueryParam("createLocalTopicOnly") @DefaultValue("false") boolean createLocalTopicOnly, - PartitionedTopicMetadata metadata) { try { validateNamespaceName(tenant, namespace); validateGlobalNamespaceOwnership(); @@ -297,32 +256,6 @@ public void createPartitionedTopic( @ApiResponse(code = 500, message = "Internal server error"), @ApiResponse(code = 503, message = "Failed to validate global cluster configuration") }) - public void createNonPartitionedTopic( - @ApiParam(value = "Specify the tenant", required = true) - @PathParam("tenant") String tenant, - @ApiParam(value = "Specify the namespace", required = true) - @PathParam("namespace") String namespace, - @ApiParam(value = "Specify topic name", required = true) - @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "Is authentication required to perform this operation") - @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { - createNonPartitionedTopic(tenant, namespace, encodedTopic, authoritative, null); - } - - @PUT - @Path("/{tenant}/{namespace}/{topic}/properties") - @ApiOperation(value = "Create a non-partitioned topic.", - notes = "This is the only REST endpoint from which non-partitioned topics could be created.") - @ApiResponses(value = { - @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), - @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), - @ApiResponse(code = 403, message = "Don't have admin permission"), - @ApiResponse(code = 409, message = "Partitioned topic already exist"), - @ApiResponse(code = 412, - message = "Failed Reason : Name is invalid or Namespace does not have any clusters configured"), - @ApiResponse(code = 500, message = "Internal server error"), - @ApiResponse(code = 503, message = "Failed to validate global cluster configuration") - }) public void createNonPartitionedTopic( @ApiParam(value = "Specify the tenant", required = true) @PathParam("tenant") String tenant, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java index 1ee7a41f5ae0b..872ee7810c790 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java @@ -70,6 +70,7 @@ import org.apache.pulsar.common.conf.InternalConfigurationData; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData; import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType; @@ -760,7 +761,7 @@ public void persistentTopics() throws Exception { assertEquals(persistentTopics.getPartitionedTopicList(property, cluster, namespace), Lists.newArrayList()); response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response, property, cluster, namespace, topic, 5, false); + persistentTopics.createPartitionedTopic(response, property, cluster, namespace, topic, new PartitionedTopicMetadata(5), false); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); assertEquals(persistentTopics.getPartitionedTopicList(property, cluster, namespace), Lists @@ -811,13 +812,13 @@ public void testUpdatePartitionedTopicCoontainedInOldTopic() throws Exception { AsyncResponse response1 = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response1, property, cluster, namespace, partitionedTopicName, 5, false); + persistentTopics.createPartitionedTopic(response1, property, cluster, namespace, partitionedTopicName, new PartitionedTopicMetadata(5), false); verify(response1, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); AsyncResponse response2 = mock(AsyncResponse.class); responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response2, property, cluster, namespace, partitionedTopicName2, 2, false); + persistentTopics.createPartitionedTopic(response2, property, cluster, namespace, partitionedTopicName2, new PartitionedTopicMetadata(2), false); verify(response2, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 942b1ca9828e5..8ab51329b523a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -185,7 +185,7 @@ public void testGetSubscriptions() { // 3) Create the partitioned topic response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, testLocalTopicName, 3, true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, testLocalTopicName, new PartitionedTopicMetadata(3), true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); @@ -325,7 +325,7 @@ public void testTerminatePartitionedTopic() { // 3) Create the partitioned topic AsyncResponse response = mock(AsyncResponse.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, testLocalTopicName, 1, true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, testLocalTopicName, new PartitionedTopicMetadata(1), true); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); @@ -427,7 +427,7 @@ public void testCreateNonPartitionedTopic() { public void testCreatePartitionedTopic() { AsyncResponse response = mock(AsyncResponse.class); final String topicName = "standard-partitioned-topic-a"; - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2, true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, new PartitionedTopicMetadata(2), true); Awaitility.await().untilAsserted(() -> { PartitionedTopicMetadata pMetadata = persistentTopics.getPartitionedMetadata( testTenant, testNamespace, topicName, true, false); @@ -437,7 +437,7 @@ public void testCreatePartitionedTopic() { Map topicMetadata = Maps.newHashMap(); topicMetadata.put("key1", "value1"); PartitionedTopicMetadata metadata = new PartitionedTopicMetadata(2, topicMetadata); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName2, true, metadata); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName2, metadata, true); Awaitility.await().untilAsserted(() -> { PartitionedTopicMetadata pMetadata2 = persistentTopics.getPartitionedMetadata( testTenant, testNamespace, topicName2, true, false); @@ -475,7 +475,7 @@ public void testCreatePartitionedTopicHavingNonPartitionTopicWithPartitionSuffix doReturn(new Policies()).when(persistentTopics).getNamespacePolicies(any()); AsyncResponse response = mock(AsyncResponse.class); ArgumentCaptor errCaptor = ArgumentCaptor.forClass(RestException.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionedTopicName, 5, true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionedTopicName, new PartitionedTopicMetadata(5), true); verify(response, timeout(5000).times(1)).resume(errCaptor.capture()); Assert.assertEquals(errCaptor.getValue().getResponse().getStatus(), Response.Status.CONFLICT.getStatusCode()); } @@ -495,7 +495,7 @@ public void testUpdatePartitionedTopicHavingNonPartitionTopicWithPartitionSuffix doNothing().when(persistentTopics).validateAdminAccessForTenant(anyString()); AsyncResponse response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionedTopicName, 5, true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionedTopicName, new PartitionedTopicMetadata(5), true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); persistentTopics.updatePartitionedTopic(testTenant, testNamespace, partitionedTopicName, false, false, false, @@ -525,7 +525,7 @@ public void testUnloadTopic() { // 3) create partitioned topic and unload response = mock(AsyncResponse.class); responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionTopicName, 6, true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionTopicName, new PartitionedTopicMetadata(6), true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); @@ -555,13 +555,13 @@ public void testUnloadTopicShallThrowNotFoundWhenTopicNotExist() { public void testGetPartitionedTopicsList() throws KeeperException, InterruptedException, PulsarAdminException { AsyncResponse response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, "test-topic1", 3, true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, "test-topic1", new PartitionedTopicMetadata(3), true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); responseCaptor = ArgumentCaptor.forClass(Response.class); - nonPersistentTopic.createPartitionedTopic(response, testTenant, testNamespace, "test-topic2", 3, true); + nonPersistentTopic.createPartitionedTopic(response, testTenant, testNamespace, "test-topic2", new PartitionedTopicMetadata(3), true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); @@ -591,7 +591,7 @@ public void testGrantNonPartitionedTopic() { public void testCreateExistedPartition() { final AsyncResponse response = mock(AsyncResponse.class); final String topicName = "test-create-existed-partition"; - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 3, true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, new PartitionedTopicMetadata(3), true); final String partitionName = TopicName.get(topicName).getPartition(0).getLocalName(); try { @@ -611,7 +611,7 @@ public void testGrantPartitionedTopic() { AsyncResponse response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); persistentTopics.createPartitionedTopic( - response, testTenant, testNamespace, partitionedTopicName, numPartitions, true); + response, testTenant, testNamespace, partitionedTopicName, new PartitionedTopicMetadata(numPartitions), true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); @@ -652,7 +652,7 @@ public void testRevokePartitionedTopic() { AsyncResponse response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); persistentTopics.createPartitionedTopic( - response, testTenant, testNamespace, partitionedTopicName, numPartitions, true); + response, testTenant, testNamespace, partitionedTopicName, new PartitionedTopicMetadata(numPartitions), true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); String role = "role"; @@ -695,7 +695,7 @@ public void testTriggerCompactionTopic() { // create partitioned topic and compaction on it response = mock(AsyncResponse.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionTopicName, 2, true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionTopicName, new PartitionedTopicMetadata(2), true); persistentTopics.compact(response, testTenant, testNamespace, partitionTopicName, true); responseCaptor = ArgumentCaptor.forClass(Response.class); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); @@ -902,7 +902,7 @@ public void testSetReplicatedSubscriptionStatus() { // 3) Create the partitioned topic response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2,true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, new PartitionedTopicMetadata(2),true); verify(response, timeout(10000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/partition/PartitionedTopicMetadata.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/partition/PartitionedTopicMetadata.java index 4d0f6b8e313d9..7d3fda3ef0b09 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/partition/PartitionedTopicMetadata.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/partition/PartitionedTopicMetadata.java @@ -32,11 +32,12 @@ public class PartitionedTopicMetadata { public Map properties; public PartitionedTopicMetadata() { - this.partitions = 0; + this(0); } public PartitionedTopicMetadata(int partitions) { this.partitions = partitions; + this.properties = null; } public PartitionedTopicMetadata(int partitions, Map properties) { diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java index 07465c942c877..3249e6e4b4b3b 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java @@ -330,7 +330,7 @@ public void createMissedPartitions(String topic) throws PulsarAdminException { @Override public CompletableFuture createNonPartitionedTopicAsync(String topic, Map properties){ TopicName tn = validateTopic(topic); - WebTarget path = topicPath(tn, "properties"); + WebTarget path = topicPath(tn); properties = properties == null ? new HashMap<>() : properties; return asyncPutRequest(path, Entity.entity(properties, MediaType.APPLICATION_JSON)); } @@ -345,7 +345,7 @@ public CompletableFuture createPartitionedTopicAsync( String topic, int numPartitions, boolean createLocalTopicOnly, Map properties) { checkArgument(numPartitions > 0, "Number of partitions should be more than 0"); TopicName tn = validateTopic(topic); - WebTarget path = topicPath(tn, "partitions", "properties") + WebTarget path = topicPath(tn, "partitions") .queryParam("createLocalTopicOnly", Boolean.toString(createLocalTopicOnly)); PartitionedTopicMetadata metadata = new PartitionedTopicMetadata(numPartitions, properties); return asyncPutRequest(path, Entity.entity(metadata, MediaType.APPLICATION_JSON)); From 244e71e550a09deb4fbefd978343b4630d88f312 Mon Sep 17 00:00:00 2001 From: technoboy Date: Wed, 19 Jan 2022 13:47:52 +0800 Subject: [PATCH 21/25] updates --- .../broker/admin/PersistentTopicsTest.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 8ab51329b523a..32f2504236cb1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -351,7 +351,7 @@ public void testTerminate() { String testLocalTopicName = "topic-not-found"; // 1) Create the nonPartitionTopic topic - persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, testLocalTopicName, true); + persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, testLocalTopicName, true, null); // 2) Create a subscription AsyncResponse response = mock(AsyncResponse.class); @@ -393,7 +393,7 @@ public void testNonPartitionedTopics() { Assert.assertTrue(errorCaptor.getValue().getMessage().contains("zero partitions")); final String nonPartitionTopic2 = "secondary-non-partitioned-topic"; - persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, nonPartitionTopic2, true); + persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, nonPartitionTopic2, true, null); Assert.assertEquals(persistentTopics .getPartitionedMetadata(testTenant, testNamespace, nonPartitionTopic, true, false).partitions, 0); @@ -406,7 +406,7 @@ public void testNonPartitionedTopics() { @Test public void testCreateNonPartitionedTopic() { final String topicName = "standard-topic-partition-a"; - persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, true); + persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, true, null); PartitionedTopicMetadata pMetadata = persistentTopics.getPartitionedMetadata( testTenant, testNamespace, topicName, true, false); Assert.assertEquals(pMetadata.partitions, 0); @@ -454,7 +454,7 @@ public void testCreateNonPartitionedTopicWithInvalidName() { assert(partitionedTopicname.getLocalName().equals("standard-topic")); return new PartitionedTopicMetadata(10); }).when(persistentTopics).getPartitionedTopicMetadata(any(), anyBoolean(), anyBoolean()); - persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, true); + persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, true, null); } @Test @@ -516,7 +516,7 @@ public void testUnloadTopic() { // 2) create non partitioned topic and unload response = mock(AsyncResponse.class); - persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, true); + persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, true, null); persistentTopics.unloadTopic(response, testTenant, testNamespace, topicName, true); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); @@ -578,7 +578,7 @@ public void testGetPartitionedTopicsList() throws KeeperException, InterruptedEx @Test public void testGrantNonPartitionedTopic() { final String topicName = "non-partitioned-topic"; - persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, true); + persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, true, null); String role = "role"; Set expectActions = new HashSet<>(); expectActions.add(AuthAction.produce); @@ -595,7 +595,7 @@ public void testCreateExistedPartition() { final String partitionName = TopicName.get(topicName).getPartition(0).getLocalName(); try { - persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, partitionName, false); + persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, partitionName, false, null); Assert.fail(); } catch (RestException e) { log.error("Failed to create {}: {}", partitionName, e.getMessage()); @@ -635,7 +635,7 @@ public void testGrantPartitionedTopic() { @Test public void testRevokeNonPartitionedTopic() { final String topicName = "non-partitioned-topic"; - persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, true); + persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, true, null); String role = "role"; Set expectActions = new HashSet<>(); expectActions.add(AuthAction.produce); @@ -687,7 +687,7 @@ public void testTriggerCompactionTopic() { // create non partitioned topic and compaction on it response = mock(AsyncResponse.class); - persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, nonPartitionTopicName, true); + persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, nonPartitionTopicName, true, null); persistentTopics.compact(response, testTenant, testNamespace, nonPartitionTopicName, true); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); @@ -865,7 +865,7 @@ public void testExamineMessageMetadata() throws Exception { @Test public void testOffloadWithNullMessageId() { final String topicName = "topic-123"; - persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, true); + persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, true, null); try { persistentTopics.triggerOffload(testTenant, testNamespace, topicName, true, null); @@ -1139,7 +1139,7 @@ public void testDeleteTopic() throws Exception { final String topicName = "topic-1"; BrokerService brokerService = spy(pulsar.getBrokerService()); doReturn(brokerService).when(pulsar).getBrokerService(); - persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, false); + persistentTopics.createNonPartitionedTopic(testTenant, testNamespace, topicName, false, null); CompletableFuture deleteTopicFuture = new CompletableFuture<>(); deleteTopicFuture.completeExceptionally(new MetadataStoreException.NotFoundException()); doReturn(deleteTopicFuture).when(brokerService).deleteTopic(anyString(), anyBoolean(), anyBoolean()); From ed7f06e4e303628d75cdcc05cd38cf69e81d94d9 Mon Sep 17 00:00:00 2001 From: technoboy Date: Fri, 21 Jan 2022 15:23:54 +0800 Subject: [PATCH 22/25] updates --- .../broker/admin/v1/PersistentTopics.java | 11 +-- .../broker/admin/v2/NonPersistentTopics.java | 7 +- .../broker/admin/v2/PersistentTopics.java | 8 +- .../broker/admin/v3/PersistentTopics.java | 93 +++++++++++++++++++ .../pulsar/broker/service/BrokerService.java | 27 +++--- .../apache/pulsar/broker/admin/AdminTest.java | 6 +- .../broker/admin/PersistentTopicsTest.java | 39 +++++--- 7 files changed, 149 insertions(+), 42 deletions(-) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/PersistentTopics.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java index 4f2e3e377c872..92c791da4bf96 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/PersistentTopics.java @@ -162,13 +162,11 @@ public void createPartitionedTopic( @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "The metadata for the topic", required = true, type = "PartitionedTopicMetadata") - PartitionedTopicMetadata metadata, + int numPartitions, @QueryParam("createLocalTopicOnly") @DefaultValue("false") boolean createLocalTopicOnly) { try { validateTopicName(property, cluster, namespace, encodedTopic); - internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly, - metadata.properties); + internalCreatePartitionedTopic(asyncResponse, numPartitions, createLocalTopicOnly); } catch (Exception e) { log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); resumeAsyncResponseExceptionally(asyncResponse, e); @@ -199,12 +197,11 @@ public void createNonPartitionedTopic( @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, @ApiParam(value = "Is authentication required to perform this operation") - @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, - @ApiParam(value = "Key value pair properties for the topic metadata") Map topicMetadata) { + @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateNamespaceName(tenant, cluster, namespace); validateTopicName(tenant, cluster, namespace, encodedTopic); validateGlobalNamespaceOwnership(); - internalCreateNonPartitionedTopic(authoritative, topicMetadata); + internalCreateNonPartitionedTopic(authoritative, null); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java index aaf204632336c..5eb33ab542040 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java @@ -196,14 +196,15 @@ public void createPartitionedTopic( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "The metadata for the topic", - required = true, type = "PartitionedTopicMetadata") PartitionedTopicMetadata metadata, + @ApiParam(value = "The number of partitions for the topic", + required = true, type = "int", defaultValue = "0") + int numPartitions, @QueryParam("createLocalTopicOnly") @DefaultValue("false") boolean createLocalTopicOnly) { try { validateNamespaceName(tenant, namespace); validateGlobalNamespaceOwnership(); validateTopicName(tenant, namespace, encodedTopic); - internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly); + internalCreatePartitionedTopic(asyncResponse, numPartitions, createLocalTopicOnly); } catch (Exception e) { log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); resumeAsyncResponseExceptionally(asyncResponse, e); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index af027ab46d859..4f9231429995f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -225,8 +225,9 @@ public void createPartitionedTopic( @PathParam("namespace") String namespace, @ApiParam(value = "Specify topic name", required = true) @PathParam("topic") @Encoded String encodedTopic, - @ApiParam(value = "The metadata for the topic", - required = true, type = "PartitionedTopicMetadata") PartitionedTopicMetadata metadata, + @ApiParam(value = "The number of partitions for the topic", + required = true, type = "int", defaultValue = "0") + int numPartitions, @QueryParam("createLocalTopicOnly") @DefaultValue("false") boolean createLocalTopicOnly) { try { validateNamespaceName(tenant, namespace); @@ -234,8 +235,7 @@ public void createPartitionedTopic( validatePartitionedTopicName(tenant, namespace, encodedTopic); validateTopicPolicyOperation(topicName, PolicyName.PARTITION, PolicyOperation.WRITE); validateCreateTopic(topicName); - internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly, - metadata.properties); + internalCreatePartitionedTopic(asyncResponse, numPartitions, createLocalTopicOnly); } catch (Exception e) { log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); resumeAsyncResponseExceptionally(asyncResponse, e); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/PersistentTopics.java new file mode 100644 index 0000000000000..f7960d8392cc0 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v3/PersistentTopics.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin.v3; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.Encoded; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.container.AsyncResponse; +import javax.ws.rs.container.Suspended; +import javax.ws.rs.core.MediaType; +import org.apache.pulsar.broker.admin.impl.PersistentTopicsBase; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.PolicyName; +import org.apache.pulsar.common.policies.data.PolicyOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + */ +@Path("/persistent") +@Produces(MediaType.APPLICATION_JSON) +@Api(value = "/persistent", description = "Persistent topic admin apis", tags = "persistent topic") +public class PersistentTopics extends PersistentTopicsBase { + + @PUT + @Path("/{tenant}/{namespace}/{topic}/partitions") + @ApiOperation(value = "Create a partitioned topic.", + notes = "It needs to be called before creating a producer on a partitioned topic.") + @ApiResponses(value = { + @ApiResponse(code = 307, message = "Current broker doesn't serve the namespace of this topic"), + @ApiResponse(code = 401, message = "Don't have permission to administrate resources on this tenant"), + @ApiResponse(code = 403, message = "Don't have admin permission"), + @ApiResponse(code = 404, message = "Tenant does not exist"), + @ApiResponse(code = 406, message = "The number of partitions should be more than 0 and" + + " less than or equal to maxNumPartitionsPerPartitionedTopic"), + @ApiResponse(code = 409, message = "Partitioned topic already exist"), + @ApiResponse(code = 412, + message = "Failed Reason : Name is invalid or Namespace does not have any clusters configured"), + @ApiResponse(code = 500, message = "Internal server error"), + @ApiResponse(code = 503, message = "Failed to validate global cluster configuration") + }) + public void createPartitionedTopic( + @Suspended final AsyncResponse asyncResponse, + @ApiParam(value = "Specify the tenant", required = true) + @PathParam("tenant") String tenant, + @ApiParam(value = "Specify the namespace", required = true) + @PathParam("namespace") String namespace, + @ApiParam(value = "Specify topic name", required = true) + @PathParam("topic") @Encoded String encodedTopic, + @ApiParam(value = "The metadata for the topic", + required = true, type = "PartitionedTopicMetadata") PartitionedTopicMetadata metadata, + @QueryParam("createLocalTopicOnly") @DefaultValue("false") boolean createLocalTopicOnly) { + try { + validateNamespaceName(tenant, namespace); + validateGlobalNamespaceOwnership(); + validatePartitionedTopicName(tenant, namespace, encodedTopic); + validateTopicPolicyOperation(topicName, PolicyName.PARTITION, PolicyOperation.WRITE); + validateCreateTopic(topicName); + internalCreatePartitionedTopic(asyncResponse, metadata.partitions, createLocalTopicOnly, + metadata.properties); + } catch (Exception e) { + log.error("[{}] Failed to create partitioned topic {}", clientAppId(), topicName, e); + resumeAsyncResponseExceptionally(asyncResponse, e); + } + } + + private static final Logger log = LoggerFactory.getLogger(PersistentTopics.class); +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 5e99abcbca7c1..cfe67a44196c5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -75,6 +75,7 @@ import java.util.function.Predicate; import javax.ws.rs.core.Response; import lombok.AccessLevel; +import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; import org.apache.bookkeeper.common.util.OrderedExecutor; @@ -89,8 +90,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerFactory; import org.apache.bookkeeper.mledger.util.Futures; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.ImmutableTriple; -import org.apache.commons.lang3.tuple.Triple; import org.apache.pulsar.bookie.rackawareness.IsolatedBookieEnsemblePlacementPolicy; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; @@ -215,8 +214,7 @@ public class BrokerService implements Closeable { prepareDynamicConfigurationMap(); private final ConcurrentOpenHashMap> configRegisteredListeners; - private final ConcurrentLinkedQueue>, Map>> - pendingTopicLoadingQueue; + private final ConcurrentLinkedQueue pendingTopicLoadingQueue; private AuthorizationService authorizationService = null; private final ScheduledExecutorService statsUpdater; @@ -1311,7 +1309,7 @@ protected CompletableFuture> loadOrCreatePersistentTopic(final S return null; }); } else { - pendingTopicLoadingQueue.add(new ImmutableTriple<>(topic, topicFuture, properties)); + pendingTopicLoadingQueue.add(new TopicLoadingContext(topic, topicFuture, properties)); if (log.isDebugEnabled()) { log.debug("topic-loading for {} added into pending queue", topic); } @@ -2405,18 +2403,17 @@ private ConcurrentOpenHashMap getRuntimeConfigurationMap() { * permit if it was successful to acquire it. */ private void createPendingLoadTopic() { - Triple>, Map> pendingTopic = - pendingTopicLoadingQueue.poll(); + TopicLoadingContext pendingTopic = pendingTopicLoadingQueue.poll(); if (pendingTopic == null) { return; } - final String topic = pendingTopic.getLeft(); + final String topic = pendingTopic.getTopic(); checkTopicNsOwnership(topic).thenRun(() -> { - CompletableFuture> pendingFuture = pendingTopic.getMiddle(); + CompletableFuture> pendingFuture = pendingTopic.getTopicFuture(); final Semaphore topicLoadSemaphore = topicLoadRequestSemaphore.get(); final boolean acquiredPermit = topicLoadSemaphore.tryAcquire(); - createPersistentTopic(topic, true, pendingFuture, pendingTopic.getRight()); + createPersistentTopic(topic, true, pendingFuture, pendingTopic.getProperties()); pendingFuture.handle((persistentTopic, ex) -> { // release permit and process next pending topic if (acquiredPermit) { @@ -2427,7 +2424,7 @@ private void createPendingLoadTopic() { }); }).exceptionally(e -> { log.error("Failed to create pending topic {}", topic, e); - pendingTopic.getMiddle() + pendingTopic.getTopicFuture() .completeExceptionally((e instanceof RuntimeException && e.getCause() != null) ? e.getCause() : e); // schedule to process next pending topic inactivityMonitor.schedule(this::createPendingLoadTopic, 100, TimeUnit.MILLISECONDS); @@ -2829,4 +2826,12 @@ public long getPausedConnections() { public void setPulsarChannelInitializerFactory(PulsarChannelInitializer.Factory factory) { this.pulsarChannelInitFactory = factory; } + + @AllArgsConstructor + @Getter + private static class TopicLoadingContext { + private final String topic; + private final CompletableFuture> topicFuture; + private final Map properties; + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java index 872ee7810c790..8bd17f06fa3fa 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java @@ -761,7 +761,7 @@ public void persistentTopics() throws Exception { assertEquals(persistentTopics.getPartitionedTopicList(property, cluster, namespace), Lists.newArrayList()); response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response, property, cluster, namespace, topic, new PartitionedTopicMetadata(5), false); + persistentTopics.createPartitionedTopic(response, property, cluster, namespace, topic, 5, false); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); assertEquals(persistentTopics.getPartitionedTopicList(property, cluster, namespace), Lists @@ -812,13 +812,13 @@ public void testUpdatePartitionedTopicCoontainedInOldTopic() throws Exception { AsyncResponse response1 = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response1, property, cluster, namespace, partitionedTopicName, new PartitionedTopicMetadata(5), false); + persistentTopics.createPartitionedTopic(response1, property, cluster, namespace, partitionedTopicName, 5, false); verify(response1, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); AsyncResponse response2 = mock(AsyncResponse.class); responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response2, property, cluster, namespace, partitionedTopicName2, new PartitionedTopicMetadata(2), false); + persistentTopics.createPartitionedTopic(response2, property, cluster, namespace, partitionedTopicName2, 2, false); verify(response2, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 32f2504236cb1..4be224490ff78 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -102,6 +102,7 @@ public class PersistentTopicsTest extends MockedPulsarServiceBaseTest { private PersistentTopics persistentTopics; + private org.apache.pulsar.broker.admin.v3.PersistentTopics persistentTopicsV3; private final String testTenant = "my-tenant"; private final String testLocalCluster = "use"; private final String testNamespace = "my-namespace"; @@ -123,6 +124,9 @@ protected void setup() throws Exception { persistentTopics = spy(PersistentTopics.class); persistentTopics.setServletContext(new MockServletContext()); persistentTopics.setPulsar(pulsar); + persistentTopicsV3 = spy(org.apache.pulsar.broker.admin.v3.PersistentTopics.class); + persistentTopicsV3.setServletContext(new MockServletContext()); + persistentTopicsV3.setPulsar(pulsar); doReturn(false).when(persistentTopics).isRequestHttps(); doReturn(null).when(persistentTopics).originalPrincipal(); doReturn("test").when(persistentTopics).clientAppId(); @@ -130,6 +134,13 @@ protected void setup() throws Exception { doNothing().when(persistentTopics).validateAdminAccessForTenant(this.testTenant); doReturn(mock(AuthenticationDataHttps.class)).when(persistentTopics).clientAuthData(); + doReturn(false).when(persistentTopicsV3).isRequestHttps(); + doReturn(null).when(persistentTopicsV3).originalPrincipal(); + doReturn("test").when(persistentTopicsV3).clientAppId(); + doReturn(TopicDomain.persistent.value()).when(persistentTopicsV3).domain(); + doNothing().when(persistentTopicsV3).validateAdminAccessForTenant(this.testTenant); + doReturn(mock(AuthenticationDataHttps.class)).when(persistentTopicsV3).clientAuthData(); + nonPersistentTopic = spy(NonPersistentTopics.class); nonPersistentTopic.setServletContext(new MockServletContext()); nonPersistentTopic.setPulsar(pulsar); @@ -185,7 +196,7 @@ public void testGetSubscriptions() { // 3) Create the partitioned topic response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, testLocalTopicName, new PartitionedTopicMetadata(3), true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, testLocalTopicName, 3, true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); @@ -325,7 +336,7 @@ public void testTerminatePartitionedTopic() { // 3) Create the partitioned topic AsyncResponse response = mock(AsyncResponse.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, testLocalTopicName, new PartitionedTopicMetadata(1), true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, testLocalTopicName, 1, true); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); @@ -427,7 +438,7 @@ public void testCreateNonPartitionedTopic() { public void testCreatePartitionedTopic() { AsyncResponse response = mock(AsyncResponse.class); final String topicName = "standard-partitioned-topic-a"; - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, new PartitionedTopicMetadata(2), true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2, true); Awaitility.await().untilAsserted(() -> { PartitionedTopicMetadata pMetadata = persistentTopics.getPartitionedMetadata( testTenant, testNamespace, topicName, true, false); @@ -437,7 +448,7 @@ public void testCreatePartitionedTopic() { Map topicMetadata = Maps.newHashMap(); topicMetadata.put("key1", "value1"); PartitionedTopicMetadata metadata = new PartitionedTopicMetadata(2, topicMetadata); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName2, metadata, true); + persistentTopicsV3.createPartitionedTopic(response, testTenant, testNamespace, topicName2, metadata, true); Awaitility.await().untilAsserted(() -> { PartitionedTopicMetadata pMetadata2 = persistentTopics.getPartitionedMetadata( testTenant, testNamespace, topicName2, true, false); @@ -475,7 +486,7 @@ public void testCreatePartitionedTopicHavingNonPartitionTopicWithPartitionSuffix doReturn(new Policies()).when(persistentTopics).getNamespacePolicies(any()); AsyncResponse response = mock(AsyncResponse.class); ArgumentCaptor errCaptor = ArgumentCaptor.forClass(RestException.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionedTopicName, new PartitionedTopicMetadata(5), true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionedTopicName, 5, true); verify(response, timeout(5000).times(1)).resume(errCaptor.capture()); Assert.assertEquals(errCaptor.getValue().getResponse().getStatus(), Response.Status.CONFLICT.getStatusCode()); } @@ -495,7 +506,7 @@ public void testUpdatePartitionedTopicHavingNonPartitionTopicWithPartitionSuffix doNothing().when(persistentTopics).validateAdminAccessForTenant(anyString()); AsyncResponse response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionedTopicName, new PartitionedTopicMetadata(5), true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionedTopicName, 5, true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); persistentTopics.updatePartitionedTopic(testTenant, testNamespace, partitionedTopicName, false, false, false, @@ -525,7 +536,7 @@ public void testUnloadTopic() { // 3) create partitioned topic and unload response = mock(AsyncResponse.class); responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionTopicName, new PartitionedTopicMetadata(6), true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionTopicName, 6, true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); @@ -555,13 +566,13 @@ public void testUnloadTopicShallThrowNotFoundWhenTopicNotExist() { public void testGetPartitionedTopicsList() throws KeeperException, InterruptedException, PulsarAdminException { AsyncResponse response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, "test-topic1", new PartitionedTopicMetadata(3), true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, "test-topic1", 3, true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); response = mock(AsyncResponse.class); responseCaptor = ArgumentCaptor.forClass(Response.class); - nonPersistentTopic.createPartitionedTopic(response, testTenant, testNamespace, "test-topic2", new PartitionedTopicMetadata(3), true); + nonPersistentTopic.createPartitionedTopic(response, testTenant, testNamespace, "test-topic2", 3, true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); @@ -591,7 +602,7 @@ public void testGrantNonPartitionedTopic() { public void testCreateExistedPartition() { final AsyncResponse response = mock(AsyncResponse.class); final String topicName = "test-create-existed-partition"; - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, new PartitionedTopicMetadata(3), true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 3, true); final String partitionName = TopicName.get(topicName).getPartition(0).getLocalName(); try { @@ -611,7 +622,7 @@ public void testGrantPartitionedTopic() { AsyncResponse response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); persistentTopics.createPartitionedTopic( - response, testTenant, testNamespace, partitionedTopicName, new PartitionedTopicMetadata(numPartitions), true); + response, testTenant, testNamespace, partitionedTopicName, numPartitions, true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); @@ -652,7 +663,7 @@ public void testRevokePartitionedTopic() { AsyncResponse response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); persistentTopics.createPartitionedTopic( - response, testTenant, testNamespace, partitionedTopicName, new PartitionedTopicMetadata(numPartitions), true); + response, testTenant, testNamespace, partitionedTopicName, numPartitions, true); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); String role = "role"; @@ -695,7 +706,7 @@ public void testTriggerCompactionTopic() { // create partitioned topic and compaction on it response = mock(AsyncResponse.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionTopicName, new PartitionedTopicMetadata(2), true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, partitionTopicName, 2, true); persistentTopics.compact(response, testTenant, testNamespace, partitionTopicName, true); responseCaptor = ArgumentCaptor.forClass(Response.class); verify(response, timeout(5000).times(1)).resume(responseCaptor.capture()); @@ -902,7 +913,7 @@ public void testSetReplicatedSubscriptionStatus() { // 3) Create the partitioned topic response = mock(AsyncResponse.class); ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); - persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, new PartitionedTopicMetadata(2),true); + persistentTopics.createPartitionedTopic(response, testTenant, testNamespace, topicName, 2,true); verify(response, timeout(10000).times(1)).resume(responseCaptor.capture()); Assert.assertEquals(responseCaptor.getValue().getStatus(), Response.Status.NO_CONTENT.getStatusCode()); From 8a009e16d8c60f6bcc0f7789916b08a6c47a4f07 Mon Sep 17 00:00:00 2001 From: technoboy Date: Fri, 21 Jan 2022 15:26:35 +0800 Subject: [PATCH 23/25] fix checkstyle --- .../src/test/java/org/apache/pulsar/broker/admin/AdminTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java index 8bd17f06fa3fa..1ee7a41f5ae0b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java @@ -70,7 +70,6 @@ import org.apache.pulsar.common.conf.InternalConfigurationData; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData; import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType; From 20ff27f2a3076c8a666bf41a478f6f24716544c4 Mon Sep 17 00:00:00 2001 From: technoboy Date: Fri, 21 Jan 2022 18:35:29 +0800 Subject: [PATCH 24/25] updates --- .../client/admin/internal/TopicsImpl.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java index 3249e6e4b4b3b..b27eb21ed1022 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java @@ -95,6 +95,7 @@ public class TopicsImpl extends BaseResource implements Topics { private final WebTarget adminTopics; private final WebTarget adminV2Topics; + private final WebTarget adminV3Topics; // CHECKSTYLE.OFF: MemberName private static final String BATCH_HEADER = "X-Pulsar-num-batch-message"; private static final String BATCH_SIZE_HEADER = "X-Pulsar-batch-size"; @@ -132,6 +133,7 @@ public TopicsImpl(WebTarget web, Authentication auth, long readTimeoutMs) { super(auth, readTimeoutMs); adminTopics = web.path("/admin"); adminV2Topics = web.path("/admin/v2"); + adminV3Topics = web.path("/admin/v3"); } @Override @@ -345,10 +347,16 @@ public CompletableFuture createPartitionedTopicAsync( String topic, int numPartitions, boolean createLocalTopicOnly, Map properties) { checkArgument(numPartitions > 0, "Number of partitions should be more than 0"); TopicName tn = validateTopic(topic); - WebTarget path = topicPath(tn, "partitions") + WebTarget path = topicPath(tn, properties, "partitions") .queryParam("createLocalTopicOnly", Boolean.toString(createLocalTopicOnly)); - PartitionedTopicMetadata metadata = new PartitionedTopicMetadata(numPartitions, properties); - return asyncPutRequest(path, Entity.entity(metadata, MediaType.APPLICATION_JSON)); + Entity entity; + if (properties != null) { + PartitionedTopicMetadata metadata = new PartitionedTopicMetadata(numPartitions, properties); + entity = Entity.entity(metadata, MediaType.APPLICATION_JSON); + } else { + entity = Entity.entity(numPartitions, MediaType.APPLICATION_JSON); + } + return asyncPutRequest(path, entity); } @Override @@ -1243,6 +1251,13 @@ private WebTarget namespacePath(String domain, NamespaceName namespace, String.. return namespacePath; } + private WebTarget topicPath(TopicName topic, Map metadata, String... parts) { + final WebTarget base = metadata != null ? adminV3Topics : (topic.isV2() ? adminV2Topics : adminTopics); + WebTarget topicPath = base.path(topic.getRestPath()); + topicPath = WebTargets.addParts(topicPath, parts); + return topicPath; + } + private WebTarget topicPath(TopicName topic, String... parts) { final WebTarget base = topic.isV2() ? adminV2Topics : adminTopics; WebTarget topicPath = base.path(topic.getRestPath()); From b22e555cb64c3a1096464310bbbd4893b020d448 Mon Sep 17 00:00:00 2001 From: technoboy Date: Mon, 24 Jan 2022 20:57:39 +0800 Subject: [PATCH 25/25] add comment for `topicPath` --- .../apache/pulsar/client/admin/internal/TopicsImpl.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java index b27eb21ed1022..7afbf9ae3dbe4 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java @@ -1251,6 +1251,15 @@ private WebTarget namespacePath(String domain, NamespaceName namespace, String.. return namespacePath; } + /** + * As we support topic metadata, user can add some properties when create topic. + * For compatibility, we have to define a new method, so when metadata is not null, v3 will be called. + * Details could be found here : https://github.com/apache/pulsar/pull/12818#discussion_r789340203 + * @param topic + * @param metadata + * @param parts + * @return + */ private WebTarget topicPath(TopicName topic, Map metadata, String... parts) { final WebTarget base = metadata != null ? adminV3Topics : (topic.isV2() ? adminV2Topics : adminTopics); WebTarget topicPath = base.path(topic.getRestPath());