From ccc483a6bff76728918f879b97bfa5d27d6d7242 Mon Sep 17 00:00:00 2001 From: Anonymitaet Date: Tue, 2 Feb 2021 14:58:51 +0800 Subject: [PATCH 1/6] add --- .../pulsar/admin/cli/CmdNamespaceIsolationPolicy.java | 2 +- .../org/apache/pulsar/admin/cli/CmdNamespaces.java | 2 +- site2/docs/concepts-architecture-overview.md | 10 ++++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java index 6709a3f90809e..9d324293e2d11 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java @@ -46,7 +46,7 @@ private class SetPolicy extends CliCommand { @Parameter(names = "--namespaces", description = "comma separated namespaces-regex list", required = true, splitter = CommaParameterSplitter.class) private List namespaces; - @Parameter(names = "--primary", description = "comma separated primary-broker-regex list", required = true, splitter = CommaParameterSplitter.class) + @Parameter(names = "--primary", description = "comma separated primary-broker-regex list. Before topics are assigned to brokers, you can set the namespace isolation policy with a primary or secondary regex to select desired brokers. If no broker matches the specified regex, you cannot create a topic. If the number of primary brokers are not enough, topics are assigned to secondary brokers. If the number of secondary brokers are not enough, topics are assigned to other brokers which do not have any isolation policies.", required = true, splitter = CommaParameterSplitter.class) private List primary; @Parameter(names = "--secondary", description = "comma separated secondary-broker-regex list", required = false, splitter = CommaParameterSplitter.class) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index ec63649952057..f9c1f3de8beed 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -607,7 +607,7 @@ private class SetBookieAffinityGroup extends CliCommand { "-pg" }, description = "Bookie-affinity primary-groups (comma separated) name where namespace messages should be written", required = true) private String bookieAffinityGroupNamePrimary; @Parameter(names = { "--secondary-group", - "-sg" }, description = "Bookie-affinity secondary-group (comma separated) name where namespace messages should be written", required = false) + "-sg" }, description = "Bookie-affinity secondary-group (comma separated) name where namespace messages should be written. If you want to verify whether there are enough bookies in groups, use `--secondary-group` flag. Messages in this namespace are stored in secondary groups. If a group does not contain enough bookies, a topic cannot be created.", required = false) private String bookieAffinityGroupNameSecondary; diff --git a/site2/docs/concepts-architecture-overview.md b/site2/docs/concepts-architecture-overview.md index f066db65e62d8..67ebeb0a5785f 100644 --- a/site2/docs/concepts-architecture-overview.md +++ b/site2/docs/concepts-architecture-overview.md @@ -22,7 +22,7 @@ At the broader instance level, an instance-wide ZooKeeper cluster called the con The Pulsar message broker is a stateless component that's primarily responsible for running two other components: -* An HTTP server that exposes a {@inject: rest:REST:/} API for both administrative tasks and [topic lookup](concepts-clients.md#client-setup-phase) for producers and consumers +* An HTTP server that exposes a {@inject: rest:REST:/} API for both administrative tasks and [topic lookup](concepts-clients.md#client-setup-phase) for producers and consumers. The producers connect to the brokers to publish messages and the consumers connect to the brokers to consume the messages. * A dispatcher, which is an asynchronous TCP server over a custom [binary protocol](developing-binary-protocol.md) used for all data transfers Messages are typically dispatched out of a [managed ledger](#managed-ledgers) cache for the sake of performance, *unless* the backlog exceeds the cache size. If the backlog grows too large for the cache, the broker will start reading entries from BookKeeper. @@ -45,11 +45,17 @@ Clusters can replicate amongst themselves using [geo-replication](concepts-repli ## Metadata store -Pulsar uses [Apache Zookeeper](https://zookeeper.apache.org/) for metadata storage, cluster configuration, and coordination. In a Pulsar instance: +The Pulsar metadata store maintains all the metadata of a Pulsar cluster, such as topic metadata, schema, broker load data, and so on. Pulsar uses [Apache Zookeeper](https://zookeeper.apache.org/) for metadata storage, cluster configuration, and coordination. The Pulsar metadata store can be deployed on a separate ZooKeeper cluster or deployed on an existing ZooKeeper cluster. + +In a Pulsar instance: * A configuration store quorum stores configuration for tenants, namespaces, and other entities that need to be globally consistent. * Each cluster has its own local ZooKeeper ensemble that stores cluster-specific configuration and coordination such as which brokers are responsible for which topics as well as ownership metadata, broker load reports, BookKeeper ledger metadata, and more. +## Configuration store + +The configuration store maintains all the configurations of a Pulsar instance, such as clusters, tenants, namespaces, partitioned topic related configurations, and so on. A Pulsar instance can have a single local cluster, multiple local clusters, or multiple cross-region clusters. Consequently, the configuration store can share the configurations across multiple clusters under a Pulsar instance. The configuration store can be deployed on a separate ZooKeeper cluster or deployed on an existing ZooKeeper cluster. + ## Persistent storage Pulsar provides guaranteed message delivery for applications. If a message successfully reaches a Pulsar broker, it will be delivered to its intended target. From 7466e53b862927c5ef773c98cb7799a7f17d4eb9 Mon Sep 17 00:00:00 2001 From: Yu Liu <50226895+Anonymitaet@users.noreply.github.com> Date: Tue, 2 Feb 2021 15:23:03 +0800 Subject: [PATCH 2/6] Update pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java Co-authored-by: Huanli Meng <48120384+Huanli-Meng@users.noreply.github.com> --- .../apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java index 9d324293e2d11..fc40e7c77c8e2 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java @@ -46,7 +46,7 @@ private class SetPolicy extends CliCommand { @Parameter(names = "--namespaces", description = "comma separated namespaces-regex list", required = true, splitter = CommaParameterSplitter.class) private List namespaces; - @Parameter(names = "--primary", description = "comma separated primary-broker-regex list. Before topics are assigned to brokers, you can set the namespace isolation policy with a primary or secondary regex to select desired brokers. If no broker matches the specified regex, you cannot create a topic. If the number of primary brokers are not enough, topics are assigned to secondary brokers. If the number of secondary brokers are not enough, topics are assigned to other brokers which do not have any isolation policies.", required = true, splitter = CommaParameterSplitter.class) + @Parameter(names = "--primary", description = "comma separated primary-broker-regex list. Before topics are assigned to brokers, you can set the namespace isolation policy with a primary or secondary regex to select desired brokers. If no broker matches the specified regex, you cannot create a topic. If there are not enough primary brokers, topics are assigned to secondary brokers. If there are not enough secondary brokers, topics are assigned to other brokers which do not have any isolation policies.", required = true, splitter = CommaParameterSplitter.class) private List primary; @Parameter(names = "--secondary", description = "comma separated secondary-broker-regex list", required = false, splitter = CommaParameterSplitter.class) From 2c35ec58777be1ad15d2a6be027abdaae8bd1166 Mon Sep 17 00:00:00 2001 From: Yu Liu <50226895+Anonymitaet@users.noreply.github.com> Date: Tue, 2 Feb 2021 15:24:13 +0800 Subject: [PATCH 3/6] update --- site2/docs/concepts-architecture-overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site2/docs/concepts-architecture-overview.md b/site2/docs/concepts-architecture-overview.md index 67ebeb0a5785f..e771d26919141 100644 --- a/site2/docs/concepts-architecture-overview.md +++ b/site2/docs/concepts-architecture-overview.md @@ -45,7 +45,7 @@ Clusters can replicate amongst themselves using [geo-replication](concepts-repli ## Metadata store -The Pulsar metadata store maintains all the metadata of a Pulsar cluster, such as topic metadata, schema, broker load data, and so on. Pulsar uses [Apache Zookeeper](https://zookeeper.apache.org/) for metadata storage, cluster configuration, and coordination. The Pulsar metadata store can be deployed on a separate ZooKeeper cluster or deployed on an existing ZooKeeper cluster. +The Pulsar metadata store maintains all the metadata of a Pulsar cluster, such as topic metadata, schema, broker load data, and so on. Pulsar uses [Apache ZooKeeper](https://zookeeper.apache.org/) for metadata storage, cluster configuration, and coordination. The Pulsar metadata store can be deployed on a separate ZooKeeper cluster or deployed on an existing ZooKeeper cluster. In a Pulsar instance: From cdd9f6da663099b5e87d464ca288ff34f01de4c8 Mon Sep 17 00:00:00 2001 From: Yu Liu <50226895+Anonymitaet@users.noreply.github.com> Date: Wed, 3 Feb 2021 08:41:35 +0800 Subject: [PATCH 4/6] update --- site2/docs/concepts-architecture-overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site2/docs/concepts-architecture-overview.md b/site2/docs/concepts-architecture-overview.md index e771d26919141..fad4847a72dee 100644 --- a/site2/docs/concepts-architecture-overview.md +++ b/site2/docs/concepts-architecture-overview.md @@ -45,7 +45,7 @@ Clusters can replicate amongst themselves using [geo-replication](concepts-repli ## Metadata store -The Pulsar metadata store maintains all the metadata of a Pulsar cluster, such as topic metadata, schema, broker load data, and so on. Pulsar uses [Apache ZooKeeper](https://zookeeper.apache.org/) for metadata storage, cluster configuration, and coordination. The Pulsar metadata store can be deployed on a separate ZooKeeper cluster or deployed on an existing ZooKeeper cluster. +The Pulsar metadata store maintains all the metadata of a Pulsar cluster, such as topic metadata, schema, broker load data, and so on. Pulsar uses [Apache ZooKeeper](https://zookeeper.apache.org/) for metadata storage, cluster configuration, and coordination. The Pulsar metadata store can be deployed on a separate ZooKeeper cluster or deployed on an existing ZooKeeper cluster. You can use one ZooKeeper cluster for both Pulsar metadata store and [BookKeeper metadata store](https://bookkeeper.apache.org/docs/latest/getting-started/concepts/#metadata-storage). If you want to deploy Pulsar brokers connected to an existing BookKeeper cluster, you need to deploy separate ZooKeeper clusters for Pulsar metadata store and BookKeeper metadata store respectively. In a Pulsar instance: From d48fd0a83c3ece77ee58b2c62c7f23e6d6a6c7ef Mon Sep 17 00:00:00 2001 From: Yu Liu <50226895+Anonymitaet@users.noreply.github.com> Date: Wed, 3 Feb 2021 14:14:47 +0800 Subject: [PATCH 5/6] update --- .../apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java index fc40e7c77c8e2..1abdb48076a3d 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java @@ -46,7 +46,7 @@ private class SetPolicy extends CliCommand { @Parameter(names = "--namespaces", description = "comma separated namespaces-regex list", required = true, splitter = CommaParameterSplitter.class) private List namespaces; - @Parameter(names = "--primary", description = "comma separated primary-broker-regex list. Before topics are assigned to brokers, you can set the namespace isolation policy with a primary or secondary regex to select desired brokers. If no broker matches the specified regex, you cannot create a topic. If there are not enough primary brokers, topics are assigned to secondary brokers. If there are not enough secondary brokers, topics are assigned to other brokers which do not have any isolation policies.", required = true, splitter = CommaParameterSplitter.class) + @Parameter(names = "--primary", description = "comma separated primary-broker-regex list. In Pulsar, when namespaces (more specifically, namespace bundles) are assigned dynamically to brokers, the namespace isolation policy limits the set of brokers that can be used for assignment. Before topics are assigned to brokers, you can set the namespace isolation policy with a primary or secondary regex to select desired brokers. If no broker matches the specified regex, you cannot create a topic. If there are not enough primary brokers, topics are assigned to secondary brokers. If there are not enough secondary brokers, topics are assigned to other brokers which do not have any isolation policies.", required = true, splitter = CommaParameterSplitter.class) private List primary; @Parameter(names = "--secondary", description = "comma separated secondary-broker-regex list", required = false, splitter = CommaParameterSplitter.class) From 328a1f957e5a211a194bfbdddff52991d08f66c1 Mon Sep 17 00:00:00 2001 From: Anonymitaet Date: Thu, 4 Feb 2021 14:02:53 +0800 Subject: [PATCH 6/6] test2 --- .../apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java index 1abdb48076a3d..3c8c084df8ce2 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaceIsolationPolicy.java @@ -46,7 +46,7 @@ private class SetPolicy extends CliCommand { @Parameter(names = "--namespaces", description = "comma separated namespaces-regex list", required = true, splitter = CommaParameterSplitter.class) private List namespaces; - @Parameter(names = "--primary", description = "comma separated primary-broker-regex list. In Pulsar, when namespaces (more specifically, namespace bundles) are assigned dynamically to brokers, the namespace isolation policy limits the set of brokers that can be used for assignment. Before topics are assigned to brokers, you can set the namespace isolation policy with a primary or secondary regex to select desired brokers. If no broker matches the specified regex, you cannot create a topic. If there are not enough primary brokers, topics are assigned to secondary brokers. If there are not enough secondary brokers, topics are assigned to other brokers which do not have any isolation policies.", required = true, splitter = CommaParameterSplitter.class) + @Parameter(names = "--primary", description = "comma separated primary-broker-regex list. In Pulsar, when namespaces (more specifically, namespace bundles) are assigned dynamically to brokers, the namespace isolation policy limits the set of brokers that can be used for assignment. Before topics are assigned to brokers, you can set the namespace isolation policy with a primary or a secondary regex to select desired brokers. If no broker matches the specified regex, you cannot create a topic. If there are not enough primary brokers, topics are assigned to secondary brokers. If there are not enough secondary brokers, topics are assigned to other brokers which do not have any isolation policies.", required = true, splitter = CommaParameterSplitter.class) private List primary; @Parameter(names = "--secondary", description = "comma separated secondary-broker-regex list", required = false, splitter = CommaParameterSplitter.class)