From 066b6926546ea00af05020106165ad2d454c3ec7 Mon Sep 17 00:00:00 2001 From: LiLi Date: Tue, 2 Nov 2021 10:03:12 +0800 Subject: [PATCH 1/5] [website][upgrade]feat: docs migration - 2.7.1 / io Signed-off-by: LiLi --- .../versioned_docs/version-2.7.1/io-cdc.md | 30 + .../versioned_docs/version-2.7.1/io-cli.md | 662 ++++++ .../version-2.7.1/io-connectors.md | 236 ++ .../versioned_docs/version-2.7.1/io-debug.md | 411 ++++ .../version-2.7.1/io-develop.md | 267 +++ .../version-2.7.1/io-overview.md | 176 ++ .../version-2.7.1/io-quickstart.md | 968 ++++++++ .../versioned_docs/version-2.7.1/io-use.md | 1981 +++++++++++++++++ .../version-2.7.1-sidebars.json | 40 +- 9 files changed, 4770 insertions(+), 1 deletion(-) create mode 100644 site2/website-next/versioned_docs/version-2.7.1/io-cdc.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/io-cli.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/io-connectors.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/io-debug.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/io-develop.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/io-overview.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/io-quickstart.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/io-use.md diff --git a/site2/website-next/versioned_docs/version-2.7.1/io-cdc.md b/site2/website-next/versioned_docs/version-2.7.1/io-cdc.md new file mode 100644 index 0000000000000..9ce8f7fc39101 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/io-cdc.md @@ -0,0 +1,30 @@ +--- +id: io-cdc +title: CDC connector +sidebar_label: "CDC connector" +original_id: io-cdc +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +CDC source connectors capture log changes of databases (such as MySQL, MongoDB, and PostgreSQL) into Pulsar. + +> CDC source connectors are built on top of [Canal](https://github.com/alibaba/canal) and [Debezium](https://debezium.io/) and store all data into Pulsar cluster in a persistent, replicated, and partitioned way. + +Currently, Pulsar has the following CDC connectors. + +Name|Java Class +|---|--- +[Canal source connector](io-canal-source)|[org.apache.pulsar.io.canal.CanalStringSource.java](https://github.com/apache/pulsar/blob/master/pulsar-io/canal/src/main/java/org/apache/pulsar/io/canal/CanalStringSource.java) +[Debezium source connector](io-cdc-debezium)|
  • [org.apache.pulsar.io.debezium.DebeziumSource.java](https://github.com/apache/pulsar/blob/master/pulsar-io/debezium/core/src/main/java/org/apache/pulsar/io/debezium/DebeziumSource.java)
  • [org.apache.pulsar.io.debezium.mysql.DebeziumMysqlSource.java](https://github.com/apache/pulsar/blob/master/pulsar-io/debezium/mysql/src/main/java/org/apache/pulsar/io/debezium/mysql/DebeziumMysqlSource.java)
  • [org.apache.pulsar.io.debezium.postgres.DebeziumPostgresSource.java](https://github.com/apache/pulsar/blob/master/pulsar-io/debezium/postgres/src/main/java/org/apache/pulsar/io/debezium/postgres/DebeziumPostgresSource.java)
    • + +For more information about Canal and Debezium, see the information below. + +Subject | Reference +|---|--- +How to use Canal source connector with MySQL|[Canal guide](https://github.com/alibaba/canal/wiki) +How does Canal work | [Canal tutorial](https://github.com/alibaba/canal/wiki) +How to use Debezium source connector with MySQL | [Debezium guide](https://debezium.io/docs/connectors/mysql/) +How does Debezium work | [Debezium tutorial](https://debezium.io/docs/tutorial/) diff --git a/site2/website-next/versioned_docs/version-2.7.1/io-cli.md b/site2/website-next/versioned_docs/version-2.7.1/io-cli.md new file mode 100644 index 0000000000000..886affed7bfcb --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/io-cli.md @@ -0,0 +1,662 @@ +--- +id: io-cli +title: Connector Admin CLI +sidebar_label: "CLI" +original_id: io-cli +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +The `pulsar-admin` tool helps you manage Pulsar connectors. + +## `sources` + +An interface for managing Pulsar IO sources (ingress data into Pulsar). + +```bash + +$ pulsar-admin sources subcommands + +``` + +Subcommands are: + +* `create` + +* `update` + +* `delete` + +* `get` + +* `status` + +* `list` + +* `stop` + +* `start` + +* `restart` + +* `localrun` + +* `available-sources` + +* `reload` + + +### `create` + +Submit a Pulsar IO source connector to run in a Pulsar cluster. + +#### Usage + +```bash + +$ pulsar-admin sources create options + +``` + +#### Options + +|Flag|Description| +|----|---| +| `-a`, `--archive` | The path to the NAR archive for the source.
    It also supports url-path (http/https/file [file protocol assumes that file already exists on worker host]) from which worker can download the package. +| `--classname` | The source's class name if `archive` is file-url-path (file://). +| `--cpu` | The CPU (in cores) that needs to be allocated per source instance (applicable only to Docker runtime). +| `--deserialization-classname` | The SerDe classname for the source. +| `--destination-topic-name` | The Pulsar topic to which data is sent. +| `--disk` | The disk (in bytes) that needs to be allocated per source instance (applicable only to Docker runtime). +|`--name` | The source's name. +| `--namespace` | The source's namespace. +| ` --parallelism` | The source's parallelism factor, that is, the number of source instances to run. +| `--processing-guarantees` | The processing guarantees (also named as delivery semantics) applied to the source. A source connector receives messages from external system and writes messages to a Pulsar topic. The `--processing-guarantees` is used to ensure the processing guarantees for writing messages to the Pulsar topic.
    The available values are ATLEAST_ONCE, ATMOST_ONCE, EFFECTIVELY_ONCE. +| `--ram` | The RAM (in bytes) that needs to be allocated per source instance (applicable only to the process and Docker runtimes). +| `-st`, `--schema-type` | The schema type.
    Either a builtin schema (for example, AVRO and JSON) or custom schema class name to be used to encode messages emitted from source. +| `--source-config` | Source config key/values. +| `--source-config-file` | The path to a YAML config file specifying the source's configuration. +| `-t`, `--source-type` | The source's connector provider. +| `--tenant` | The source's tenant. +|`--producer-config`| The custom producer configuration (as a JSON string). + +### `update` + +Update a already submitted Pulsar IO source connector. + +#### Usage + +```bash + +$ pulsar-admin sources update options + +``` + +#### Options + +|Flag|Description| +|----|---| +| `-a`, `--archive` | The path to the NAR archive for the source.
    It also supports url-path (http/https/file [file protocol assumes that file already exists on worker host]) from which worker can download the package. +| `--classname` | The source's class name if `archive` is file-url-path (file://). +| `--cpu` | The CPU (in cores) that needs to be allocated per source instance (applicable only to Docker runtime). +| `--deserialization-classname` | The SerDe classname for the source. +| `--destination-topic-name` | The Pulsar topic to which data is sent. +| `--disk` | The disk (in bytes) that needs to be allocated per source instance (applicable only to Docker runtime). +|`--name` | The source's name. +| `--namespace` | The source's namespace. +| ` --parallelism` | The source's parallelism factor, that is, the number of source instances to run. +| `--processing-guarantees` | The processing guarantees (also named as delivery semantics) applied to the source. A source connector receives messages from external system and writes messages to a Pulsar topic. The `--processing-guarantees` is used to ensure the processing guarantees for writing messages to the Pulsar topic.
    The available values are ATLEAST_ONCE, ATMOST_ONCE, EFFECTIVELY_ONCE. +| `--ram` | The RAM (in bytes) that needs to be allocated per source instance (applicable only to the process and Docker runtimes). +| `-st`, `--schema-type` | The schema type.
    Either a builtin schema (for example, AVRO and JSON) or custom schema class name to be used to encode messages emitted from source. +| `--source-config` | Source config key/values. +| `--source-config-file` | The path to a YAML config file specifying the source's configuration. +| `-t`, `--source-type` | The source's connector provider. The `source-type` parameter of the currently built-in connectors is determined by the setting of the `name` parameter specified in the pulsar-io.yaml file. +| `--tenant` | The source's tenant. +| `--update-auth-data` | Whether or not to update the auth data.
    **Default value: false.** + + +### `delete` + +Delete a Pulsar IO source connector. + +#### Usage + +```bash + +$ pulsar-admin sources delete options + +``` + +#### Option + +|Flag|Description| +|---|---| +|`--name`|The source's name.| +|`--namespace`|The source's namespace.| +|`--tenant`|The source's tenant.| + +### `get` + +Get the information about a Pulsar IO source connector. + +#### Usage + +```bash + +$ pulsar-admin sources get options + +``` + +#### Options +|Flag|Description| +|---|---| +|`--name`|The source's name.| +|`--namespace`|The source's namespace.| +|`--tenant`|The source's tenant.| + + +### `status` + +Check the current status of a Pulsar Source. + +#### Usage + +```bash + +$ pulsar-admin sources status options + +``` + +#### Options + +|Flag|Description| +|---|---| +|`--instance-id`|The source ID.
    If `instance-id` is not provided, Pulasr gets status of all instances.| +|`--name`|The source's name.| +|`--namespace`|The source's namespace.| +|`--tenant`|The source's tenant.| + +### `list` + +List all running Pulsar IO source connectors. + +#### Usage + +```bash + +$ pulsar-admin sources list options + +``` + +#### Options + +|Flag|Description| +|---|---| +|`--namespace`|The source's namespace.| +|`--tenant`|The source's tenant.| + + +### `stop` + +Stop a source instance. + +#### Usage + +```bash + +$ pulsar-admin sources stop options + +``` + +#### Options + +|Flag|Description| +|---|---| +|`--instance-id`|The source instanceID.
    If `instance-id` is not provided, Pulsar stops all instances.| +|`--name`|The source's name.| +|`--namespace`|The source's namespace.| +|`--tenant`|The source's tenant.| + +### `start` + +Start a source instance. + +#### Usage + +```bash + +$ pulsar-admin sources start options + +``` + +#### Options + +|Flag|Description| +|---|---| +|`--instance-id`|The source instanceID.
    If `instance-id` is not provided, Pulsar starts all instances.| +|`--name`|The source's name.| +|`--namespace`|The source's namespace.| +|`--tenant`|The source's tenant.| + + +### `restart` + +Restart a source instance. + +#### Usage + +```bash + +$ pulsar-admin sources restart options + +``` + +#### Options +|Flag|Description| +|---|---| +|`--instance-id`|The source instanceID.
    If `instance-id` is not provided, Pulsar restarts all instances. +|`--name`|The source's name.| +|`--namespace`|The source's namespace.| +|`--tenant`|The source's tenant.| + + +### `localrun` + +Run a Pulsar IO source connector locally rather than deploying it to the Pulsar cluster. + +#### Usage + +```bash + +$ pulsar-admin sources localrun options + +``` + +#### Options + +|Flag|Description| +|----|---| +| `-a`, `--archive` | The path to the NAR archive for the Source.
    It also supports url-path (http/https/file [file protocol assumes that file already exists on worker host]) from which worker can download the package. +| `--broker-service-url` | The URL for the Pulsar broker. +|`--classname`|The source's class name if `archive` is file-url-path (file://). +| `--client-auth-params` | Client authentication parameter. +| `--client-auth-plugin` | Client authentication plugin using which function-process can connect to broker. +|`--cpu`|The CPU (in cores) that needs to be allocated per source instance (applicable only to the Docker runtime).| +|`--deserialization-classname`|The SerDe classname for the source. +|`--destination-topic-name`|The Pulsar topic to which data is sent. +|`--disk`|The disk (in bytes) that needs to be allocated per source instance (applicable only to the Docker runtime).| +|`--hostname-verification-enabled`|Enable hostname verification.
    **Default value: false**. +|`--name`|The source’s name.| +|`--namespace`|The source’s namespace.| +|`--parallelism`|The source’s parallelism factor, that is, the number of source instances to run).| +|`--processing-guarantees` | The processing guarantees (also named as delivery semantics) applied to the source. A source connector receives messages from external system and writes messages to a Pulsar topic. The `--processing-guarantees` is used to ensure the processing guarantees for writing messages to the Pulsar topic.
    The available values are ATLEAST_ONCE, ATMOST_ONCE, EFFECTIVELY_ONCE. +|`--ram`|The RAM (in bytes) that needs to be allocated per source instance (applicable only to the Docker runtime).| +| `-st`, `--schema-type` | The schema type.
    Either a builtin schema (for example, AVRO and JSON) or custom schema class name to be used to encode messages emitted from source. +|`--source-config`|Source config key/values. +|`--source-config-file`|The path to a YAML config file specifying the source’s configuration. +|`--source-type`|The source's connector provider. +|`--tenant`|The source’s tenant. +|`--tls-allow-insecure`|Allow insecure tls connection.
    **Default value: false**. +|`--tls-trust-cert-path`|The tls trust cert file path. +|`--use-tls`|Use tls connection.
    **Default value: false**. +|`--producer-config`| The custom producer configuration (as a JSON string). + +### `available-sources` + +Get the list of Pulsar IO connector sources supported by Pulsar cluster. + +#### Usage + +```bash + +$ pulsar-admin sources available-sources + +``` + +### `reload` + +Reload the available built-in connectors. + +#### Usage + +```bash + +$ pulsar-admin sources reload + +``` + +## `sinks` + +An interface for managing Pulsar IO sinks (egress data from Pulsar). + +```bash + +$ pulsar-admin sinks subcommands + +``` + +Subcommands are: + +* `create` + +* `update` + +* `delete` + +* `get` + +* `status` + +* `list` + +* `stop` + +* `start` + +* `restart` + +* `localrun` + +* `available-sinks` + +* `reload` + + +### `create` + +Submit a Pulsar IO sink connector to run in a Pulsar cluster. + +#### Usage + +```bash + +$ pulsar-admin sinks create options + +``` + +#### Options + +|Flag|Description| +|----|---| +| `-a`, `--archive` | The path to the archive file for the sink.
    It also supports url-path (http/https/file [file protocol assumes that file already exists on worker host]) from which worker can download the package. +| `--auto-ack` | Whether or not the framework will automatically acknowledge messages. +| `--classname` | The sink's class name if `archive` is file-url-path (file://). +| `--cpu` | The CPU (in cores) that needs to be allocated per sink instance (applicable only to Docker runtime). +| `--custom-schema-inputs` | The map of input topics to schema types or class names (as a JSON string). +| `--custom-serde-inputs` | The map of input topics to SerDe class names (as a JSON string). +| `--disk` | The disk (in bytes) that needs to be allocated per sink instance (applicable only to Docker runtime). +|`-i, --inputs` | The sink's input topic or topics (multiple topics can be specified as a comma-separated list). +|`--name` | The sink's name. +| `--namespace` | The sink's namespace. +| ` --parallelism` | The sink's parallelism factor, that is, the number of sink instances to run. +| `--processing-guarantees` | The processing guarantees (also known as delivery semantics) applied to the sink. The `--processing-guarantees` implementation in Pulsar also relies on sink implementation.
    The available values are ATLEAST_ONCE, ATMOST_ONCE, EFFECTIVELY_ONCE. +| `--ram` | The RAM (in bytes) that needs to be allocated per sink instance (applicable only to the process and Docker runtimes). +| `--retain-ordering` | Sink consumes and sinks messages in order. +| `--sink-config` | sink config key/values. +| `--sink-config-file` | The path to a YAML config file specifying the sink's configuration. +| `-t`, `--sink-type` | The sink's connector provider. The `sink-type` parameter of the currently built-in connectors is determined by the setting of the `name` parameter specified in the pulsar-io.yaml file. +| `--subs-name` | Pulsar source subscription name if user wants a specific subscription-name for input-topic consumer. +| `--tenant` | The sink's tenant. +| `--timeout-ms` | The message timeout in milliseconds. +| `--topics-pattern` | TopicsPattern to consume from list of topics under a namespace that match the pattern.
    `--input` and `--topics-Pattern` are mutually exclusive.
    Add SerDe class name for a pattern in `--customSerdeInputs` (supported for java fun only). + +### `update` + +Update a Pulsar IO sink connector. + +#### Usage + +```bash + +$ pulsar-admin sinks update options + +``` + +#### Options + +|Flag|Description| +|----|---| +| `-a`, `--archive` | The path to the archive file for the sink.
    It also supports url-path (http/https/file [file protocol assumes that file already exists on worker host]) from which worker can download the package. +| `--auto-ack` | Whether or not the framework will automatically acknowledge messages. +| `--classname` | The sink's class name if `archive` is file-url-path (file://). +| `--cpu` | The CPU (in cores) that needs to be allocated per sink instance (applicable only to Docker runtime). +| `--custom-schema-inputs` | The map of input topics to schema types or class names (as a JSON string). +| `--custom-serde-inputs` | The map of input topics to SerDe class names (as a JSON string). +| `--disk` | The disk (in bytes) that needs to be allocated per sink instance (applicable only to Docker runtime). +|`-i, --inputs` | The sink's input topic or topics (multiple topics can be specified as a comma-separated list). +|`--name` | The sink's name. +| `--namespace` | The sink's namespace. +| ` --parallelism` | The sink's parallelism factor, that is, the number of sink instances to run. +| `--processing-guarantees` | The processing guarantees (also known as delivery semantics) applied to the sink. The `--processing-guarantees` implementation in Pulsar also relies on sink implementation.
    The available values are ATLEAST_ONCE, ATMOST_ONCE, EFFECTIVELY_ONCE. +| `--ram` | The RAM (in bytes) that needs to be allocated per sink instance (applicable only to the process and Docker runtimes). +| `--retain-ordering` | Sink consumes and sinks messages in order. +| `--sink-config` | sink config key/values. +| `--sink-config-file` | The path to a YAML config file specifying the sink's configuration. +| `-t`, `--sink-type` | The sink's connector provider. +| `--subs-name` | Pulsar source subscription name if user wants a specific subscription-name for input-topic consumer. +| `--tenant` | The sink's tenant. +| `--timeout-ms` | The message timeout in milliseconds. +| `--topics-pattern` | TopicsPattern to consume from list of topics under a namespace that match the pattern.
    `--input` and `--topics-Pattern` are mutually exclusive.
    Add SerDe class name for a pattern in `--customSerdeInputs` (supported for java fun only). +| `--update-auth-data` | Whether or not to update the auth data.
    **Default value: false.** + +### `delete` + +Delete a Pulsar IO sink connector. + +#### Usage + +```bash + +$ pulsar-admin sinks delete options + +``` + +#### Option + +|Flag|Description| +|---|---| +|`--name`|The sink's name.| +|`--namespace`|The sink's namespace.| +|`--tenant`|The sink's tenant.| + +### `get` + +Get the information about a Pulsar IO sink connector. + +#### Usage + +```bash + +$ pulsar-admin sinks get options + +``` + +#### Options +|Flag|Description| +|---|---| +|`--name`|The sink's name.| +|`--namespace`|The sink's namespace.| +|`--tenant`|The sink's tenant.| + + +### `status` + +Check the current status of a Pulsar sink. + +#### Usage + +```bash + +$ pulsar-admin sinks status options + +``` + +#### Options + +|Flag|Description| +|---|---| +|`--instance-id`|The sink ID.
    If `instance-id` is not provided, Pulasr gets status of all instances.| +|`--name`|The sink's name.| +|`--namespace`|The sink's namespace.| +|`--tenant`|The sink's tenant.| + + +### `list` + +List all running Pulsar IO sink connectors. + +#### Usage + +```bash + +$ pulsar-admin sinks list options + +``` + +#### Options + +|Flag|Description| +|---|---| +|`--namespace`|The sink's namespace.| +|`--tenant`|The sink's tenant.| + + +### `stop` + +Stop a sink instance. + +#### Usage + +```bash + +$ pulsar-admin sinks stop options + +``` + +#### Options + +|Flag|Description| +|---|---| +|`--instance-id`|The sink instanceID.
    If `instance-id` is not provided, Pulsar stops all instances.| +|`--name`|The sink's name.| +|`--namespace`|The sink's namespace.| +|`--tenant`|The sink's tenant.| + +### `start` + +Start a sink instance. + +#### Usage + +```bash + +$ pulsar-admin sinks start options + +``` + +#### Options + +|Flag|Description| +|---|---| +|`--instance-id`|The sink instanceID.
    If `instance-id` is not provided, Pulsar starts all instances.| +|`--name`|The sink's name.| +|`--namespace`|The sink's namespace.| +|`--tenant`|The sink's tenant.| + + +### `restart` + +Restart a sink instance. + +#### Usage + +```bash + +$ pulsar-admin sinks restart options + +``` + +#### Options + +|Flag|Description| +|---|---| +|`--instance-id`|The sink instanceID.
    If `instance-id` is not provided, Pulsar restarts all instances. +|`--name`|The sink's name.| +|`--namespace`|The sink's namespace.| +|`--tenant`|The sink's tenant.| + + +### `localrun` + +Run a Pulsar IO sink connector locally rather than deploying it to the Pulsar cluster. + +#### Usage + +```bash + +$ pulsar-admin sinks localrun options + +``` + +#### Options + +|Flag|Description| +|----|---| +| `-a`, `--archive` | The path to the archive file for the sink.
    It also supports url-path (http/https/file [file protocol assumes that file already exists on worker host]) from which worker can download the package. +| `--auto-ack` | Whether or not the framework will automatically acknowledge messages. +| `--broker-service-url` | The URL for the Pulsar broker. +|`--classname`|The sink's class name if `archive` is file-url-path (file://). +| `--client-auth-params` | Client authentication parameter. +| `--client-auth-plugin` | Client authentication plugin using which function-process can connect to broker. +|`--cpu`|The CPU (in cores) that needs to be allocated per sink instance (applicable only to the Docker runtime). +| `--custom-schema-inputs` | The map of input topics to Schema types or class names (as a JSON string). +| `--max-redeliver-count` | Maximum number of times that a message is redelivered before being sent to the dead letter queue. +| `--dead-letter-topic` | Name of the dead letter topic where the failing messages are sent. +| `--custom-serde-inputs` | The map of input topics to SerDe class names (as a JSON string). +|`--disk`|The disk (in bytes) that needs to be allocated per sink instance (applicable only to the Docker runtime).| +|`--hostname-verification-enabled`|Enable hostname verification.
    **Default value: false**. +| `-i`, `--inputs` | The sink's input topic or topics (multiple topics can be specified as a comma-separated list). +|`--name`|The sink’s name.| +|`--namespace`|The sink’s namespace.| +|`--parallelism`|The sink’s parallelism factor, that is, the number of sink instances to run).| +|`--processing-guarantees`|The processing guarantees (also known as delivery semantics) applied to the sink. The `--processing-guarantees` implementation in Pulsar also relies on sink implementation.
    The available values are ATLEAST_ONCE, ATMOST_ONCE, EFFECTIVELY_ONCE. +|`--ram`|The RAM (in bytes) that needs to be allocated per sink instance (applicable only to the Docker runtime).| +|`--retain-ordering` | Sink consumes and sinks messages in order. +|`--sink-config`|sink config key/values. +|`--sink-config-file`|The path to a YAML config file specifying the sink’s configuration. +|`--sink-type`|The sink's connector provider. +|`--subs-name` | Pulsar source subscription name if user wants a specific subscription-name for input-topic consumer. +|`--tenant`|The sink’s tenant. +| `--timeout-ms` | The message timeout in milliseconds. +| `--negative-ack-redelivery-delay-ms` | The negatively-acknowledged message redelivery delay in milliseconds. | +|`--tls-allow-insecure`|Allow insecure tls connection.
    **Default value: false**. +|`--tls-trust-cert-path`|The tls trust cert file path. +| `--topics-pattern` | TopicsPattern to consume from list of topics under a namespace that match the pattern.
    `--input` and `--topics-Pattern` are mutually exclusive.
    Add SerDe class name for a pattern in `--customSerdeInputs` (supported for java fun only). +|`--use-tls`|Use tls connection.
    **Default value: false**. + +### `available-sinks` + +Get the list of Pulsar IO connector sinks supported by Pulsar cluster. + +#### Usage + +```bash + +$ pulsar-admin sinks available-sinks + +``` + +### `reload` + +Reload the available built-in connectors. + +#### Usage + +```bash + +$ pulsar-admin sinks reload + +``` + diff --git a/site2/website-next/versioned_docs/version-2.7.1/io-connectors.md b/site2/website-next/versioned_docs/version-2.7.1/io-connectors.md new file mode 100644 index 0000000000000..3e0924a7f3c91 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/io-connectors.md @@ -0,0 +1,236 @@ +--- +id: io-connectors +title: Built-in connector +sidebar_label: "Built-in connector" +original_id: io-connectors +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +Pulsar distribution includes a set of common connectors that have been packaged and tested with the rest of Apache Pulsar. These connectors import and export data from some of the most commonly used data systems. + +Using any of these connectors is as easy as writing a simple connector and running the connector locally or submitting the connector to a Pulsar Functions cluster. + +## Source connector + +Pulsar has various source connectors, which are sorted alphabetically as below. + +### Canal + +* [Configuration](io-canal-source.md#configuration) + +* [Example](io-canal-source.md#usage) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/canal/src/main/java/org/apache/pulsar/io/canal/CanalStringSource.java) + + +### Debezium MySQL + +* [Configuration](io-debezium-source.md#configuration) + +* [Example](io-debezium-source.md#example-of-mysql) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/debezium/mysql/src/main/java/org/apache/pulsar/io/debezium/mysql/DebeziumMysqlSource.java) + +### Debezium PostgreSQL + +* [Configuration](io-debezium-source.md#configuration) + +* [Example](io-debezium-source.md#example-of-postgresql) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/debezium/postgres/src/main/java/org/apache/pulsar/io/debezium/postgres/DebeziumPostgresSource.java) + +### Debezium MongoDB + +* [Configuration](io-debezium-source.md#configuration) + +* [Example](io-debezium-source.md#example-of-mongodb) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/debezium/mongodb/src/main/java/org/apache/pulsar/io/debezium/mongodb/DebeziumMongoDbSource.java) + +### DynamoDB + +* [Configuration](io-dynamodb-source.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/dynamodb/src/main/java/org/apache/pulsar/io/dynamodb/DynamoDBSource.java) + +### File + +* [Configuration](io-file-source.md#configuration) + +* [Example](io-file-source.md#usage) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/file/src/main/java/org/apache/pulsar/io/file/FileSource.java) + +### Flume + +* [Configuration](io-flume-source.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/flume/src/main/java/org/apache/pulsar/io/flume/FlumeConnector.java) + +### Twitter firehose + +* [Configuration](io-twitter-source.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/twitter/src/main/java/org/apache/pulsar/io/twitter/TwitterFireHose.java) + +### Kafka + +* [Configuration](io-kafka-source.md#configuration) + +* [Example](io-kafka-source.md#usage) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java) + +### Kinesis + +* [Configuration](io-kinesis-source.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSource.java) + +### Netty + +* [Configuration](io-netty-source.md#configuration) + +* [Example of TCP](io-netty-source.md#tcp) + +* [Example of HTTP](io-netty-source.md#http) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/netty/src/main/java/org/apache/pulsar/io/netty/NettySource.java) + +### NSQ + +* [Configuration](io-nsq-source.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/nsq/src/main/java/org/apache/pulsar/io/nsq/NSQSource.java) + +### RabbitMQ + +* [Configuration](io-rabbitmq-source.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSource.java) + +## Sink connector + +Pulsar has various sink connectors, which are sorted alphabetically as below. + +### Aerospike + +* [Configuration](io-aerospike-sink.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/aerospike/src/main/java/org/apache/pulsar/io/aerospike/AerospikeStringSink.java) + +### Cassandra + +* [Configuration](io-cassandra-sink.md#configuration) + +* [Example](io-cassandra-sink.md#usage) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/cassandra/src/main/java/org/apache/pulsar/io/cassandra/CassandraStringSink.java) + +### ElasticSearch + +* [Configuration](io-elasticsearch-sink.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSink.java) + +### Flume + +* [Configuration](io-flume-sink.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/flume/src/main/java/org/apache/pulsar/io/flume/sink/StringSink.java) + +### HBase + +* [Configuration](io-hbase-sink.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/hbase/src/main/java/org/apache/pulsar/io/hbase/HbaseAbstractConfig.java) + +### HDFS2 + +* [Configuration](io-hdfs2-sink.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/hdfs2/src/main/java/org/apache/pulsar/io/hdfs2/AbstractHdfsConnector.java) + +### HDFS3 + +* [Configuration](io-hdfs3-sink.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/hdfs3/src/main/java/org/apache/pulsar/io/hdfs3/AbstractHdfsConnector.java) + +### InfluxDB + +* [Configuration](io-influxdb-sink.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/influxdb/src/main/java/org/apache/pulsar/io/influxdb/InfluxDBGenericRecordSink.java) + +### JDBC ClickHouse + +* [Configuration](io-jdbc-sink.md#configuration) + +* [Example](io-jdbc-sink.md#example-for-clickhouse) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/jdbc/clickhouse/src/main/java/org/apache/pulsar/io/jdbc/ClickHouseJdbcAutoSchemaSink.java) + +### JDBC MariaDB + +* [Configuration](io-jdbc-sink.md#configuration) + +* [Example](io-jdbc-sink.md#example-for-mariadb) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/jdbc/mariadb/src/main/java/org/apache/pulsar/io/jdbc/MariadbJdbcAutoSchemaSink.java) + +### JDBC PostgreSQL + +* [Configuration](io-jdbc-sink.md#configuration) + +* [Example](io-jdbc-sink.md#example-for-postgresql) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/jdbc/postgres/src/main/java/org/apache/pulsar/io/jdbc/PostgresJdbcAutoSchemaSink.java) + +### JDBC SQLite + +* [Configuration](io-jdbc-sink.md#configuration) + +* [Example](io-jdbc-sink.md#example-for-sqlite) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/jdbc/sqlite/src/main/java/org/apache/pulsar/io/jdbc/SqliteJdbcAutoSchemaSink.java) + +### Kafka + +* [Configuration](io-kafka-sink.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSink.java) + +### Kinesis + +* [Configuration](io-kinesis-sink.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java) + +### MongoDB + +* [Configuration](io-mongo-sink.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/mongo/src/main/java/org/apache/pulsar/io/mongodb/MongoSink.java) + +### RabbitMQ + +* [Configuration](io-rabbitmq-sink.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/rabbitmq/src/main/java/org/apache/pulsar/io/rabbitmq/RabbitMQSink.java) + +### Redis + +* [Configuration](io-redis-sink.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/redis/src/main/java/org/apache/pulsar/io/redis/RedisAbstractConfig.java) + +### Solr + +* [Configuration](io-solr-sink.md#configuration) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/solr/src/main/java/org/apache/pulsar/io/solr/SolrSinkConfig.java) + diff --git a/site2/website-next/versioned_docs/version-2.7.1/io-debug.md b/site2/website-next/versioned_docs/version-2.7.1/io-debug.md new file mode 100644 index 0000000000000..f815e862cae42 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/io-debug.md @@ -0,0 +1,411 @@ +--- +id: io-debug +title: How to debug Pulsar connectors +sidebar_label: "Debug" +original_id: io-debug +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +This guide explains how to debug connectors in localrun or cluster mode and gives a debugging checklist. +To better demonstrate how to debug Pulsar connectors, here takes a Mongo sink connector as an example. + +**Deploy a Mongo sink environment** +1. Start a Mongo service. + + ```bash + + docker pull mongo:4 + docker run -d -p 27017:27017 --name pulsar-mongo -v $PWD/data:/data/db mongo:4 + + ``` + +2. Create a DB and a collection. + + ```bash + + docker exec -it pulsar-mongo /bin/bash + mongo + > use pulsar + > db.createCollection('messages') + > exit + + ``` + +3. Start Pulsar standalone. + + ```bash + + docker pull apachepulsar/pulsar:2.4.0 + docker run -d -it -p 6650:6650 -p 8080:8080 -v $PWD/data:/pulsar/data --link pulsar-mongo --name pulsar-mongo-standalone apachepulsar/pulsar:2.4.0 bin/pulsar standalone + + ``` + +4. Configure the Mongo sink with the `mongo-sink-config.yaml` file. + + ```bash + + configs: + mongoUri: "mongodb://pulsar-mongo:27017" + database: "pulsar" + collection: "messages" + batchSize: 2 + batchTimeMs: 500 + + ``` + + ```bash + + docker cp mongo-sink-config.yaml pulsar-mongo-standalone:/pulsar/ + + ``` + +5. Download the Mongo sink nar package. + + ```bash + + docker exec -it pulsar-mongo-standalone /bin/bash + curl -O http://apache.01link.hk/pulsar/pulsar-2.4.0/connectors/pulsar-io-mongo-2.4.0.nar + + ``` + +## Debug in localrun mode +Start the Mongo sink in localrun mode using the `localrun` command. +:::tip + +For more information about the `localrun` command, see [`localrun`](reference-connector-admin.md/#localrun-1). + +::: + +```bash + +./bin/pulsar-admin sinks localrun \ +--archive pulsar-io-mongo-2.4.0.nar \ +--tenant public --namespace default \ +--inputs test-mongo \ +--name pulsar-mongo-sink \ +--sink-config-file mongo-sink-config.yaml \ +--parallelism 1 + +``` + +### Use connector log +Use one of the following methods to get a connector log in localrun mode: +* After executing the `localrun` command, the **log is automatically printed on the console**. +* The log is located at: + + ```bash + + logs/functions/tenant/namespace/function-name/function-name-instance-id.log + + ``` + + **Example** + + The path of the Mongo sink connector is: + + ```bash + + logs/functions/public/default/pulsar-mongo-sink/pulsar-mongo-sink-0.log + + ``` + +To clearly explain the log information, here breaks down the large block of information into small blocks and add descriptions for each block. +* This piece of log information shows the storage path of the nar package after decompression. + + ``` + + 08:21:54.132 [main] INFO org.apache.pulsar.common.nar.NarClassLoader - Created class loader with paths: [file:/tmp/pulsar-nar/pulsar-io-mongo-2.4.0.nar-unpacked/, file:/tmp/pulsar-nar/pulsar-io-mongo-2.4.0.nar-unpacked/META-INF/bundled-dependencies/, + + ``` + + :::tip + + If `class cannot be found` exception is thrown, check whether the nar file is decompressed in the folder `file:/tmp/pulsar-nar/pulsar-io-mongo-2.4.0.nar-unpacked/META-INF/bundled-dependencies/` or not. + + ::: + +* This piece of log information illustrates the basic information about the Mongo sink connector, such as tenant, namespace, name, parallelism, resources, and so on, which can be used to **check whether the Mongo sink connector is configured correctly or not**. + + ```bash + + 08:21:55.390 [main] INFO org.apache.pulsar.functions.runtime.ThreadRuntime - ThreadContainer starting function with instance config InstanceConfig(instanceId=0, functionId=853d60a1-0c48-44d5-9a5c-6917386476b2, functionVersion=c2ce1458-b69e-4175-88c0-a0a856a2be8c, functionDetails=tenant: "public" + namespace: "default" + name: "pulsar-mongo-sink" + className: "org.apache.pulsar.functions.api.utils.IdentityFunction" + autoAck: true + parallelism: 1 + source { + typeClassName: "[B" + inputSpecs { + key: "test-mongo" + value { + } + } + cleanupSubscription: true + } + sink { + className: "org.apache.pulsar.io.mongodb.MongoSink" + configs: "{\"mongoUri\":\"mongodb://pulsar-mongo:27017\",\"database\":\"pulsar\",\"collection\":\"messages\",\"batchSize\":2,\"batchTimeMs\":500}" + typeClassName: "[B" + } + resources { + cpu: 1.0 + ram: 1073741824 + disk: 10737418240 + } + componentType: SINK + , maxBufferedTuples=1024, functionAuthenticationSpec=null, port=38459, clusterName=local) + + ``` + +* This piece of log information demonstrates the status of the connections to Mongo and configuration information. + + ```bash + + 08:21:56.231 [cluster-ClusterId{value='5d6396a3c9e77c0569ff00eb', description='null'}-pulsar-mongo:27017] INFO org.mongodb.driver.connection - Opened connection [connectionId{localValue:1, serverValue:8}] to pulsar-mongo:27017 + 08:21:56.326 [cluster-ClusterId{value='5d6396a3c9e77c0569ff00eb', description='null'}-pulsar-mongo:27017] INFO org.mongodb.driver.cluster - Monitor thread successfully connected to server with description ServerDescription{address=pulsar-mongo:27017, type=STANDALONE, state=CONNECTED, ok=true, version=ServerVersion{versionList=[4, 2, 0]}, minWireVersion=0, maxWireVersion=8, maxDocumentSize=16777216, logicalSessionTimeoutMinutes=30, roundTripTimeNanos=89058800} + + ``` + +* This piece of log information explains the configuration of consumers and clients, including the topic name, subscription name, subscription type, and so on. + + ```bash + + 08:21:56.719 [pulsar-client-io-1-1] INFO org.apache.pulsar.client.impl.ConsumerStatsRecorderImpl - Starting Pulsar consumer status recorder with config: { + "topicNames" : [ "test-mongo" ], + "topicsPattern" : null, + "subscriptionName" : "public/default/pulsar-mongo-sink", + "subscriptionType" : "Shared", + "receiverQueueSize" : 1000, + "acknowledgementsGroupTimeMicros" : 100000, + "negativeAckRedeliveryDelayMicros" : 60000000, + "maxTotalReceiverQueueSizeAcrossPartitions" : 50000, + "consumerName" : null, + "ackTimeoutMillis" : 0, + "tickDurationMillis" : 1000, + "priorityLevel" : 0, + "cryptoFailureAction" : "CONSUME", + "properties" : { + "application" : "pulsar-sink", + "id" : "public/default/pulsar-mongo-sink", + "instance_id" : "0" + }, + "readCompacted" : false, + "subscriptionInitialPosition" : "Latest", + "patternAutoDiscoveryPeriod" : 1, + "regexSubscriptionMode" : "PersistentOnly", + "deadLetterPolicy" : null, + "autoUpdatePartitions" : true, + "replicateSubscriptionState" : false, + "resetIncludeHead" : false + } + 08:21:56.726 [pulsar-client-io-1-1] INFO org.apache.pulsar.client.impl.ConsumerStatsRecorderImpl - Pulsar client config: { + "serviceUrl" : "pulsar://localhost:6650", + "authPluginClassName" : null, + "authParams" : null, + "operationTimeoutMs" : 30000, + "statsIntervalSeconds" : 60, + "numIoThreads" : 1, + "numListenerThreads" : 1, + "connectionsPerBroker" : 1, + "useTcpNoDelay" : true, + "useTls" : false, + "tlsTrustCertsFilePath" : null, + "tlsAllowInsecureConnection" : false, + "tlsHostnameVerificationEnable" : false, + "concurrentLookupRequest" : 5000, + "maxLookupRequest" : 50000, + "maxNumberOfRejectedRequestPerConnection" : 50, + "keepAliveIntervalSeconds" : 30, + "connectionTimeoutMs" : 10000, + "requestTimeoutMs" : 60000, + "defaultBackoffIntervalNanos" : 100000000, + "maxBackoffIntervalNanos" : 30000000000 + } + + ``` + +## Debug in cluster mode +You can use the following methods to debug a connector in cluster mode: +* [Use connector log](#use-connector-log) +* [Use admin CLI](#use-admin-cli) +### Use connector log +In cluster mode, multiple connectors can run on a worker. To find the log path of a specified connector, use the `workerId` to locate the connector log. +### Use admin CLI +Pulsar admin CLI helps you debug Pulsar connectors with the following subcommands: +* [`get`](#get) + +* [`status`](#status) +* [`topics stats`](#topics-stats) + +**Create a Mongo sink** + +```bash + +./bin/pulsar-admin sinks create \ +--archive pulsar-io-mongo-2.4.0.nar \ +--tenant public \ +--namespace default \ +--inputs test-mongo \ +--name pulsar-mongo-sink \ +--sink-config-file mongo-sink-config.yaml \ +--parallelism 1 + +``` + +### `get` +Use the `get` command to get the basic information about the Mongo sink connector, such as tenant, namespace, name, parallelism, and so on. + +```bash + +./bin/pulsar-admin sinks get --tenant public --namespace default --name pulsar-mongo-sink +{ + "tenant": "public", + "namespace": "default", + "name": "pulsar-mongo-sink", + "className": "org.apache.pulsar.io.mongodb.MongoSink", + "inputSpecs": { + "test-mongo": { + "isRegexPattern": false + } + }, + "configs": { + "mongoUri": "mongodb://pulsar-mongo:27017", + "database": "pulsar", + "collection": "messages", + "batchSize": 2.0, + "batchTimeMs": 500.0 + }, + "parallelism": 1, + "processingGuarantees": "ATLEAST_ONCE", + "retainOrdering": false, + "autoAck": true +} + +``` + +:::tip + +For more information about the `get` command, see [`get`](reference-connector-admin.md/#get-1). + +::: + +### `status` +Use the `status` command to get the current status about the Mongo sink connector, such as the number of instance, the number of running instance, instanceId, workerId and so on. + +```bash + +./bin/pulsar-admin sinks status +--tenant public \ +--namespace default \ +--name pulsar-mongo-sink +{ +"numInstances" : 1, +"numRunning" : 1, +"instances" : [ { + "instanceId" : 0, + "status" : { + "running" : true, + "error" : "", + "numRestarts" : 0, + "numReadFromPulsar" : 0, + "numSystemExceptions" : 0, + "latestSystemExceptions" : [ ], + "numSinkExceptions" : 0, + "latestSinkExceptions" : [ ], + "numWrittenToSink" : 0, + "lastReceivedTime" : 0, + "workerId" : "c-standalone-fw-5d202832fd18-8080" + } +} ] +} + +``` + +:::tip + +For more information about the `status` command, see [`status`](reference-connector-admin.md/#stauts-1). +If there are multiple connectors running on a worker, `workerId` can locate the worker on which the specified connector is running. + +::: + +### `topics stats` +Use the `topics stats` command to get the stats for a topic and its connected producer and consumer, such as whether the topic has received messages or not, whether there is a backlog of messages or not, the available permits and other key information. All rates are computed over a 1-minute window and are relative to the last completed 1-minute period. + +```bash + +./bin/pulsar-admin topics stats test-mongo +{ + "msgRateIn" : 0.0, + "msgThroughputIn" : 0.0, + "msgRateOut" : 0.0, + "msgThroughputOut" : 0.0, + "averageMsgSize" : 0.0, + "storageSize" : 1, + "publishers" : [ ], + "subscriptions" : { + "public/default/pulsar-mongo-sink" : { + "msgRateOut" : 0.0, + "msgThroughputOut" : 0.0, + "msgRateRedeliver" : 0.0, + "msgBacklog" : 0, + "blockedSubscriptionOnUnackedMsgs" : false, + "msgDelayed" : 0, + "unackedMessages" : 0, + "type" : "Shared", + "msgRateExpired" : 0.0, + "consumers" : [ { + "msgRateOut" : 0.0, + "msgThroughputOut" : 0.0, + "msgRateRedeliver" : 0.0, + "consumerName" : "dffdd", + "availablePermits" : 999, + "unackedMessages" : 0, + "blockedConsumerOnUnackedMsgs" : false, + "metadata" : { + "instance_id" : "0", + "application" : "pulsar-sink", + "id" : "public/default/pulsar-mongo-sink" + }, + "connectedSince" : "2019-08-26T08:48:07.582Z", + "clientVersion" : "2.4.0", + "address" : "/172.17.0.3:57790" + } ], + "isReplicated" : false + } + }, + "replication" : { }, + "deduplicationStatus" : "Disabled" +} + +``` + +:::tip + +For more information about the `topic stats` command, see [`topic stats`](http://pulsar.apache.org/docs/en/pulsar-admin/#stats-1). + +::: + +## Checklist +This checklist indicates the major areas to check when you debug connectors. It is a reminder of what to look for to ensure a thorough review and an evaluation tool to get the status of connectors. +* Does Pulsar start successfully? + +* Does the external service run normally? + +* Is the nar package complete? + +* Is the connector configuration file correct? + +* In localrun mode, run a connector and check the printed information (connector log) on the console. + +* In cluster mode： + + * Use the `get` command to get the basic information. + + * Use the `status` command to get the current status. + * Use the `topics stats` command to get the stats for a specified topic and its connected producers and consumers. + + * Check the connector log. +* Enter into the external system and verify the result. diff --git a/site2/website-next/versioned_docs/version-2.7.1/io-develop.md b/site2/website-next/versioned_docs/version-2.7.1/io-develop.md new file mode 100644 index 0000000000000..d7531a0d30d67 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/io-develop.md @@ -0,0 +1,267 @@ +--- +id: io-develop +title: How to develop Pulsar connectors +sidebar_label: "Develop" +original_id: io-develop +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +This guide describes how to develop Pulsar connectors to move data +between Pulsar and other systems. + +Pulsar connectors are special [Pulsar Functions](functions-overview), so creating +a Pulsar connector is similar to creating a Pulsar function. + +Pulsar connectors come in two types: + +| Type | Description | Example +|---|---|--- +{@inject: github:Source:/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/Source.java}|Import data from another system to Pulsar.|[RabbitMQ source connector](io-rabbitmq) imports the messages of a RabbitMQ queue to a Pulsar topic. +{@inject: github:Sink:/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/Sink.java}|Export data from Pulsar to another system.|[Kinesis sink connector](io-kinesis) exports the messages of a Pulsar topic to a Kinesis stream. + +## Develop + +You can develop Pulsar source connectors and sink connectors. + +### Source + +Developing a source connector is to implement the {@inject: github:Source:/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/Source.java} +interface, which means you need to implement the {@inject: github:open:/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/Source.java} method and the {@inject: github:read:/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/Source.java} method. + +1. Implement the {@inject: github:open:/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/Source.java} method. + + ```java + + /** + * Open connector with configuration + * + * @param config initialization config + * @param sourceContext + * @throws Exception IO type exceptions when opening a connector + */ + void open(final Map config, SourceContext sourceContext) throws Exception; + + ``` + + This method is called when the source connector is initialized. + + In this method, you can retrieve all connector specific settings through the passed-in `config` parameter and initialize all necessary resources. + + For example, a Kafka connector can create a Kafka client in this `open` method. + + Besides, Pulsar runtime also provides a `SourceContext` for the + connector to access runtime resources for tasks like collecting metrics. The implementation can save the `SourceContext` for future use. + +2. Implement the {@inject: github:read:/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/Source.java} method. + + ```java + + /** + * Reads the next message from source. + * If source does not have any new messages, this call should block. + * @return next message from source. The return result should never be null + * @throws Exception + */ + Record read() throws Exception; + + ``` + + If nothing to return, the implementation should be blocking rather than returning `null`. + + The returned {@inject: github:Record:/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java} should encapsulate the following information, which is needed by Pulsar IO runtime. + + * {@inject: github:Record:/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java} should provide the following variables: + + |Variable|Required|Description + |---|---|--- + `TopicName`|No|Pulsar topic name from which the record is originated from. + `Key`|No| Messages can optionally be tagged with keys.

    For more information, see [Routing modes](concepts-messaging.md#routing-modes).| + `Value`|Yes|Actual data of the record. + `EventTime`|No|Event time of the record from the source. + `PartitionId`|No| If the record is originated from a partitioned source, it returns its `PartitionId`.

    `PartitionId` is used as a part of the unique identifier by Pulsar IO runtime to deduplicate messages and achieve exactly-once processing guarantee. + `RecordSequence`|No|If the record is originated from a sequential source, it returns its `RecordSequence`.

    `RecordSequence` is used as a part of the unique identifier by Pulsar IO runtime to deduplicate messages and achieve exactly-once processing guarantee. + `Properties` |No| If the record carries user-defined properties, it returns those properties. + `DestinationTopic`|No|Topic to which message should be written. + `Message`|No|A class which carries data sent by users.

    For more information, see [Message.java](https://github.com/apache/pulsar/blob/master/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/Message.java).| + + * {@inject: github:Record:/pulsar-functions/api-java/src/main/java/org/apache/pulsar/functions/api/Record.java} should provide the following methods: + + Method|Description + |---|--- + `ack` |Acknowledge that the record is fully processed. + `fail`|Indicate that the record fails to be processed. + +:::tip + +For more information about **how to create a source connector**, see {@inject: github:KafkaSource:/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java}. + +::: + +### Sink + +Developing a sink connector **is similar to** developing a source connector, that is, you need to implement the {@inject: github:Sink:/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/Sink.java} interface, which means implementing the {@inject: github:open:/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/Sink.java} method and the {@inject: github:write:/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/Sink.java} method. + +1. Implement the {@inject: github:open:/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/Sink.java} method. + + ```java + + /** + * Open connector with configuration + * + * @param config initialization config + * @param sinkContext + * @throws Exception IO type exceptions when opening a connector + */ + void open(final Map config, SinkContext sinkContext) throws Exception; + + ``` + +2. Implement the {@inject: github:write:/pulsar-io/core/src/main/java/org/apache/pulsar/io/core/Sink.java} method. + + ```java + + /** + * Write a message to Sink + * @param record record to write to sink + * @throws Exception + */ + void write(Record record) throws Exception; + + ``` + + During the implementation, you can decide how to write the `Value` and + the `Key` to the actual source, and leverage all the provided information such as + `PartitionId` and `RecordSequence` to achieve different processing guarantees. + + You also need to ack records (if messages are sent successfully) or fail records (if messages fail to send). + +## Test + +Testing connectors can be challenging because Pulsar IO connectors interact with two systems +that may be difficult to mock—Pulsar and the system to which the connector is connecting. + +It is +recommended writing special tests to test the connector functionalities as below +while mocking the external service. + +### Unit test + +You can create unit tests for your connector. + +### Integration test + +Once you have written sufficient unit tests, you can add +separate integration tests to verify end-to-end functionality. + +Pulsar uses [testcontainers](https://www.testcontainers.org/) **for all integration tests**. + +:::tip + +For more information about **how to create integration tests for Pulsar connectors**, see {@inject: github:IntegrationTests:/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io}. + +::: + +## Package + +Once you've developed and tested your connector, you need to package it so that it can be submitted +to a [Pulsar Functions](functions-overview) cluster. + +There are two methods to +work with Pulsar Functions' runtime, that is, [NAR](#nar) and [uber JAR](#uber-jar). + +:::note + +If you plan to package and distribute your connector for others to use, you are obligated to + +::: + +license and copyright your own code properly. Remember to add the license and copyright to +all libraries your code uses and to your distribution. +> +> If you use the [NAR](#nar) method, the NAR plugin +automatically creates a `DEPENDENCIES` file in the generated NAR package, including the proper +licensing and copyrights of all libraries of your connector. + +### NAR + +**NAR** stands for NiFi Archive, which is a custom packaging mechanism used by Apache NiFi, to provide +a bit of Java ClassLoader isolation. + +:::tip + +For more information about **how NAR works**, see [here](https://medium.com/hashmapinc/nifi-nar-files-explained-14113f7796fd). + +::: + +Pulsar uses the same mechanism for packaging **all** [built-in connectors](io-connectors). + +The easiest approach to package a Pulsar connector is to create a NAR package using [nifi-nar-maven-plugin](https://mvnrepository.com/artifact/org.apache.nifi/nifi-nar-maven-plugin). + +Include this [nifi-nar-maven-plugin](https://mvnrepository.com/artifact/org.apache.nifi/nifi-nar-maven-plugin) in your maven project for your connector as below. + +```xml + + + + org.apache.nifi + nifi-nar-maven-plugin + 1.2.0 + + + +``` + +You must also create a `resources/META-INF/services/pulsar-io.yaml` file with the following contents: + +```yaml + +name: connector name +description: connector description +sourceClass: fully qualified class name (only if source connector) +sinkClass: fully qualified class name (only if sink connector) + +``` + +For Gradle users, there is a [Gradle Nar plugin available on the Gradle Plugin Portal](https://plugins.gradle.org/plugin/io.github.lhotari.gradle-nar-plugin). + +:::tip + +For more information about an **how to use NAR for Pulsar connectors**, see {@inject: github:TwitterFirehose:/pulsar-io/twitter/pom.xml}. + +::: + +### Uber JAR + +An alternative approach is to create an **uber JAR** that contains all of the connector's JAR files +and other resource files. No directory internal structure is necessary. + +You can use [maven-shade-plugin](https://maven.apache.org/plugins/maven-shade-plugin/examples/includes-excludes.html) to create a uber JAR as below: + +```xml + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.1 + + + package + + shade + + + + + *:* + + + + + + + +``` + diff --git a/site2/website-next/versioned_docs/version-2.7.1/io-overview.md b/site2/website-next/versioned_docs/version-2.7.1/io-overview.md new file mode 100644 index 0000000000000..3a55ff2aef2cc --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/io-overview.md @@ -0,0 +1,176 @@ +--- +id: io-overview +title: Pulsar connector overview +sidebar_label: "Overview" +original_id: io-overview +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +Messaging systems are most powerful when you can easily use them with external systems like databases and other messaging systems. + +**Pulsar IO connectors** enable you to easily create, deploy, and manage connectors that interact with external systems, such as [Apache Cassandra](https://cassandra.apache.org), [Aerospike](https://www.aerospike.com), and many others. + + +## Concept + +Pulsar IO connectors come in two types: **source** and **sink**. + +This diagram illustrates the relationship between source, Pulsar, and sink: + +![Pulsar IO diagram](/assets/pulsar-io.png "Pulsar IO connectors (sources and sinks)") + + +### Source + +> Sources **feed data from external systems into Pulsar**. + +Common sources include other messaging systems and firehose-style data pipeline APIs. + +For the complete list of Pulsar built-in source connectors, see [source connector](io-connectors.md#source-connector). + +### Sink + +> Sinks **feed data from Pulsar into external systems**. + +Common sinks include other messaging systems and SQL and NoSQL databases. + +For the complete list of Pulsar built-in sink connectors, see [sink connector](io-connectors.md#sink-connector). + +## Processing guarantee + +Processing guarantees are used to handle errors when writing messages to Pulsar topics. + +> Pulsar connectors and Functions use the **same** processing guarantees as below. + +Delivery semantic | Description +:------------------|:------- +`at-most-once` | Each message sent to a connector is to be **processed once** or **not to be processed**. +`at-least-once` | Each message sent to a connector is to be **processed once** or **more than once**. +`effectively-once` | Each message sent to a connector has **one output associated** with it. + +> Processing guarantees for connectors not just rely on Pulsar guarantee but also **relate to external systems**, that is, **the implementation of source and sink**. + +* Source: Pulsar ensures that writing messages to Pulsar topics respects to the processing guarantees. It is within Pulsar's control. + +* Sink: the processing guarantees rely on the sink implementation. If the sink implementation does not handle retries in an idempotent way, the sink does not respect to the processing guarantees. + +### Set + +When creating a connector, you can set the processing guarantee with the following semantics: + +* ATLEAST_ONCE + +* ATMOST_ONCE + +* EFFECTIVELY_ONCE + +> If `--processing-guarantees` is not specified when creating a connector, the default semantic is `ATLEAST_ONCE`. + +Here takes **Admin CLI** as an example. For more information about **REST API** or **JAVA Admin API**, see [here](io-use.md#create). + + + + + +```bash + +$ bin/pulsar-admin sources create \ + --processing-guarantees ATMOST_ONCE \ + # Other source configs + +``` + +For more information about the options of `pulsar-admin sources create`, see [here](reference-connector-admin.md#create). + + + + +```bash + +$ bin/pulsar-admin sinks create \ + --processing-guarantees EFFECTIVELY_ONCE \ + # Other sink configs + +``` + +For more information about the options of `pulsar-admin sinks create`, see [here](reference-connector-admin.md#create-1). + + + + + +### Update + +After creating a connector, you can update the processing guarantee with the following semantics: + +* ATLEAST_ONCE + +* ATMOST_ONCE + +* EFFECTIVELY_ONCE + +Here takes **Admin CLI** as an example. For more information about **REST API** or **JAVA Admin API**, see [here](io-use.md#create). + + + + + +```bash + +$ bin/pulsar-admin sources update \ + --processing-guarantees EFFECTIVELY_ONCE \ + # Other source configs + +``` + +For more information about the options of `pulsar-admin sources update`, see [here](reference-connector-admin.md#update). + + + + +```bash + +$ bin/pulsar-admin sinks update \ + --processing-guarantees ATMOST_ONCE \ + # Other sink configs + +``` + +For more information about the options of `pulsar-admin sinks update`, see [here](reference-connector-admin.md#update-1). + + + + + + +## Work with connector + +You can manage Pulsar connectors (for example, create, update, start, stop, restart, reload, delete and perform other operations on connectors) via the [Connector Admin CLI](reference-connector-admin) with [sources](reference-connector-admin.md#sources) and [sinks](reference-connector-admin.md#sinks) subcommands. + +Connectors (sources and sinks) and Functions are components of instances, and they all run on Functions workers. When managing a source, sink or function via [Connector Admin CLI](reference-connector-admin.md) or [Functions Admin CLI](functions-cli), an instance is started on a worker. For more information, see [Functions worker](functions-worker.md#run-functions-worker-separately). + diff --git a/site2/website-next/versioned_docs/version-2.7.1/io-quickstart.md b/site2/website-next/versioned_docs/version-2.7.1/io-quickstart.md new file mode 100644 index 0000000000000..67ca9710a10a8 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/io-quickstart.md @@ -0,0 +1,968 @@ +--- +id: io-quickstart +title: How to connect Pulsar to database +sidebar_label: "Get started" +original_id: io-quickstart +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +This tutorial provides a hands-on look at how you can move data out of Pulsar without writing a single line of code. + +It is helpful to review the [concepts](io-overview) for Pulsar I/O with running the steps in this guide to gain a deeper understanding. + +At the end of this tutorial, you are able to: + +- [Connect Pulsar to Cassandra](#Connect-Pulsar-to-Cassandra) + +- [Connect Pulsar to PostgreSQL](#Connect-Pulsar-to-PostgreSQL) + +:::tip + +* These instructions assume you are running Pulsar in [standalone mode](getting-started-standalone). However, all +the commands used in this tutorial can be used in a multi-nodes Pulsar cluster without any changes. +* All the instructions are assumed to run at the root directory of a Pulsar binary distribution. + +::: + +## Install Pulsar and built-in connector + +Before connecting Pulsar to a database, you need to install Pulsar and the desired built-in connector. + +For more information about **how to install a standalone Pulsar and built-in connectors**, see [here](getting-started-standalone.md/#installing-pulsar). + +## Start Pulsar standalone + +1. Start Pulsar locally. + + ```bash + + bin/pulsar standalone + + ``` + + All the components of a Pulsar service are start in order. + + You can curl those pulsar service endpoints to make sure Pulsar service is up running correctly. + +2. Check Pulsar binary protocol port. + + ```bash + + telnet localhost 6650 + + ``` + +3. Check Pulsar Function cluster. + + ```bash + + curl -s http://localhost:8080/admin/v2/worker/cluster + + ``` + + **Example output** + + ```json + + [{"workerId":"c-standalone-fw-localhost-6750","workerHostname":"localhost","port":6750}] + + ``` + +4. Make sure a public tenant and a default namespace exist. + + ```bash + + curl -s http://localhost:8080/admin/v2/namespaces/public + + ``` + + **Example output** + + ```json + + ["public/default","public/functions"] + + ``` + +5. All built-in connectors should be listed as available. + + ```bash + + curl -s http://localhost:8080/admin/v2/functions/connectors + + ``` + + **Example output** + + ```json + + [{"name":"aerospike","description":"Aerospike database sink","sinkClass":"org.apache.pulsar.io.aerospike.AerospikeStringSink"},{"name":"cassandra","description":"Writes data into Cassandra","sinkClass":"org.apache.pulsar.io.cassandra.CassandraStringSink"},{"name":"kafka","description":"Kafka source and sink connector","sourceClass":"org.apache.pulsar.io.kafka.KafkaStringSource","sinkClass":"org.apache.pulsar.io.kafka.KafkaBytesSink"},{"name":"kinesis","description":"Kinesis sink connector","sinkClass":"org.apache.pulsar.io.kinesis.KinesisSink"},{"name":"rabbitmq","description":"RabbitMQ source connector","sourceClass":"org.apache.pulsar.io.rabbitmq.RabbitMQSource"},{"name":"twitter","description":"Ingest data from Twitter firehose","sourceClass":"org.apache.pulsar.io.twitter.TwitterFireHose"}] + + ``` + + If an error occurs when starting Pulsar service, you may see an exception at the terminal running `pulsar/standalone`, + or you can navigate to the `logs` directory under the Pulsar directory to view the logs. + +## Connect Pulsar to Cassandra + +This section demonstrates how to connect Pulsar to Cassandra. + +:::tip + +* Make sure you have Docker installed. If you do not have one, see [install Docker](https://docs.docker.com/docker-for-mac/install/). +* The Cassandra sink connector reads messages from Pulsar topics and writes the messages into Cassandra tables. For more information, see [Cassandra sink connector](io-cassandra-sink). + +::: + +### Setup a Cassandra cluster + +This example uses `cassandra` Docker image to start a single-node Cassandra cluster in Docker. + +1. Start a Cassandra cluster. + + ```bash + + docker run -d --rm --name=cassandra -p 9042:9042 cassandra + + ``` + + :::note + + Before moving to the next steps, make sure the Cassandra cluster is running. + + ::: + +2. Make sure the Docker process is running. + + ```bash + + docker ps + + ``` + +3. Check the Cassandra logs to make sure the Cassandra process is running as expected. + + ```bash + + docker logs cassandra + + ``` + +4. Check the status of the Cassandra cluster. + + ```bash + + docker exec cassandra nodetool status + + ``` + + **Example output** + + ``` + + Datacenter: datacenter1 + ======================= + Status=Up/Down + |/ State=Normal/Leaving/Joining/Moving + -- Address Load Tokens Owns (effective) Host ID Rack + UN 172.17.0.2 103.67 KiB 256 100.0% af0e4b2f-84e0-4f0b-bb14-bd5f9070ff26 rack1 + + ``` + +5. Use `cqlsh` to connect to the Cassandra cluster. + + ```bash + + $ docker exec -ti cassandra cqlsh localhost + Connected to Test Cluster at localhost:9042. + [cqlsh 5.0.1 | Cassandra 3.11.2 | CQL spec 3.4.4 | Native protocol v4] + Use HELP for help. + cqlsh> + + ``` + +6. Create a keyspace `pulsar_test_keyspace`. + + ```bash + + cqlsh> CREATE KEYSPACE pulsar_test_keyspace WITH replication = {'class':'SimpleStrategy', 'replication_factor':1}; + + ``` + +7. Create a table `pulsar_test_table`. + + ```bash + + cqlsh> USE pulsar_test_keyspace; + cqlsh:pulsar_test_keyspace> CREATE TABLE pulsar_test_table (key text PRIMARY KEY, col text); + + ``` + +### Configure a Cassandra sink + +Now that we have a Cassandra cluster running locally. + +In this section, you need to configure a Cassandra sink connector. + +To run a Cassandra sink connector, you need to prepare a configuration file including the information that Pulsar connector runtime needs to know. + +For example, how Pulsar connector can find the Cassandra cluster, what is the keyspace and the table that Pulsar connector uses for writing Pulsar messages to, and so on. + +You can create a configuration file through one of the following methods. + +* JSON + + ```json + + { + "roots": "localhost:9042", + "keyspace": "pulsar_test_keyspace", + "columnFamily": "pulsar_test_table", + "keyname": "key", + "columnName": "col" + } + + ``` + +* YAML + + ```yaml + + configs: + roots: "localhost:9042" + keyspace: "pulsar_test_keyspace" + columnFamily: "pulsar_test_table" + keyname: "key" + columnName: "col" + + ``` + +For more information, see [Cassandra sink connector](io-cassandra-sink). + +### Create a Cassandra sink + +You can use the [Connector Admin CLI](io-cli) +to create a sink connector and perform other operations on them. + +Run the following command to create a Cassandra sink connector with sink type _cassandra_ and the config file _examples/cassandra-sink.yml_ created previously. + +#### Note +> The `sink-type` parameter of the currently built-in connectors is determined by the setting of the `name` parameter specified in the pulsar-io.yaml file. + +```bash + +bin/pulsar-admin sinks create \ + --tenant public \ + --namespace default \ + --name cassandra-test-sink \ + --sink-type cassandra \ + --sink-config-file examples/cassandra-sink.yml \ + --inputs test_cassandra + +``` + +Once the command is executed, Pulsar creates the sink connector _cassandra-test-sink_. + +This sink connector runs +as a Pulsar Function and writes the messages produced in the topic _test_cassandra_ to the Cassandra table _pulsar_test_table_. + +### Inspect a Cassandra sink + +You can use the [Connector Admin CLI](io-cli) +to monitor a connector and perform other operations on it. + +* Get the information of a Cassandra sink. + + ```bash + + bin/pulsar-admin sinks get \ + --tenant public \ + --namespace default \ + --name cassandra-test-sink + + ``` + + **Example output** + + ```json + + { + "tenant": "public", + "namespace": "default", + "name": "cassandra-test-sink", + "className": "org.apache.pulsar.io.cassandra.CassandraStringSink", + "inputSpecs": { + "test_cassandra": { + "isRegexPattern": false + } + }, + "configs": { + "roots": "localhost:9042", + "keyspace": "pulsar_test_keyspace", + "columnFamily": "pulsar_test_table", + "keyname": "key", + "columnName": "col" + }, + "parallelism": 1, + "processingGuarantees": "ATLEAST_ONCE", + "retainOrdering": false, + "autoAck": true, + "archive": "builtin://cassandra" + } + + ``` + +* Check the status of a Cassandra sink. + + ```bash + + bin/pulsar-admin sinks status \ + --tenant public \ + --namespace default \ + --name cassandra-test-sink + + ``` + + **Example output** + + ```json + + { + "numInstances" : 1, + "numRunning" : 1, + "instances" : [ { + "instanceId" : 0, + "status" : { + "running" : true, + "error" : "", + "numRestarts" : 0, + "numReadFromPulsar" : 0, + "numSystemExceptions" : 0, + "latestSystemExceptions" : [ ], + "numSinkExceptions" : 0, + "latestSinkExceptions" : [ ], + "numWrittenToSink" : 0, + "lastReceivedTime" : 0, + "workerId" : "c-standalone-fw-localhost-8080" + } + } ] + } + + ``` + +### Verify a Cassandra sink + +1. Produce some messages to the input topic of the Cassandra sink _test_cassandra_. + + ```bash + + for i in {0..9}; do bin/pulsar-client produce -m "key-$i" -n 1 test_cassandra; done + + ``` + +2. Inspect the status of the Cassandra sink _test_cassandra_. + + ```bash + + bin/pulsar-admin sinks status \ + --tenant public \ + --namespace default \ + --name cassandra-test-sink + + ``` + + You can see 10 messages are processed by the Cassandra sink _test_cassandra_. + + **Example output** + + ```json + + { + "numInstances" : 1, + "numRunning" : 1, + "instances" : [ { + "instanceId" : 0, + "status" : { + "running" : true, + "error" : "", + "numRestarts" : 0, + "numReadFromPulsar" : 10, + "numSystemExceptions" : 0, + "latestSystemExceptions" : [ ], + "numSinkExceptions" : 0, + "latestSinkExceptions" : [ ], + "numWrittenToSink" : 10, + "lastReceivedTime" : 1551685489136, + "workerId" : "c-standalone-fw-localhost-8080" + } + } ] + } + + ``` + +3. Use `cqlsh` to connect to the Cassandra cluster. + + ```bash + + docker exec -ti cassandra cqlsh localhost + + ``` + +4. Check the data of the Cassandra table _pulsar_test_table_. + + ```bash + + cqlsh> use pulsar_test_keyspace; + cqlsh:pulsar_test_keyspace> select * from pulsar_test_table; + + key | col + --------+-------- + key-5 | key-5 + key-0 | key-0 + key-9 | key-9 + key-2 | key-2 + key-1 | key-1 + key-3 | key-3 + key-6 | key-6 + key-7 | key-7 + key-4 | key-4 + key-8 | key-8 + + ``` + +### Delete a Cassandra Sink + +You can use the [Connector Admin CLI](io-cli) +to delete a connector and perform other operations on it. + +```bash + +bin/pulsar-admin sinks delete \ + --tenant public \ + --namespace default \ + --name cassandra-test-sink + +``` + +## Connect Pulsar to PostgreSQL + +This section demonstrates how to connect Pulsar to PostgreSQL. + +:::tip + +* Make sure you have Docker installed. If you do not have one, see [install Docker](https://docs.docker.com/docker-for-mac/install/). +* The JDBC sink connector pulls messages from Pulsar topics + +::: + +and persists the messages to ClickHouse, MariaDB, PostgreSQL, or SQlite. +>For more information, see [JDBC sink connector](io-jdbc-sink). + + +### Setup a PostgreSQL cluster + +This example uses the PostgreSQL 12 docker image to start a single-node PostgreSQL cluster in Docker. + +1. Pull the PostgreSQL 12 image from Docker. + + ```bash + + $ docker pull postgres:12 + + ``` + +2. Start PostgreSQL. + + ```bash + + $ docker run -d -it --rm \ + --name pulsar-postgres \ + -p 5432:5432 \ + -e POSTGRES_PASSWORD=password \ + -e POSTGRES_USER=postgres \ + postgres:12 + + ``` + + #### Tip + + Flag | Description | This example + ---|---|---| + `-d` | To start a container in detached mode. | / + `-it` | Keep STDIN open even if not attached and allocate a terminal. | / + `--rm` | Remove the container automatically when it exits. | / + `-name` | Assign a name to the container. | This example specifies _pulsar-postgres_ for the container. + `-p` | Publish the port of the container to the host. | This example publishes the port _5432_ of the container to the host. + `-e` | Set environment variables. | This example sets the following variables:
    - The password for the user is _password_.
    - The name for the user is _postgres_. + + :::tip + + For more information about Docker commands, see [Docker CLI](https://docs.docker.com/engine/reference/commandline/run/). + + ::: + +3. Check if PostgreSQL has been started successfully. + + ```bash + + $ docker logs -f pulsar-postgres + + ``` + + PostgreSQL has been started successfully if the following message appears. + + ```text + + 2020-05-11 20:09:24.492 UTC [1] LOG: starting PostgreSQL 12.2 (Debian 12.2-2.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit + 2020-05-11 20:09:24.492 UTC [1] LOG: listening on IPv4 address "0.0.0.0", port 5432 + 2020-05-11 20:09:24.492 UTC [1] LOG: listening on IPv6 address "::", port 5432 + 2020-05-11 20:09:24.499 UTC [1] LOG: listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432" + 2020-05-11 20:09:24.523 UTC [55] LOG: database system was shut down at 2020-05-11 20:09:24 UTC + 2020-05-11 20:09:24.533 UTC [1] LOG: database system is ready to accept connections + + ``` + +4. Access to PostgreSQL. + + ```bash + + $ docker exec -it pulsar-postgres /bin/bash + + ``` + +5. Create a PostgreSQL table _pulsar_postgres_jdbc_sink_. + + ```bash + + $ psql -U postgres postgres + + postgres=# create table if not exists pulsar_postgres_jdbc_sink + ( + id serial PRIMARY KEY, + name VARCHAR(255) NOT NULL + ); + + ``` + +### Configure a JDBC sink + +Now we have a PostgreSQL running locally. + +In this section, you need to configure a JDBC sink connector. + +1. Add a configuration file. + + To run a JDBC sink connector, you need to prepare a YAML configuration file including the information that Pulsar connector runtime needs to know. + + For example, how Pulsar connector can find the PostgreSQL cluster, what is the JDBC URL and the table that Pulsar connector uses for writing messages to. + + Create a _pulsar-postgres-jdbc-sink.yaml_ file, copy the following contents to this file, and place the file in the `pulsar/connectors` folder. + + ```yaml + + configs: + userName: "postgres" + password: "password" + jdbcUrl: "jdbc:postgresql://localhost:5432/pulsar_postgres_jdbc_sink" + tableName: "pulsar_postgres_jdbc_sink" + + ``` + +2. Create a schema. + + Create a _avro-schema_ file, copy the following contents to this file, and place the file in the `pulsar/connectors` folder. + + ```json + + { + "type": "AVRO", + "schema": "{\"type\":\"record\",\"name\":\"Test\",\"fields\":[{\"name\":\"id\",\"type\":[\"null\",\"int\"]},{\"name\":\"name\",\"type\":[\"null\",\"string\"]}]}", + "properties": {} + } + + ``` + + :::tip + + For more information about AVRO, see [Apache Avro](https://avro.apache.org/docs/1.9.1/). + + ::: + +3. Upload a schema to a topic. + + This example uploads the _avro-schema_ schema to the _pulsar-postgres-jdbc-sink-topic_ topic. + + ```bash + + $ bin/pulsar-admin schemas upload pulsar-postgres-jdbc-sink-topic -f ./connectors/avro-schema + + ``` + +4. Check if the schema has been uploaded successfully. + + ```bash + + $ bin/pulsar-admin schemas get pulsar-postgres-jdbc-sink-topic + + ``` + + The schema has been uploaded successfully if the following message appears. + + ```json + + {"name":"pulsar-postgres-jdbc-sink-topic","schema":"{\"type\":\"record\",\"name\":\"Test\",\"fields\":[{\"name\":\"id\",\"type\":[\"null\",\"int\"]},{\"name\":\"name\",\"type\":[\"null\",\"string\"]}]}","type":"AVRO","properties":{}} + + ``` + +### Create a JDBC sink + +You can use the [Connector Admin CLI](io-cli) +to create a sink connector and perform other operations on it. + +This example creates a sink connector and specifies the desired information. + +```bash + +$ bin/pulsar-admin sinks create \ +--archive ./connectors/pulsar-io-jdbc-postgres-@pulsar:version@.nar \ +--inputs pulsar-postgres-jdbc-sink-topic \ +--name pulsar-postgres-jdbc-sink \ +--sink-config-file ./connectors/pulsar-postgres-jdbc-sink.yaml \ +--parallelism 1 + +``` + +Once the command is executed, Pulsar creates a sink connector _pulsar-postgres-jdbc-sink_. + +This sink connector runs as a Pulsar Function and writes the messages produced in the topic _pulsar-postgres-jdbc-sink-topic_ to the PostgreSQL table _pulsar_postgres_jdbc_sink_. + + #### Tip + + Flag | Description | This example + ---|---|---| + `--archive` | The path to the archive file for the sink. | _pulsar-io-jdbc-postgres-@pulsar:version@.nar_ | + `--inputs` | The input topic(s) of the sink.

    Multiple topics can be specified as a comma-separated list.|| + `--name` | The name of the sink. | _pulsar-postgres-jdbc-sink_ | + `--sink-config-file` | The path to a YAML config file specifying the configuration of the sink. | _pulsar-postgres-jdbc-sink.yaml_ | + `--parallelism` | The parallelism factor of the sink.

    For example, the number of sink instances to run. | _1_ | + +:::tip + +For more information about `pulsar-admin sinks create options`, see [here](io-cli.md#sinks). + +::: + +The sink has been created successfully if the following message appears. + +```bash + +"Created successfully" + +``` + +### Inspect a JDBC sink + +You can use the [Connector Admin CLI](io-cli) +to monitor a connector and perform other operations on it. + +* List all running JDBC sink(s). + + ```bash + + $ bin/pulsar-admin sinks list \ + --tenant public \ + --namespace default + + ``` + + :::tip + + For more information about `pulsar-admin sinks list options`, see [here](io-cli.md/#list-1). + + ::: + + The result shows that only the _postgres-jdbc-sink_ sink is running. + + ```json + + [ + "pulsar-postgres-jdbc-sink" + ] + + ``` + +* Get the information of a JDBC sink. + + ```bash + + $ bin/pulsar-admin sinks get \ + --tenant public \ + --namespace default \ + --name pulsar-postgres-jdbc-sink + + ``` + + :::tip + + For more information about `pulsar-admin sinks get options`, see [here](io-cli.md/#get-1). + + ::: + + The result shows the information of the sink connector, including tenant, namespace, topic and so on. + + ```json + + { + "tenant": "public", + "namespace": "default", + "name": "pulsar-postgres-jdbc-sink", + "className": "org.apache.pulsar.io.jdbc.PostgresJdbcAutoSchemaSink", + "inputSpecs": { + "pulsar-postgres-jdbc-sink-topic": { + "isRegexPattern": false + } + }, + "configs": { + "password": "password", + "jdbcUrl": "jdbc:postgresql://localhost:5432/pulsar_postgres_jdbc_sink", + "userName": "postgres", + "tableName": "pulsar_postgres_jdbc_sink" + }, + "parallelism": 1, + "processingGuarantees": "ATLEAST_ONCE", + "retainOrdering": false, + "autoAck": true + } + + ``` + +* Get the status of a JDBC sink + + ```bash + + $ bin/pulsar-admin sinks status \ + --tenant public \ + --namespace default \ + --name pulsar-postgres-jdbc-sink + + ``` + + :::tip + + For more information about `pulsar-admin sinks status options`, see [here](io-cli.md/#status-1). + + ::: + + The result shows the current status of sink connector, including the number of instance, running status, worker ID and so on. + + ```json + + { + "numInstances" : 1, + "numRunning" : 1, + "instances" : [ { + "instanceId" : 0, + "status" : { + "running" : true, + "error" : "", + "numRestarts" : 0, + "numReadFromPulsar" : 0, + "numSystemExceptions" : 0, + "latestSystemExceptions" : [ ], + "numSinkExceptions" : 0, + "latestSinkExceptions" : [ ], + "numWrittenToSink" : 0, + "lastReceivedTime" : 0, + "workerId" : "c-standalone-fw-192.168.2.52-8080" + } + } ] + } + + ``` + +### Stop a JDBC sink + +You can use the [Connector Admin CLI](io-cli) +to stop a connector and perform other operations on it. + +```bash + +$ bin/pulsar-admin sinks stop \ +--tenant public \ +--namespace default \ +--name pulsar-postgres-jdbc-sink + +``` + +:::tip + +For more information about `pulsar-admin sinks stop options`, see [here](io-cli.md/#stop-1). + +::: + +The sink instance has been stopped successfully if the following message disappears. + +```bash + +"Stopped successfully" + +``` + +### Restart a JDBC sink + +You can use the [Connector Admin CLI](io-cli) +to restart a connector and perform other operations on it. + +```bash + +$ bin/pulsar-admin sinks restart \ +--tenant public \ +--namespace default \ +--name pulsar-postgres-jdbc-sink + +``` + +:::tip + +For more information about `pulsar-admin sinks restart options`, see [here](io-cli.md/#restart-1). + +::: + +The sink instance has been started successfully if the following message disappears. + +```bash + +"Started successfully" + +``` + +:::tip + +* Optionally, you can run a standalone sink connector using `pulsar-admin sinks localrun options`. +Note that `pulsar-admin sinks localrun options` **runs a sink connector locally**, while `pulsar-admin sinks start options` **starts a sink connector in a cluster**. +* For more information about `pulsar-admin sinks localrun options`, see [here](io-cli.md#localrun-1). + +::: + +### Update a JDBC sink + +You can use the [Connector Admin CLI](io-cli) +to update a connector and perform other operations on it. + +This example updates the parallelism of the _pulsar-postgres-jdbc-sink_ sink connector to 2. + +```bash + +$ bin/pulsar-admin sinks update \ +--name pulsar-postgres-jdbc-sink \ +--parallelism 2 + +``` + +:::tip + +For more information about `pulsar-admin sinks update options`, see [here](io-cli.md/#update-1). + +::: + +The sink connector has been updated successfully if the following message disappears. + +```bash + +"Updated successfully" + +``` + +This example double-checks the information. + +```bash + +$ bin/pulsar-admin sinks get \ +--tenant public \ +--namespace default \ +--name pulsar-postgres-jdbc-sink + +``` + +The result shows that the parallelism is 2. + +```json + +{ + "tenant": "public", + "namespace": "default", + "name": "pulsar-postgres-jdbc-sink", + "className": "org.apache.pulsar.io.jdbc.PostgresJdbcAutoSchemaSink", + "inputSpecs": { + "pulsar-postgres-jdbc-sink-topic": { + "isRegexPattern": false + } + }, + "configs": { + "password": "password", + "jdbcUrl": "jdbc:postgresql://localhost:5432/pulsar_postgres_jdbc_sink", + "userName": "postgres", + "tableName": "pulsar_postgres_jdbc_sink" + }, + "parallelism": 2, + "processingGuarantees": "ATLEAST_ONCE", + "retainOrdering": false, + "autoAck": true +} + +``` + +### Delete a JDBC sink + +You can use the [Connector Admin CLI](io-cli) +to delete a connector and perform other operations on it. + +This example deletes the _pulsar-postgres-jdbc-sink_ sink connector. + +```bash + +$ bin/pulsar-admin sinks delete \ +--tenant public \ +--namespace default \ +--name pulsar-postgres-jdbc-sink + +``` + +:::tip + +For more information about `pulsar-admin sinks delete options`, see [here](io-cli.md/#delete-1). + +::: + +The sink connector has been deleted successfully if the following message appears. + +```text + +"Deleted successfully" + +``` + +This example double-checks the status of the sink connector. + +```bash + +$ bin/pulsar-admin sinks get \ +--tenant public \ +--namespace default \ +--name pulsar-postgres-jdbc-sink + +``` + +The result shows that the sink connector does not exist. + +```text + +HTTP 404 Not Found + +Reason: Sink pulsar-postgres-jdbc-sink doesn't exist + +``` + diff --git a/site2/website-next/versioned_docs/version-2.7.1/io-use.md b/site2/website-next/versioned_docs/version-2.7.1/io-use.md new file mode 100644 index 0000000000000..43b7a30cadc33 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/io-use.md @@ -0,0 +1,1981 @@ +--- +id: io-use +title: How to use Pulsar connectors +sidebar_label: "Use" +original_id: io-use +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +This guide describes how to use Pulsar connectors. + +## Install a connector + +Pulsar bundles several [builtin connectors](io-connectors) used to move data in and out of commonly used systems (such as database and messaging system). Optionally, you can create and use your desired non-builtin connectors. + +:::note + +When using a non-builtin connector, you need to specify the path of a archive file for the connector. + +::: + +To set up a builtin connector, follow +the instructions [here](getting-started-standalone.md#installing-builtin-connectors). + +After the setup, the builtin connector is automatically discovered by Pulsar brokers (or function-workers), so no additional installation steps are required. + +## Configure a connector + +You can configure the following information: + +* [Configure a default storage location for a connector](#configure-a-default-storage-location-for-a-connector) + +* [Configure a connector with a YAML file](#configure-a-connector-with-yaml-file) + +### Configure a default storage location for a connector + +To configure a default folder for builtin connectors, set the `connectorsDirectory` parameter in the `./conf/functions_worker.yml` configuration file. + +**Example** + +Set the `./connectors` folder as the default storage location for builtin connectors. + +``` + +######################## +# Connectors +######################## + +connectorsDirectory: ./connectors + +``` + +### Configure a connector with a YAML file + +To configure a connector, you need to provide a YAML configuration file when creating a connector. + +The YAML configuration file tells Pulsar where to locate connectors and how to connect connectors with Pulsar topics. + +**Example 1** + +Below is a YAML configuration file of a Cassandra sink, which tells Pulsar: + +* Which Cassandra cluster to connect + +* What is the `keyspace` and `columnFamily` to be used in Cassandra for collecting data + +* How to map Pulsar messages into Cassandra table key and columns + +```shell + +tenant: public +namespace: default +name: cassandra-test-sink +... +# cassandra specific config +configs: + roots: "localhost:9042" + keyspace: "pulsar_test_keyspace" + columnFamily: "pulsar_test_table" + keyname: "key" + columnName: "col" + +``` + +**Example 2** + +Below is a YAML configuration file of a Kafka source. + +```shell + +configs: + bootstrapServers: "pulsar-kafka:9092" + groupId: "test-pulsar-io" + topic: "my-topic" + sessionTimeoutMs: "10000" + autoCommitEnabled: "false" + +``` + +**Example 3** + +Below is a YAML configuration file of a PostgreSQL JDBC sink. + +```shell + +configs: + userName: "postgres" + password: "password" + jdbcUrl: "jdbc:postgresql://localhost:5432/test_jdbc" + tableName: "test_jdbc" + +``` + +## Get available connectors + +Before starting using connectors, you can perform the following operations: + +* [Reload connectors](#reload) + +* [Get a list of available connectors](#get-available-connectors) + +### `reload` + +If you add or delete a nar file in a connector folder, reload the available builtin connector before using it. + +#### Source + +Use the `reload` subcommand. + +```shell + +$ pulsar-admin sources reload + +``` + +For more information, see [`here`](io-cli.md#reload). + +#### Sink + +Use the `reload` subcommand. + +```shell + +$ pulsar-admin sinks reload + +``` + +For more information, see [`here`](io-cli.md#reload-1). + +### `available` + +After reloading connectors (optional), you can get a list of available connectors. + +#### Source + +Use the `available-sources` subcommand. + +```shell + +$ pulsar-admin sources available-sources + +``` + +#### Sink + +Use the `available-sinks` subcommand. + +```shell + +$ pulsar-admin sinks available-sinks + +``` + +## Run a connector + +To run a connector, you can perform the following operations: + +* [Create a connector](#create) + +* [Start a connector](#start) + +* [Run a connector locally](#localrun) + +### `create` + +You can create a connector using **Admin CLI**, **REST API** or **JAVA admin API**.f + +#### Source + +Create a source connector. + + + + + +Use the `create` subcommand. + +``` + +$ pulsar-admin sources create options + +``` + +For more information, see [here](io-cli.md#create). + + + + +Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sources/:tenant/:namespace/:sourceName|operation/registerSource?version=@pulsar:version_number@} + + + + +* Create a source connector with a **local file**. + + ```java + + void createSource(SourceConfig sourceConfig, + String fileName) + throws PulsarAdminException + + ``` + + **Parameter** + + |Name|Description + |---|--- + `sourceConfig` | The source configuration object + + **Exception** + + |Name|Description| + |---|--- + | `PulsarAdminException` | Unexpected error + + For more information, see [`createSource`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Source.html#createSource-SourceConfig-java.lang.String-). + +* Create a source connector using a **remote file** with a URL from which fun-pkg can be downloaded. + + ```java + + void createSourceWithUrl(SourceConfig sourceConfig, + String pkgUrl) + throws PulsarAdminException + + ``` + + Supported URLs are `http` and `file`. + + **Example** + + * HTTP: http://www.repo.com/fileName.jar + + * File: file:///dir/fileName.jar + + **Parameter** + + Parameter| Description + |---|--- + `sourceConfig` | The source configuration object + `pkgUrl` | URL from which pkg can be downloaded + + **Exception** + + |Name|Description| + |---|--- + | `PulsarAdminException` | Unexpected error + + For more information, see [`createSourceWithUrl`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Source.html#createSourceWithUrl-SourceConfig-java.lang.String-). + + + + + +#### Sink + +Create a sink connector. + + + + + +Use the `create` subcommand. + +``` + +$ pulsar-admin sinks create options + +``` + +For more information, see [here](io-cli.md#create-1). + + + + +Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sinks/:tenant/:namespace/:sinkName|operation/registerSink?version=@pulsar:version_number@} + + + + +* Create a sink connector with a **local file**. + + ```java + + void createSink(SinkConfig sinkConfig, + String fileName) + throws PulsarAdminException + + ``` + + **Parameter** + + |Name|Description + |---|--- + `sinkConfig` | The sink configuration object + + **Exception** + + |Name|Description| + |---|--- + | `PulsarAdminException` | Unexpected error + + For more information, see [`createSink`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Sink.html#createSink-SinkConfig-java.lang.String-). + +* Create a sink connector using a **remote file** with a URL from which fun-pkg can be downloaded. + + ```java + + void createSinkWithUrl(SinkConfig sinkConfig, + String pkgUrl) + throws PulsarAdminException + + ``` + + Supported URLs are `http` and `file`. + + **Example** + + * HTTP: http://www.repo.com/fileName.jar + + * File: file:///dir/fileName.jar + + **Parameter** + + Parameter| Description + |---|--- + `sinkConfig` | The sink configuration object + `pkgUrl` | URL from which pkg can be downloaded + + **Exception** + + |Name|Description| + |---|--- + | `PulsarAdminException` | Unexpected error + + For more information, see [`createSinkWithUrl`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Sink.html#createSinkWithUrl-SinkConfig-java.lang.String-). + + + + + +### `start` + +You can start a connector using **Admin CLI** or **REST API**. + +#### Source + +Start a source connector. + + + + + +Use the `start` subcommand. + +``` + +$ pulsar-admin sources start options + +``` + +For more information, see [here](io-cli.md#start). + + + + +* Start **all** source connectors. + + Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sources/:tenant/:namespace/:sourceName/start|operation/startSource?version=@pulsar:version_number@} + +* Start a **specified** source connector. + + Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sources/:tenant/:namespace/:sourceName/:instanceId/start|operation/startSource?version=@pulsar:version_number@} + + + + + +#### Sink + +Start a sink connector. + + + + + +Use the `start` subcommand. + +``` + +$ pulsar-admin sinks start options + +``` + +For more information, see [here](io-cli.md#start-1). + + + + +* Start **all** sink connectors. + + Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sources/:tenant/:namespace/:sinkName/start|operation/startSink?version=@pulsar:version_number@} + +* Start a **specified** sink connector. + + Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sinks/:tenant/:namespace/:sourceName/:instanceId/start|operation/startSink?version=@pulsar:version_number@} + + + + + +### `localrun` + +You can run a connector locally rather than deploying it on a Pulsar cluster using **Admin CLI**. + +#### Source + +Run a source connector locally. + + + + + +Use the `localrun` subcommand. + +``` + +$ pulsar-admin sources localrun options + +``` + +For more information, see [here](io-cli.md#localrun). + + + + + +#### Sink + +Run a sink connector locally. + + + + + +Use the `localrun` subcommand. + +``` + +$ pulsar-admin sinks localrun options + +``` + +For more information, see [here](io-cli.md#localrun-1). + + + + + +## Monitor a connector + +To monitor a connector, you can perform the following operations: + +* [Get the information of a connector](#get) + +* [Get the list of all running connectors](#list) + +* [Get the current status of a connector](#status) + +### `get` + +You can get the information of a connector using **Admin CLI**, **REST API** or **JAVA admin API**. + +#### Source + +Get the information of a source connector. + + + + + +Use the `get` subcommand. + +``` + +$ pulsar-admin sources get options + +``` + +For more information, see [here](io-cli.md#get). + + + + +Send a `GET` request to this endpoint: {@inject: endpoint|GET|/admin/v3/sources/:tenant/:namespace/:sourceName|operation/getSourceInfo?version=@pulsar:version_number@} + + + + +```java + +SourceConfig getSource(String tenant, + String namespace, + String source) + throws PulsarAdminException + +``` + +**Example** + +This is a sourceConfig. + +```java + +{ + "tenant": "tenantName", + "namespace": "namespaceName", + "name": "sourceName", + "className": "className", + "topicName": "topicName", + "configs": {}, + "parallelism": 1, + "processingGuarantees": "ATLEAST_ONCE", + "resources": { + "cpu": 1.0, + "ram": 1073741824, + "disk": 10737418240 + } +} + +``` + +This is a sourceConfig example. + +``` + +{ + "tenant": "public", + "namespace": "default", + "name": "debezium-mysql-source", + "className": "org.apache.pulsar.io.debezium.mysql.DebeziumMysqlSource", + "topicName": "debezium-mysql-topic", + "configs": { + "database.user": "debezium", + "database.server.id": "184054", + "database.server.name": "dbserver1", + "database.port": "3306", + "database.hostname": "localhost", + "database.password": "dbz", + "database.history.pulsar.service.url": "pulsar://127.0.0.1:6650", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "database.whitelist": "inventory", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "database.history": "org.apache.pulsar.io.debezium.PulsarDatabaseHistory", + "pulsar.service.url": "pulsar://127.0.0.1:6650", + "database.history.pulsar.topic": "history-topic2" + }, + "parallelism": 1, + "processingGuarantees": "ATLEAST_ONCE", + "resources": { + "cpu": 1.0, + "ram": 1073741824, + "disk": 10737418240 + } +} + +``` + +**Exception** + +Exception name | Description +|---|--- +`PulsarAdminException.NotAuthorizedException` | You don't have the admin permission +`PulsarAdminException.NotFoundException` | Cluster doesn't exist +`PulsarAdminException` | Unexpected error + +For more information, see [`getSource`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Source.html#getSource-java.lang.String-java.lang.String-java.lang.String-). + + + + + +#### Sink + +Get the information of a sink connector. + + + + + +Use the `get` subcommand. + +``` + +$ pulsar-admin sinks get options + +``` + +For more information, see [here](io-cli.md#get-1). + + + + +Send a `GET` request to this endpoint: {@inject: endpoint|GET|/admin/v3/sinks/:tenant/:namespace/:sinkName|operation/getSinkInfo?version=@pulsar:version_number@} + + + + +```java + +SinkConfig getSink(String tenant, + String namespace, + String sink) + throws PulsarAdminException + +``` + +**Example** + +This is a sinkConfig. + +```json + +{ +"tenant": "tenantName", +"namespace": "namespaceName", +"name": "sinkName", +"className": "className", +"inputSpecs": { +"topicName": { + "isRegexPattern": false +} +}, +"configs": {}, +"parallelism": 1, +"processingGuarantees": "ATLEAST_ONCE", +"retainOrdering": false, +"autoAck": true +} + +``` + +This is a sinkConfig example. + +```json + +{ + "tenant": "public", + "namespace": "default", + "name": "pulsar-postgres-jdbc-sink", + "className": "org.apache.pulsar.io.jdbc.PostgresJdbcAutoSchemaSink", + "inputSpecs": { + "pulsar-postgres-jdbc-sink-topic": { + "isRegexPattern": false + } + }, + "configs": { + "password": "password", + "jdbcUrl": "jdbc:postgresql://localhost:5432/pulsar_postgres_jdbc_sink", + "userName": "postgres", + "tableName": "pulsar_postgres_jdbc_sink" + }, + "parallelism": 1, + "processingGuarantees": "ATLEAST_ONCE", + "retainOrdering": false, + "autoAck": true +} + +``` + +**Parameter description** + +Name| Description +|---|--- +`tenant` | Tenant name +`namespace` | Namespace name +`sink` | Sink name + +For more information, see [`getSink`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Sink.html#getSink-java.lang.String-java.lang.String-java.lang.String-). + + + + + +### `list` + +You can get the list of all running connectors using **Admin CLI**, **REST API** or **JAVA admin API**. + +#### Source + +Get the list of all running source connectors. + + + + + +Use the `list` subcommand. + +``` + +$ pulsar-admin sources list options + +``` + +For more information, see [here](io-cli.md#list). + + + + +Send a `GET` request to this endpoint: {@inject: endpoint|GET|/admin/v3/sources/:tenant/:namespace/|operation/listSources?version=@pulsar:version_number@} + + + + +```java + +List listSources(String tenant, + String namespace) + throws PulsarAdminException + +``` + +**Response example** + +```java + +["f1", "f2", "f3"] + +``` + +**Exception** + +Exception name | Description +|---|--- +`PulsarAdminException.NotAuthorizedException` | You don't have the admin permission +`PulsarAdminException` | Unexpected error + +For more information, see [`listSource`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Source.html#listSources-java.lang.String-java.lang.String-). + + + + + +#### Sink + +Get the list of all running sink connectors. + + + + + +Use the `list` subcommand. + +``` + +$ pulsar-admin sinks list options + +``` + +For more information, see [here](io-cli.md#list-1). + + + + +Send a `GET` request to this endpoint: {@inject: endpoint|GET|/admin/v3/sinks/:tenant/:namespace/|operation/listSinks?version=@pulsar:version_number@} + + + + +```java + +List listSinks(String tenant, + String namespace) + throws PulsarAdminException + +``` + +**Response example** + +```java + +["f1", "f2", "f3"] + +``` + +**Exception** + +Exception name | Description +|---|--- +`PulsarAdminException.NotAuthorizedException` | You don't have the admin permission +`PulsarAdminException` | Unexpected error + +For more information, see [`listSource`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Sink.html#listSinks-java.lang.String-java.lang.String-). + + + + + +### `status` + +You can get the current status of a connector using **Admin CLI**, **REST API** or **JAVA admin API**. + +#### Source + +Get the current status of a source connector. + + + + + +Use the `status` subcommand. + +``` + +$ pulsar-admin sources status options + +``` + +For more information, see [here](io-cli.md#status). + + + + +* Get the current status of **all** source connectors. + + Send a `GET` request to this endpoint: {@inject: endpoint|GET|/admin/v3/sources/:tenant/:namespace/:sourceName/status|operation/getSourceStatus?version=@pulsar:version_number@} + +* Gets the current status of a **specified** source connector. + + Send a `GET` request to this endpoint: {@inject: endpoint|GET|/admin/v3/sources/:tenant/:namespace/:sourceName/:instanceId/status|operation/getSourceStatus?version=@pulsar:version_number@} + + + + +* Get the current status of **all** source connectors. + + ```java + + SourceStatus getSourceStatus(String tenant, + String namespace, + String source) + throws PulsarAdminException + + ``` + + **Parameter** + + Parameter| Description + |---|--- + `tenant` | Tenant name + `namespace` | Namespace name + `sink` | Source name + + **Exception** + + Name | Description + |---|--- + `PulsarAdminException` | Unexpected error + + For more information, see [`getSourceStatus`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Source.html#getSource-java.lang.String-java.lang.String-java.lang.String-). + +* Gets the current status of a **specified** source connector. + + ```java + + SourceStatus.SourceInstanceStatus.SourceInstanceStatusData getSourceStatus(String tenant, + String namespace, + String source, + int id) + throws PulsarAdminException + + ``` + + **Parameter** + + Parameter| Description + |---|--- + `tenant` | Tenant name + `namespace` | Namespace name + `sink` | Source name + `id` | Source instanceID + + **Exception** + + Exception name | Description + |---|--- + `PulsarAdminException` | Unexpected error + + For more information, see [`getSourceStatus`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Source.html#getSourceStatus-java.lang.String-java.lang.String-java.lang.String-int-). + + + + + +#### Sink + +Get the current status of a Pulsar sink connector. + + + + + +Use the `status` subcommand. + +``` + +$ pulsar-admin sinks status options + +``` + +For more information, see [here](io-cli.md#status-1). + + + + +* Get the current status of **all** sink connectors. + + Send a `GET` request to this endpoint: {@inject: endpoint|GET|/admin/v3/sinks/:tenant/:namespace/:sinkName/status|operation/getSinkStatus?version=@pulsar:version_number@} + +* Gets the current status of a **specified** sink connector. + + Send a `GET` request to this endpoint: {@inject: endpoint|GET|/admin/v3/sinks/:tenant/:namespace/:sourceName/:instanceId/status|operation/getSinkInstanceStatus?version=@pulsar:version_number@} + + + + +* Get the current status of **all** sink connectors. + + ```java + + SinkStatus getSinkStatus(String tenant, + String namespace, + String sink) + throws PulsarAdminException + + ``` + + **Parameter** + + Parameter| Description + |---|--- + `tenant` | Tenant name + `namespace` | Namespace name + `sink` | Source name + + **Exception** + + Exception name | Description + |---|--- + `PulsarAdminException` | Unexpected error + + For more information, see [`getSinkStatus`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Sink.html#getSinkStatus-java.lang.String-java.lang.String-java.lang.String-). + +* Gets the current status of a **specified** source connector. + + ```java + + SinkStatus.SinkInstanceStatus.SinkInstanceStatusData getSinkStatus(String tenant, + String namespace, + String sink, + int id) + throws PulsarAdminException + + ``` + + **Parameter** + + Parameter| Description + |---|--- + `tenant` | Tenant name + `namespace` | Namespace name + `sink` | Source name + `id` | Sink instanceID + + **Exception** + + Exception name | Description + |---|--- + `PulsarAdminException` | Unexpected error + + For more information, see [`getSinkStatusWithInstanceID`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Sink.html#getSinkStatus-java.lang.String-java.lang.String-java.lang.String-int-). + + + + + +## Update a connector + +### `update` + +You can update a running connector using **Admin CLI**, **REST API** or **JAVA admin API**. + +#### Source + +Update a running Pulsar source connector. + + + + + +Use the `update` subcommand. + +``` + +$ pulsar-admin sources update options + +``` + +For more information, see [here](io-cli.md#update). + + + + +Send a `PUT` request to this endpoint: {@inject: endpoint|PUT|/admin/v3/sources/:tenant/:namespace/:sourceName|operation/updateSource?version=@pulsar:version_number@} + + + + +* Update a running source connector with a **local file**. + + ```java + + void updateSource(SourceConfig sourceConfig, + String fileName) + throws PulsarAdminException + + ``` + + **Parameter** + + | Name | Description + |---|--- + |`sourceConfig` | The source configuration object + + **Exception** + + |Name|Description| + |---|--- + |`PulsarAdminException.NotAuthorizedException`| You don't have the admin permission + | `PulsarAdminException.NotFoundException` | Cluster doesn't exist + | `PulsarAdminException` | Unexpected error + + For more information, see [`updateSource`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Source.html#updateSource-SourceConfig-java.lang.String-). + +* Update a source connector using a **remote file** with a URL from which fun-pkg can be downloaded. + + ```java + + void updateSourceWithUrl(SourceConfig sourceConfig, + String pkgUrl) + throws PulsarAdminException + + ``` + + Supported URLs are `http` and `file`. + + **Example** + + * HTTP: http://www.repo.com/fileName.jar + + * File: file:///dir/fileName.jar + + **Parameter** + + | Name | Description + |---|--- + | `sourceConfig` | The source configuration object + | `pkgUrl` | URL from which pkg can be downloaded + + **Exception** + + |Name|Description| + |---|--- + |`PulsarAdminException.NotAuthorizedException`| You don't have the admin permission + | `PulsarAdminException.NotFoundException` | Cluster doesn't exist + | `PulsarAdminException` | Unexpected error + +For more information, see [`createSourceWithUrl`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Source.html#updateSourceWithUrl-SourceConfig-java.lang.String-). + + + + + +#### Sink + +Update a running Pulsar sink connector. + + + + + +Use the `update` subcommand. + +``` + +$ pulsar-admin sinks update options + +``` + +For more information, see [here](io-cli.md#update-1). + + + + +Send a `PUT` request to this endpoint: {@inject: endpoint|PUT|/admin/v3/sinks/:tenant/:namespace/:sinkName|operation/updateSink?version=@pulsar:version_number@} + + + + +* Update a running sink connector with a **local file**. + + ```java + + void updateSink(SinkConfig sinkConfig, + String fileName) + throws PulsarAdminException + + ``` + + **Parameter** + + | Name | Description + |---|--- + |`sinkConfig` | The sink configuration object + + **Exception** + + |Name|Description| + |---|--- + |`PulsarAdminException.NotAuthorizedException`| You don't have the admin permission + | `PulsarAdminException.NotFoundException` | Cluster doesn't exist + | `PulsarAdminException` | Unexpected error + + For more information, see [`updateSink`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Sink.html#updateSink-SinkConfig-java.lang.String-). + +* Update a sink connector using a **remote file** with a URL from which fun-pkg can be downloaded. + + ```java + + void updateSinkWithUrl(SinkConfig sinkConfig, + String pkgUrl) + throws PulsarAdminException + + ``` + + Supported URLs are `http` and `file`. + + **Example** + + * HTTP: http://www.repo.com/fileName.jar + + * File: file:///dir/fileName.jar + + **Parameter** + + | Name | Description + |---|--- + | `sinkConfig` | The sink configuration object + | `pkgUrl` | URL from which pkg can be downloaded + + **Exception** + + |Name|Description| + |---|--- + |`PulsarAdminException.NotAuthorizedException`| You don't have the admin permission + |`PulsarAdminException.NotFoundException` | Cluster doesn't exist + |`PulsarAdminException` | Unexpected error + +For more information, see [`updateSinkWithUrl`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Sink.html#updateSinkWithUrl-SinkConfig-java.lang.String-). + + + + + +## Stop a connector + +### `stop` + +You can stop a connector using **Admin CLI**, **REST API** or **JAVA admin API**. + +#### Source + +Stop a source connector. + + + + + +Use the `stop` subcommand. + +``` + +$ pulsar-admin sources stop options + +``` + +For more information, see [here](io-cli.md#stop). + + + + +* Stop **all** source connectors. + + Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sources/:tenant/:namespace/:sourceName|operation/stopSource?version=@pulsar:version_number@} + +* Stop a **specified** source connector. + + Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sources/:tenant/:namespace/:sourceName/:instanceId|operation/stopSource?version=@pulsar:version_number@} + + + + +* Stop **all** source connectors. + + ```java + + void stopSource(String tenant, + String namespace, + String source) + throws PulsarAdminException + + ``` + + **Parameter** + + | Name | Description + |---|--- + `tenant` | Tenant name + `namespace` | Namespace name + `source` | Source name + + **Exception** + + |Name|Description| + |---|--- + | `PulsarAdminException` | Unexpected error + + For more information, see [`stopSource`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Source.html#stopSource-java.lang.String-java.lang.String-java.lang.String-). + +* Stop a **specified** source connector. + + ```java + + void stopSource(String tenant, + String namespace, + String source, + int instanceId) + throws PulsarAdminException + + ``` + + **Parameter** + + | Name | Description + |---|--- + `tenant` | Tenant name + `namespace` | Namespace name + `source` | Source name + `instanceId` | Source instanceID + + **Exception** + + |Name|Description| + |---|--- + | `PulsarAdminException` | Unexpected error + + For more information, see [`stopSource`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Source.html#stopSource-java.lang.String-java.lang.String-java.lang.String-int-). + + + + + +#### Sink + +Stop a sink connector. + + + + + +Use the `stop` subcommand. + +``` + +$ pulsar-admin sinks stop options + +``` + +For more information, see [here](io-cli.md#stop-1). + + + + +* Stop **all** sink connectors. + + Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sinks/:tenant/:namespace/:sinkName/stop|operation/stopSink?version=@pulsar:version_number@} + +* Stop a **specified** sink connector. + + Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sources/:tenant/:namespace/:sinkeName/:instanceId/stop|operation/stopSink?version=@pulsar:version_number@} + + + + +* Stop **all** sink connectors. + + ```java + + void stopSink(String tenant, + String namespace, + String sink) + throws PulsarAdminException + + ``` + + **Parameter** + + | Name | Description + |---|--- + `tenant` | Tenant name + `namespace` | Namespace name + `source` | Source name + + **Exception** + + |Name|Description| + |---|--- + | `PulsarAdminException` | Unexpected error + + For more information, see [`stopSink`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Sink.html#stopSink-java.lang.String-java.lang.String-java.lang.String-). + +* Stop a **specified** sink connector. + + ```java + + void stopSink(String tenant, + String namespace, + String sink, + int instanceId) + throws PulsarAdminException + + ``` + + **Parameter** + + | Name | Description + |---|--- + `tenant` | Tenant name + `namespace` | Namespace name + `source` | Source name + `instanceId` | Source instanceID + + **Exception** + + |Name|Description| + |---|--- + | `PulsarAdminException` | Unexpected error + + For more information, see [`stopSink`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Sink.html#stopSink-java.lang.String-java.lang.String-java.lang.String-int-). + + + + + +## Restart a connector + +### `restart` + +You can restart a connector using **Admin CLI**, **REST API** or **JAVA admin API**. + +#### Source + +Restart a source connector. + + + + + +Use the `restart` subcommand. + +``` + +$ pulsar-admin sources restart options + +``` + +For more information, see [here](io-cli.md#restart). + + + + +* Restart **all** source connectors. + + Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sources/:tenant/:namespace/:sourceName/restart|operation/restartSource?version=@pulsar:version_number@} + +* Restart a **specified** source connector. + + Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sources/:tenant/:namespace/:sourceName/:instanceId/restart|operation/restartSource?version=@pulsar:version_number@} + + + + +* Restart **all** source connectors. + + ```java + + void restartSource(String tenant, + String namespace, + String source) + throws PulsarAdminException + + ``` + + **Parameter** + + | Name | Description + |---|--- + `tenant` | Tenant name + `namespace` | Namespace name + `source` | Source name + + **Exception** + + |Name|Description| + |---|--- + | `PulsarAdminException` | Unexpected error + + For more information, see [`restartSource`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Source.html#restartSource-java.lang.String-java.lang.String-java.lang.String-). + +* Restart a **specified** source connector. + + ```java + + void restartSource(String tenant, + String namespace, + String source, + int instanceId) + throws PulsarAdminException + + ``` + + **Parameter** + + | Name | Description + |---|--- + `tenant` | Tenant name + `namespace` | Namespace name + `source` | Source name + `instanceId` | Source instanceID + + **Exception** + + |Name|Description| + |---|--- + | `PulsarAdminException` | Unexpected error + + For more information, see [`restartSource`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Source.html#restartSource-java.lang.String-java.lang.String-java.lang.String-int-). + + + + + +#### Sink + +Restart a sink connector. + + + + + +Use the `restart` subcommand. + +``` + +$ pulsar-admin sinks restart options + +``` + +For more information, see [here](io-cli.md#restart-1). + + + + +* Restart **all** sink connectors. + + Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sources/:tenant/:namespace/:sinkName/restart|operation/restartSource?version=@pulsar:version_number@} + +* Restart a **specified** sink connector. + + Send a `POST` request to this endpoint: {@inject: endpoint|POST|/admin/v3/sources/:tenant/:namespace/:sinkName/:instanceId/restart|operation/restartSource?version=@pulsar:version_number@} + + + + +* Restart all Pulsar sink connectors. + + ```java + + void restartSink(String tenant, + String namespace, + String sink) + throws PulsarAdminException + + ``` + + **Parameter** + + | Name | Description + |---|--- + `tenant` | Tenant name + `namespace` | Namespace name + `sink` | Sink name + + **Exception** + + |Name|Description| + |---|--- + | `PulsarAdminException` | Unexpected error + + For more information, see [`restartSink`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Sink.html#restartSink-java.lang.String-java.lang.String-java.lang.String-). + +* Restart a **specified** sink connector. + + ```java + + void restartSink(String tenant, + String namespace, + String sink, + int instanceId) + throws PulsarAdminException + + ``` + + **Parameter** + + | Name | Description + |---|--- + `tenant` | Tenant name + `namespace` | Namespace name + `source` | Source name + `instanceId` | Sink instanceID + + **Exception** + + |Name|Description| + |---|--- + | `PulsarAdminException` | Unexpected error + + For more information, see [`restartSink`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Sink.html#restartSink-java.lang.String-java.lang.String-java.lang.String-int-). + + + + + +## Delete a connector + +### `delete` + +You can delete a connector using **Admin CLI**, **REST API** or **JAVA admin API**. + +#### Source + +Delete a source connector. + + + + + +Use the `delete` subcommand. + +``` + +$ pulsar-admin sources delete options + +``` + +For more information, see [here](io-cli.md#delete). + + + + +Delete al Pulsar source connector. + +Send a `DELETE` request to this endpoint: {@inject: endpoint|DELETE|/admin/v3/sources/:tenant/:namespace/:sourceName|operation/deregisterSource?version=@pulsar:version_number@} + + + + +Delete a source connector. + +```java + +void deleteSource(String tenant, + String namespace, + String source) + throws PulsarAdminException + +``` + +**Parameter** + +| Name | Description +|---|--- +`tenant` | Tenant name +`namespace` | Namespace name +`source` | Source name + +**Exception** + +|Name|Description| +|---|--- +|`PulsarAdminException.NotAuthorizedException`| You don't have the admin permission +| `PulsarAdminException.NotFoundException` | Cluster doesn't exist +| `PulsarAdminException.PreconditionFailedException` | Cluster is not empty +| `PulsarAdminException` | Unexpected error + +For more information, see [`deleteSource`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Source.html#deleteSource-java.lang.String-java.lang.String-java.lang.String-). + + + + + +#### Sink + +Delete a sink connector. + + + + + +Use the `delete` subcommand. + +``` + +$ pulsar-admin sinks delete options + +``` + +For more information, see [here](io-cli.md#delete-1). + + + + +Delete a sink connector. + +Send a `DELETE` request to this endpoint: {@inject: endpoint|DELETE|/admin/v3/sinks/:tenant/:namespace/:sinkName|operation/deregisterSink?version=@pulsar:version_number@} + + + + +Delete a Pulsar sink connector. + +```java + +void deleteSink(String tenant, + String namespace, + String source) + throws PulsarAdminException + +``` + +**Parameter** + +| Name | Description +|---|--- +`tenant` | Tenant name +`namespace` | Namespace name +`sink` | Sink name + +**Exception** + +|Name|Description| +|---|--- +|`PulsarAdminException.NotAuthorizedException`| You don't have the admin permission +| `PulsarAdminException.NotFoundException` | Cluster doesn't exist +| `PulsarAdminException.PreconditionFailedException` | Cluster is not empty +| `PulsarAdminException` | Unexpected error + +For more information, see [`deleteSource`](https://pulsar.apache.org/api/admin/org/apache/pulsar/client/admin/Sink.html#deleteSink-java.lang.String-java.lang.String-java.lang.String-). + + + + diff --git a/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json b/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json index ac0022fc20e41..fb6d17d6e5134 100644 --- a/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json +++ b/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json @@ -127,6 +127,44 @@ "id": "version-2.7.1/window-functions-context" } ] + }, + { + "type": "category", + "label": "Pulsar IO", + "items": [ + { + "type": "doc", + "id": "version-2.7.1/io-overview" + }, + { + "type": "doc", + "id": "version-2.7.1/io-quickstart" + }, + { + "type": "doc", + "id": "version-2.7.1/io-use" + }, + { + "type": "doc", + "id": "version-2.7.1/io-debug" + }, + { + "type": "doc", + "id": "version-2.7.1/io-connectors" + }, + { + "type": "doc", + "id": "version-2.7.1/io-cdc" + }, + { + "type": "doc", + "id": "version-2.7.1/io-develop" + }, + { + "type": "doc", + "id": "version-2.7.1/io-cli" + } + ] } ] -} +} \ No newline at end of file From a190be5e4c024e256370e72a21555335b5e7bbc0 Mon Sep 17 00:00:00 2001 From: LiLi Date: Tue, 2 Nov 2021 10:07:20 +0800 Subject: [PATCH 2/5] [website][upgrade]feat: docs migration - 2.7.1 / sql Signed-off-by: LiLi --- .../sql-deployment-configurations.md | 193 +++++++++++++++++ .../version-2.7.1/sql-getting-started.md | 191 +++++++++++++++++ .../version-2.7.1/sql-overview.md | 22 ++ .../version-2.7.1/sql-rest-api.md | 196 ++++++++++++++++++ .../version-2.7.1-sidebars.json | 22 ++ 5 files changed, 624 insertions(+) create mode 100644 site2/website-next/versioned_docs/version-2.7.1/sql-deployment-configurations.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/sql-getting-started.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/sql-overview.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/sql-rest-api.md diff --git a/site2/website-next/versioned_docs/version-2.7.1/sql-deployment-configurations.md b/site2/website-next/versioned_docs/version-2.7.1/sql-deployment-configurations.md new file mode 100644 index 0000000000000..c1df08c917346 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/sql-deployment-configurations.md @@ -0,0 +1,193 @@ +--- +id: sql-deployment-configurations +title: Pulsar SQL configuration and deployment +sidebar_label: "Configuration and deployment" +original_id: sql-deployment-configurations +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +You can configure Presto Pulsar connector and deploy a cluster with the following instruction. + +## Configure Presto Pulsar Connector +You can configure Presto Pulsar Connector in the `${project.root}/conf/presto/catalog/pulsar.properties` properties file. The configuration for the connector and the default values are as follows. + +```properties + +# name of the connector to be displayed in the catalog +connector.name=pulsar + +# the url of Pulsar broker service +pulsar.broker-service-url=http://localhost:8080 + +# URI of Zookeeper cluster +pulsar.zookeeper-uri=localhost:2181 + +# minimum number of entries to read at a single time +pulsar.entry-read-batch-size=100 + +# default number of splits to use per query +pulsar.target-num-splits=4 + +``` + +You can connect Presto to a Pulsar cluster with multiple hosts. To configure multiple hosts for brokers, add multiple URLs to `pulsar.broker-service-url`. To configure multiple hosts for ZooKeeper, add multiple URIs to `pulsar.zookeeper-uri`. The following is an example. + +``` + +pulsar.broker-service-url=http://localhost:8080,localhost:8081,localhost:8082 +pulsar.zookeeper-uri=localhost1,localhost2:2181 + +``` + +## Query data from existing Presto clusters + +If you already have a Presto cluster, you can copy the Presto Pulsar connector plugin to your existing cluster. Download the archived plugin package with the following command. + +```bash + +$ wget pulsar:binary_release_url + +``` + +## Deploy a new cluster + +Since Pulsar SQL is powered by [Presto](https://prestosql.io), the configuration for deployment is the same for the Pulsar SQL worker. + +:::note + +For how to set up a standalone single node environment, refer to [Query data](sql-getting-started). + +::: + +You can use the same CLI args as the Presto launcher. + +```bash + +$ ./bin/pulsar sql-worker --help +Usage: launcher [options] command + +Commands: run, start, stop, restart, kill, status + +Options: + -h, --help show this help message and exit + -v, --verbose Run verbosely + --etc-dir=DIR Defaults to INSTALL_PATH/etc + --launcher-config=FILE + Defaults to INSTALL_PATH/bin/launcher.properties + --node-config=FILE Defaults to ETC_DIR/node.properties + --jvm-config=FILE Defaults to ETC_DIR/jvm.config + --config=FILE Defaults to ETC_DIR/config.properties + --log-levels-file=FILE + Defaults to ETC_DIR/log.properties + --data-dir=DIR Defaults to INSTALL_PATH + --pid-file=FILE Defaults to DATA_DIR/var/run/launcher.pid + --launcher-log-file=FILE + Defaults to DATA_DIR/var/log/launcher.log (only in + daemon mode) + --server-log-file=FILE + Defaults to DATA_DIR/var/log/server.log (only in + daemon mode) + -D NAME=VALUE Set a Java system property + +``` + +The default configuration for the cluster is located in `${project.root}/conf/presto`. You can customize your deployment by modifying the default configuration. + +You can set the worker to read from a different configuration directory, or set a different directory to write data. + +```bash + +$ ./bin/pulsar sql-worker run --etc-dir /tmp/incubator-pulsar/conf/presto --data-dir /tmp/presto-1 + +``` + +You can start the worker as daemon process. + +```bash + +$ ./bin/pulsar sql-worker start + +``` + +### Deploy a cluster on multiple nodes + +You can deploy a Pulsar SQL cluster or Presto cluster on multiple nodes. The following example shows how to deploy a cluster on three-node cluster. + +1. Copy the Pulsar binary distribution to three nodes. + +The first node runs as Presto coordinator. The minimal configuration requirement in the `${project.root}/conf/presto/config.properties` file is as follows. + +```properties + +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +query.max-memory=50GB +query.max-memory-per-node=1GB +discovery-server.enabled=true +discovery.uri= + +``` + +The other two nodes serve as worker nodes, you can use the following configuration for worker nodes. + +```properties + +coordinator=false +http-server.http.port=8080 +query.max-memory=50GB +query.max-memory-per-node=1GB +discovery.uri= + +``` + +2. Modify `pulsar.broker-service-url` and `pulsar.zookeeper-uri` configuration in the `${project.root}/conf/presto/catalog/pulsar.properties` file accordingly for the three nodes. + +3. Start the coordinator node. + +``` + +$ ./bin/pulsar sql-worker run + +``` + +4. Start worker nodes. + +``` + +$ ./bin/pulsar sql-worker run + +``` + +5. Start the SQL CLI and check the status of your cluster. + +```bash + +$ ./bin/pulsar sql --server + +``` + +6. Check the status of your nodes. + +```bash + +presto> SELECT * FROM system.runtime.nodes; + node_id | http_uri | node_version | coordinator | state +---------+-------------------------+--------------+-------------+-------- + 1 | http://192.168.2.1:8081 | testversion | true | active + 3 | http://192.168.2.2:8081 | testversion | false | active + 2 | http://192.168.2.3:8081 | testversion | false | active + +``` + +For more information about deployment in Presto, refer to [Presto deployment](https://prestosql.io/docs/current/installation/deployment.html). + +:::note + +The broker does not advance LAC, so when Pulsar SQL bypass broker to query data, it can only read entries up to the LAC that all the bookies learned. You can enable periodically write LAC on the broker by setting "bookkeeperExplicitLacIntervalInMills" in the broker.conf. + +::: + diff --git a/site2/website-next/versioned_docs/version-2.7.1/sql-getting-started.md b/site2/website-next/versioned_docs/version-2.7.1/sql-getting-started.md new file mode 100644 index 0000000000000..e8c4ecb101fdd --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/sql-getting-started.md @@ -0,0 +1,191 @@ +--- +id: sql-getting-started +title: Query data with Pulsar SQL +sidebar_label: "Query data" +original_id: sql-getting-started +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +Before querying data in Pulsar, you need to install Pulsar and built-in connectors. + +## Requirements +1. Install [Pulsar](getting-started-standalone.md#install-pulsar-standalone). +2. Install Pulsar [built-in connectors](getting-started-standalone.md#install-builtin-connectors-optional). + +## Query data in Pulsar +To query data in Pulsar with Pulsar SQL, complete the following steps. + +1. Start a Pulsar standalone cluster. + +```bash + +./bin/pulsar standalone + +``` + +2. Start a Pulsar SQL worker. + +```bash + +./bin/pulsar sql-worker run + +``` + +3. After initializing Pulsar standalone cluster and the SQL worker, run SQL CLI. + +```bash + +./bin/pulsar sql + +``` + +4. Test with SQL commands. + +```bash + +presto> show catalogs; + Catalog +--------- + pulsar + system +(2 rows) + +Query 20180829_211752_00004_7qpwh, FINISHED, 1 node +Splits: 19 total, 19 done (100.00%) +0:00 [0 rows, 0B] [0 rows/s, 0B/s] + + +presto> show schemas in pulsar; + Schema +----------------------- + information_schema + public/default + public/functions + sample/standalone/ns1 +(4 rows) + +Query 20180829_211818_00005_7qpwh, FINISHED, 1 node +Splits: 19 total, 19 done (100.00%) +0:00 [4 rows, 89B] [21 rows/s, 471B/s] + + +presto> show tables in pulsar."public/default"; + Table +------- +(0 rows) + +Query 20180829_211839_00006_7qpwh, FINISHED, 1 node +Splits: 19 total, 19 done (100.00%) +0:00 [0 rows, 0B] [0 rows/s, 0B/s] + +``` + +Since there is no data in Pulsar, no records is returned. + +5. Start the built-in connector _DataGeneratorSource_ and ingest some mock data. + +```bash + +./bin/pulsar-admin sources create --name generator --destinationTopicName generator_test --source-type data-generator + +``` + +And then you can query a topic in the namespace "public/default". + +```bash + +presto> show tables in pulsar."public/default"; + Table +---------------- + generator_test +(1 row) + +Query 20180829_213202_00000_csyeu, FINISHED, 1 node +Splits: 19 total, 19 done (100.00%) +0:02 [1 rows, 38B] [0 rows/s, 17B/s] + +``` + +You can now query the data within the topic "generator_test". + +```bash + +presto> select * from pulsar."public/default".generator_test; + + firstname | middlename | lastname | email | username | password | telephonenumber | age | companyemail | nationalidentitycardnumber | +-------------+-------------+-------------+----------------------------------+--------------+----------+-----------------+-----+-----------------------------------------------+----------------------------+ + Genesis | Katherine | Wiley | genesis.wiley@gmail.com | genesisw | y9D2dtU3 | 959-197-1860 | 71 | genesis.wiley@interdemconsulting.eu | 880-58-9247 | + Brayden | | Stanton | brayden.stanton@yahoo.com | braydens | ZnjmhXik | 220-027-867 | 81 | brayden.stanton@supermemo.eu | 604-60-7069 | + Benjamin | Julian | Velasquez | benjamin.velasquez@yahoo.com | benjaminv | 8Bc7m3eb | 298-377-0062 | 21 | benjamin.velasquez@hostesltd.biz | 213-32-5882 | + Michael | Thomas | Donovan | donovan@mail.com | michaeld | OqBm9MLs | 078-134-4685 | 55 | michael.donovan@memortech.eu | 443-30-3442 | + Brooklyn | Avery | Roach | brooklynroach@yahoo.com | broach | IxtBLafO | 387-786-2998 | 68 | brooklyn.roach@warst.biz | 085-88-3973 | + Skylar | | Bradshaw | skylarbradshaw@yahoo.com | skylarb | p6eC6cKy | 210-872-608 | 96 | skylar.bradshaw@flyhigh.eu | 453-46-0334 | +. +. +. + +``` + +You can query the mock data. + +## Query your own data +If you want to query your own data, you need to ingest your own data first. You can write a simple producer and write custom defined data to Pulsar. The following is an example. + +```java + +public class TestProducer { + + public static class Foo { + private int field1 = 1; + private String field2; + private long field3; + + public Foo() { + } + + public int getField1() { + return field1; + } + + public void setField1(int field1) { + this.field1 = field1; + } + + public String getField2() { + return field2; + } + + public void setField2(String field2) { + this.field2 = field2; + } + + public long getField3() { + return field3; + } + + public void setField3(long field3) { + this.field3 = field3; + } + } + + public static void main(String[] args) throws Exception { + PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); + Producer producer = pulsarClient.newProducer(AvroSchema.of(Foo.class)).topic("test_topic").create(); + + for (int i = 0; i < 1000; i++) { + Foo foo = new Foo(); + foo.setField1(i); + foo.setField2("foo" + i); + foo.setField3(System.currentTimeMillis()); + producer.newMessage().value(foo).send(); + } + producer.close(); + pulsarClient.close(); + } +} + +``` + diff --git a/site2/website-next/versioned_docs/version-2.7.1/sql-overview.md b/site2/website-next/versioned_docs/version-2.7.1/sql-overview.md new file mode 100644 index 0000000000000..752530924d9f0 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/sql-overview.md @@ -0,0 +1,22 @@ +--- +id: sql-overview +title: Pulsar SQL Overview +sidebar_label: "Overview" +original_id: sql-overview +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +Apache Pulsar is used to store streams of event data, and the event data is structured with predefined fields. With the implementation of the [Schema Registry](schema-get-started), you can store structured data in Pulsar and query the data by using [Presto](https://prestosql.io/). + +As the core of Pulsar SQL, Presto Pulsar connector enables Presto workers within a Presto cluster to query data from Pulsar. + +![The Pulsar consumer and reader interfaces](/assets/pulsar-sql-arch-2.png) + +The query performance is efficient and highly scalable, because Pulsar adopts [two level segment based architecture](concepts-architecture-overview.md#apache-bookkeeper). + +Topics in Pulsar are stored as segments in [Apache BookKeeper](https://bookkeeper.apache.org/). Each topic segment is replicated to some BookKeeper nodes, which enables concurrent reads and high read throughput. You can configure the number of BookKeeper nodes, and the default number is `3`. In Presto Pulsar connector, data is read directly from BookKeeper, so Presto workers can read concurrently from horizontally scalable number BookKeeper nodes. + +![The Pulsar consumer and reader interfaces](/assets/pulsar-sql-arch-1.png) \ No newline at end of file diff --git a/site2/website-next/versioned_docs/version-2.7.1/sql-rest-api.md b/site2/website-next/versioned_docs/version-2.7.1/sql-rest-api.md new file mode 100644 index 0000000000000..d2b078793efd0 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/sql-rest-api.md @@ -0,0 +1,196 @@ +--- +id: sql-rest-api +title: Pulsar SQL REST APIs +sidebar_label: "REST APIs" +original_id: sql-rest-api +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +This section lists resources that make up the Presto REST API v1. + +## Request for Presto services + +All requests for Presto services should use Presto REST API v1 version. + +To request services, use explicit URL `http://presto.service:8081/v1`. You need to update `presto.service:8081` with your real Presto address before sending requests. + +`POST` requests require the `X-Presto-User` header. If you use authentication, you must use the same `username` that is specified in the authentication configuration. If you do not use authentication, you can specify anything for `username`. + +```properties + +X-Presto-User: username + +``` + +For more information about headers, refer to [PrestoHeaders](https://github.com/trinodb/trino). + +## Schema + +You can use statement in the HTTP body. All data is received as JSON document that might contain a `nextUri` link. If the received JSON document contains a `nextUri` link, the request continues with the `nextUri` link until the received data does not contain a `nextUri` link. If no error is returned, the query completes successfully. If an `error` field is displayed in `stats`, it means the query fails. + +The following is an example of `show catalogs`. The query continues until the received JSON document does not contain a `nextUri` link. Since no `error` is displayed in `stats`, it means that the query completes successfully. + +```powershell + +➜ ~ curl --header "X-Presto-User: test-user" --request POST --data 'show catalogs' http://localhost:8081/v1/statement +{ + "infoUri" : "http://localhost:8081/ui/query.html?20191113_033653_00006_dg6hb", + "stats" : { + "queued" : true, + "nodes" : 0, + "userTimeMillis" : 0, + "cpuTimeMillis" : 0, + "wallTimeMillis" : 0, + "processedBytes" : 0, + "processedRows" : 0, + "runningSplits" : 0, + "queuedTimeMillis" : 0, + "queuedSplits" : 0, + "completedSplits" : 0, + "totalSplits" : 0, + "scheduled" : false, + "peakMemoryBytes" : 0, + "state" : "QUEUED", + "elapsedTimeMillis" : 0 + }, + "id" : "20191113_033653_00006_dg6hb", + "nextUri" : "http://localhost:8081/v1/statement/20191113_033653_00006_dg6hb/1" +} + +➜ ~ curl http://localhost:8081/v1/statement/20191113_033653_00006_dg6hb/1 +{ + "infoUri" : "http://localhost:8081/ui/query.html?20191113_033653_00006_dg6hb", + "nextUri" : "http://localhost:8081/v1/statement/20191113_033653_00006_dg6hb/2", + "id" : "20191113_033653_00006_dg6hb", + "stats" : { + "state" : "PLANNING", + "totalSplits" : 0, + "queued" : false, + "userTimeMillis" : 0, + "completedSplits" : 0, + "scheduled" : false, + "wallTimeMillis" : 0, + "runningSplits" : 0, + "queuedSplits" : 0, + "cpuTimeMillis" : 0, + "processedRows" : 0, + "processedBytes" : 0, + "nodes" : 0, + "queuedTimeMillis" : 1, + "elapsedTimeMillis" : 2, + "peakMemoryBytes" : 0 + } +} + +➜ ~ curl http://localhost:8081/v1/statement/20191113_033653_00006_dg6hb/2 +{ + "id" : "20191113_033653_00006_dg6hb", + "data" : [ + [ + "pulsar" + ], + [ + "system" + ] + ], + "infoUri" : "http://localhost:8081/ui/query.html?20191113_033653_00006_dg6hb", + "columns" : [ + { + "typeSignature" : { + "rawType" : "varchar", + "arguments" : [ + { + "kind" : "LONG_LITERAL", + "value" : 6 + } + ], + "literalArguments" : [], + "typeArguments" : [] + }, + "name" : "Catalog", + "type" : "varchar(6)" + } + ], + "stats" : { + "wallTimeMillis" : 104, + "scheduled" : true, + "userTimeMillis" : 14, + "progressPercentage" : 100, + "totalSplits" : 19, + "nodes" : 1, + "cpuTimeMillis" : 16, + "queued" : false, + "queuedTimeMillis" : 1, + "state" : "FINISHED", + "peakMemoryBytes" : 0, + "elapsedTimeMillis" : 111, + "processedBytes" : 0, + "processedRows" : 0, + "queuedSplits" : 0, + "rootStage" : { + "cpuTimeMillis" : 1, + "runningSplits" : 0, + "state" : "FINISHED", + "completedSplits" : 1, + "subStages" : [ + { + "cpuTimeMillis" : 14, + "runningSplits" : 0, + "state" : "FINISHED", + "completedSplits" : 17, + "subStages" : [ + { + "wallTimeMillis" : 7, + "subStages" : [], + "stageId" : "2", + "done" : true, + "nodes" : 1, + "totalSplits" : 1, + "processedBytes" : 22, + "processedRows" : 2, + "queuedSplits" : 0, + "userTimeMillis" : 1, + "cpuTimeMillis" : 1, + "runningSplits" : 0, + "state" : "FINISHED", + "completedSplits" : 1 + } + ], + "wallTimeMillis" : 92, + "nodes" : 1, + "done" : true, + "stageId" : "1", + "userTimeMillis" : 12, + "processedRows" : 2, + "processedBytes" : 51, + "queuedSplits" : 0, + "totalSplits" : 17 + } + ], + "wallTimeMillis" : 5, + "done" : true, + "nodes" : 1, + "stageId" : "0", + "userTimeMillis" : 1, + "processedRows" : 2, + "processedBytes" : 22, + "totalSplits" : 1, + "queuedSplits" : 0 + }, + "runningSplits" : 0, + "completedSplits" : 19 + } +} + +``` + +:::note + +Since the response data is not in sync with the query state from the perspective of clients, you cannot rely on the response data to determine whether the query completes. + +::: + +For more information about Presto REST API, refer to [Presto HTTP Protocol](https://github.com/prestosql/presto/wiki/HTTP-Protocol). diff --git a/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json b/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json index fb6d17d6e5134..68614e5e6b3f2 100644 --- a/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json +++ b/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json @@ -165,6 +165,28 @@ "id": "version-2.7.1/io-cli" } ] + }, + { + "type": "category", + "label": "Pulsar SQL", + "items": [ + { + "type": "doc", + "id": "version-2.7.1/sql-overview" + }, + { + "type": "doc", + "id": "version-2.7.1/sql-getting-started" + }, + { + "type": "doc", + "id": "version-2.7.1/sql-deployment-configurations" + }, + { + "type": "doc", + "id": "version-2.7.1/sql-rest-api" + } + ] } ] } \ No newline at end of file From 37e05b683a47c7c01d756f8dd913cc012ee230cf Mon Sep 17 00:00:00 2001 From: LiLi Date: Tue, 2 Nov 2021 10:08:39 +0800 Subject: [PATCH 3/5] [website][upgrade]feat: docs migration - 2.7.1 / tiered-storage Signed-off-by: LiLi --- .../version-2.7.1/tiered-storage-aws.md | 333 ++++++++++++++++++ .../version-2.7.1/tiered-storage-azure.md | 268 ++++++++++++++ .../tiered-storage-filesystem.md | 319 +++++++++++++++++ .../version-2.7.1/tiered-storage-gcs.md | 323 +++++++++++++++++ .../version-2.7.1/tiered-storage-overview.md | 55 +++ .../version-2.7.1-sidebars.json | 26 ++ 6 files changed, 1324 insertions(+) create mode 100644 site2/website-next/versioned_docs/version-2.7.1/tiered-storage-aws.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/tiered-storage-azure.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/tiered-storage-filesystem.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/tiered-storage-gcs.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/tiered-storage-overview.md diff --git a/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-aws.md b/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-aws.md new file mode 100644 index 0000000000000..9ddfcebac7750 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-aws.md @@ -0,0 +1,333 @@ +--- +id: tiered-storage-aws +title: Use AWS S3 offloader with Pulsar +sidebar_label: "AWS S3 offloader" +original_id: tiered-storage-aws +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +This chapter guides you through every step of installing and configuring the AWS S3 offloader and using it with Pulsar. + +## Installation + +Follow the steps below to install the AWS S3 offloader. + +### Prerequisite + +- Pulsar: 2.4.2 or later versions + +### Step + +This example uses Pulsar 2.5.1. + +1. Download the Pulsar tarball using one of the following ways: + + * Download from the [Apache mirror](https://archive.apache.org/dist/pulsar/pulsar-2.5.1/apache-pulsar-2.5.1-bin.tar.gz) + + * Download from the Pulsar [downloads page](https://pulsar.apache.org/download) + + * Use [wget](https://www.gnu.org/software/wget): + + ```shell + + wget https://archive.apache.org/dist/pulsar/pulsar-2.5.1/apache-pulsar-2.5.1-bin.tar.gz + + ``` + +2. Download and untar the Pulsar offloaders package. + + ```bash + + wget https://downloads.apache.org/pulsar/pulsar-2.5.1/apache-pulsar-offloaders-2.5.1-bin.tar.gz + tar xvfz apache-pulsar-offloaders-2.5.1-bin.tar.gz + + ``` + +3. Copy the Pulsar offloaders as `offloaders` in the Pulsar directory. + + ``` + + mv apache-pulsar-offloaders-2.5.1/offloaders apache-pulsar-2.5.1/offloaders + + ls offloaders + + ``` + + **Output** + + As shown from the output, Pulsar uses [Apache jclouds](https://jclouds.apache.org) to support [AWS S3](https://aws.amazon.com/s3/) and [GCS](https://cloud.google.com/storage/) for long term storage. + + ``` + + tiered-storage-file-system-2.5.1.nar + tiered-storage-jcloud-2.5.1.nar + + ``` + + :::note + + * If you are running Pulsar in a bare metal cluster, make sure that `offloaders` tarball is unzipped in every broker's Pulsar directory. + * If you are running Pulsar in Docker or deploying Pulsar using a Docker image (such as K8s and DCOS), you can use the `apachepulsar/pulsar-all` image instead of the `apachepulsar/pulsar` image. `apachepulsar/pulsar-all` image has already bundled tiered storage offloaders. + + ::: + +## Configuration + +:::note + +Before offloading data from BookKeeper to AWS S3, you need to configure some properties of the AWS S3 offload driver. + +::: + +Besides, you can also configure the AWS S3 offloader to run it automatically or trigger it manually. + +### Configure AWS S3 offloader driver + +You can configure the AWS S3 offloader driver in the configuration file `broker.conf` or `standalone.conf`. + +- **Required** configurations are as below. + + Required configuration | Description | Example value + |---|---|--- + `managedLedgerOffloadDriver` | Offloader driver name, which is case-insensitive.

    **Note**: there is a third driver type, S3, which is identical to AWS S3, though S3 requires that you specify an endpoint URL using `s3ManagedLedgerOffloadServiceEndpoint`. This is useful if using an S3 compatible data store other than AWS S3. | aws-s3 + `offloadersDirectory` | Offloader directory | offloaders + `s3ManagedLedgerOffloadBucket` | Bucket | pulsar-topic-offload + +- **Optional** configurations are as below. + + Optional | Description | Example value + |---|---|--- + `s3ManagedLedgerOffloadRegion` | Bucket region

    **Note**: before specifying a value for this parameter, you need to set the following configurations. Otherwise, you might get an error.

    - Set [`s3ManagedLedgerOffloadServiceEndpoint`](https://docs.aws.amazon.com/general/latest/gr/s3.html).

    Example
    `s3ManagedLedgerOffloadServiceEndpoint=https://s3.YOUR_REGION.amazonaws.com`

    - Grant `GetBucketLocation` permission to a user.

    For how to grant `GetBucketLocation` permission to a user, see [here](https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-buckets).| eu-west-3 + `s3ManagedLedgerOffloadReadBufferSizeInBytes`|Size of block read|1 MB + `s3ManagedLedgerOffloadMaxBlockSizeInBytes`|Size of block write|64 MB + `managedLedgerMinLedgerRolloverTimeMinutes`|Minimum time between ledger rollover for a topic

    **Note**: it is not recommended that you set this configuration in the production environment.|2 + `managedLedgerMaxEntriesPerLedger`|Maximum number of entries to append to a ledger before triggering a rollover.

    **Note**: it is not recommended that you set this configuration in the production environment.|5000 + +#### Bucket (required) + +A bucket is a basic container that holds your data. Everything you store in AWS S3 must be contained in a bucket. You can use a bucket to organize your data and control access to your data, but unlike directory and folder, you cannot nest a bucket. + +##### Example + +This example names the bucket as _pulsar-topic-offload_. + +```conf + +s3ManagedLedgerOffloadBucket=pulsar-topic-offload + +``` + +#### Bucket region + +A bucket region is a region where a bucket is located. If a bucket region is not specified, the **default** region (`US East (N. Virginia)`) is used. + +:::tip + +For more information about AWS regions and endpoints, see [here](https://docs.aws.amazon.com/general/latest/gr/rande.html). + +::: + + +##### Example + +This example sets the bucket region as _europe-west-3_. + +``` + +s3ManagedLedgerOffloadRegion=eu-west-3 + +``` + +#### Authentication (required) + +To be able to access AWS S3, you need to authenticate with AWS S3. + +Pulsar does not provide any direct methods of configuring authentication for AWS S3, +but relies on the mechanisms supported by the [DefaultAWSCredentialsProviderChain](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html). + +Once you have created a set of credentials in the AWS IAM console, you can configure credentials using one of the following methods. + +* Use EC2 instance metadata credentials. + + If you are on AWS instance with an instance profile that provides credentials, Pulsar uses these credentials if no other mechanism is provided. + +* Set the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in `conf/pulsar_env.sh`. + + "export" is important so that the variables are made available in the environment of spawned processes. + + ```bash + + export AWS_ACCESS_KEY_ID=ABC123456789 + export AWS_SECRET_ACCESS_KEY=ded7db27a4558e2ea8bbf0bf37ae0e8521618f366c + + ``` + +* Add the Java system properties `aws.accessKeyId` and `aws.secretKey` to `PULSAR_EXTRA_OPTS` in `conf/pulsar_env.sh`. + + ```bash + + PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS} ${PULSAR_MEM} ${PULSAR_GC} -Daws.accessKeyId=ABC123456789 -Daws.secretKey=ded7db27a4558e2ea8bbf0bf37ae0e8521618f366c -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.maxCapacity.default=1000 -Dio.netty.recycler.linkCapacity=1024" + + ``` + +* Set the access credentials in `~/.aws/credentials`. + + ```conf + + [default] + aws_access_key_id=ABC123456789 + aws_secret_access_key=ded7db27a4558e2ea8bbf0bf37ae0e8521618f366c + + ``` + +* Assume an IAM role. + + This example uses the `DefaultAWSCredentialsProviderChain` for assuming this role. + + The broker must be rebooted for credentials specified in `pulsar_env` to take effect. + + ```conf + + s3ManagedLedgerOffloadRole= + s3ManagedLedgerOffloadRoleSessionName=pulsar-s3-offload + + ``` + +#### Size of block read/write + +You can configure the size of a request sent to or read from AWS S3 in the configuration file `broker.conf` or `standalone.conf`. + +Configuration|Description|Default value +|---|---|--- +`s3ManagedLedgerOffloadReadBufferSizeInBytes`|Block size for each individual read when reading back data from AWS S3.|1 MB +`s3ManagedLedgerOffloadMaxBlockSizeInBytes`|Maximum size of a "part" sent during a multipart upload to AWS S3. It **cannot** be smaller than 5 MB. |64 MB + +### Configure AWS S3 offloader to run automatically + +Namespace policy can be configured to offload data automatically once a threshold is reached. The threshold is based on the size of data that a topic has stored on a Pulsar cluster. Once the topic reaches the threshold, an offloading operation is triggered automatically. + +Threshold value|Action +|---|--- +> 0 | It triggers the offloading operation if the topic storage reaches its threshold. += 0|It causes a broker to offload data as soon as possible. +< 0 |It disables automatic offloading operation. + +Automatic offloading runs when a new segment is added to a topic log. If you set the threshold on a namespace, but few messages are being produced to the topic, offloader does not work until the current segment is full. + +You can configure the threshold size using CLI tools, such as pulsar-admin. + +The offload configurations in `broker.conf` and `standalone.conf` are used for the namespaces that do not have namespace level offload policies. Each namespace can have its own offload policy. If you want to set offload policy for each namespace, use the command [`pulsar-admin namespaces set-offload-policies options`](https://pulsar.apache.org/tools/pulsar-admin/2.6.0-SNAPSHOT/#-em-set-offload-policies-em-) command. + +#### Example + +This example sets the AWS S3 offloader threshold size to 10 MB using pulsar-admin. + +```bash + +bin/pulsar-admin namespaces set-offload-threshold --size 10M my-tenant/my-namespace + +``` + +:::tip + +For more information about the `pulsar-admin namespaces set-offload-threshold options` command, including flags, descriptions, and default values, see [here](https://pulsar.apache.org/tools/pulsar-admin/2.6.0-SNAPSHOT/#-em-set-offload-threshold-em-). + +::: + +### Configure AWS S3 offloader to run manually + +For individual topics, you can trigger AWS S3 offloader manually using one of the following methods: + +- Use REST endpoint. + +- Use CLI tools (such as pulsar-admin). + + To trigger it via CLI tools, you need to specify the maximum amount of data (threshold) that should be retained on a Pulsar cluster for a topic. If the size of the topic data on the Pulsar cluster exceeds this threshold, segments from the topic are moved to AWS S3 until the threshold is no longer exceeded. Older segments are moved first. + +#### Example + +- This example triggers the AWS S3 offloader to run manually using pulsar-admin. + + ```bash + + bin/pulsar-admin topics offload --size-threshold 10M my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ```bash + + Offload triggered for persistent://my-tenant/my-namespace/topic1 for messages before 2:0:-1 + + ``` + + :::tip + + For more information about the `pulsar-admin topics offload options` command, including flags, descriptions, and default values, see [here](https://pulsar.apache.org/tools/pulsar-admin/2.6.0-SNAPSHOT/#-em-offload-em-). + + ::: + +- This example checks the AWS S3 offloader status using pulsar-admin. + + ```bash + + bin/pulsar-admin topics offload-status persistent://my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ```bash + + Offload is currently running + + ``` + + To wait for the AWS S3 offloader to complete the job, add the `-w` flag. + + ```bash + + bin/pulsar-admin topics offload-status -w persistent://my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ``` + + Offload was a success + + ``` + + If there is an error in offloading, the error is propagated to the `pulsar-admin topics offload-status` command. + + ```bash + + bin/pulsar-admin topics offload-status persistent://my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ``` + + Error in offload + null + + Reason: Error offloading: org.apache.bookkeeper.mledger.ManagedLedgerException: java.util.concurrent.CompletionException: com.amazonaws.services.s3.model.AmazonS3Exception: Anonymous users cannot initiate multipart uploads. Please authenticate. (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied; Request ID: 798758DE3F1776DF; S3 Extended Request ID: dhBFz/lZm1oiG/oBEepeNlhrtsDlzoOhocuYMpKihQGXe6EG8puRGOkK6UwqzVrMXTWBxxHcS+g=), S3 Extended Request ID: dhBFz/lZm1oiG/oBEepeNlhrtsDlzoOhocuYMpKihQGXe6EG8puRGOkK6UwqzVrMXTWBxxHcS+g= + + ``` + + :::tip + + For more information about the `pulsar-admin topics offload-status options` command, including flags, descriptions, and default values, see [here](https://pulsar.apache.org/tools/pulsar-admin/2.6.0-SNAPSHOT/#-em-offload-status-em-). + + ::: + +## Tutorial + +For the complete and step-by-step instructions on how to use the AWS S3 offloader with Pulsar, see [here](https://hub.streamnative.io/offloaders/aws-s3/2.5.1#usage). \ No newline at end of file diff --git a/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-azure.md b/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-azure.md new file mode 100644 index 0000000000000..13824394e2cc3 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-azure.md @@ -0,0 +1,268 @@ +--- +id: tiered-storage-azure +title: Use Azure BlobStore offloader with Pulsar +sidebar_label: "Azure BlobStore offloader" +original_id: tiered-storage-azure +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +This chapter guides you through every step of installing and configuring the Azure BlobStore offloader and using it with Pulsar. + +## Installation + +Follow the steps below to install the Azure BlobStore offloader. + +### Prerequisite + +- Pulsar: 2.6.2 or later versions + +### Step + +This example uses Pulsar 2.6.2. + +1. Download the Pulsar tarball using one of the following ways: + + * Download from the [Apache mirror](https://archive.apache.org/dist/pulsar/pulsar-2.6.2/apache-pulsar-2.6.2-bin.tar.gz) + + * Download from the Pulsar [downloads page](https://pulsar.apache.org/download) + + * Use [wget](https://www.gnu.org/software/wget): + + ```shell + + wget https://archive.apache.org/dist/pulsar/pulsar-2.6.2/apache-pulsar-2.6.2-bin.tar.gz + + ``` + +2. Download and untar the Pulsar offloaders package. + + ```bash + + wget https://downloads.apache.org/pulsar/pulsar-2.6.2/apache-pulsar-offloaders-2.6.2-bin.tar.gz + tar xvfz apache-pulsar-offloaders-2.6.2-bin.tar.gz + + ``` + +3. Copy the Pulsar offloaders as `offloaders` in the Pulsar directory. + + ``` + + mv apache-pulsar-offloaders-2.6.2/offloaders apache-pulsar-2.6.2/offloaders + + ls offloaders + + ``` + + **Output** + + As shown from the output, Pulsar uses [Apache jclouds](https://jclouds.apache.org) to support [AWS S3](https://aws.amazon.com/s3/), [GCS](https://cloud.google.com/storage/) and [Azure](https://portal.azure.com/#home) for long term storage. + + ``` + + tiered-storage-file-system-2.6.2.nar + tiered-storage-jcloud-2.6.2.nar + + ``` + + :::note + + * If you are running Pulsar in a bare metal cluster, make sure that `offloaders` tarball is unzipped in every broker's Pulsar directory. + * If you are running Pulsar in Docker or deploying Pulsar using a Docker image (such as K8s and DCOS), you can use the `apachepulsar/pulsar-all` image instead of the `apachepulsar/pulsar` image. `apachepulsar/pulsar-all` image has already bundled tiered storage offloaders. + + ::: + +## Configuration + +:::note + +Before offloading data from BookKeeper to Azure BlobStore, you need to configure some properties of the Azure BlobStore offload driver. + +::: + +Besides, you can also configure the Azure BlobStore offloader to run it automatically or trigger it manually. + +### Configure Azure BlobStore offloader driver + +You can configure the Azure BlobStore offloader driver in the configuration file `broker.conf` or `standalone.conf`. + +- **Required** configurations are as below. + + Required configuration | Description | Example value + |---|---|--- + `managedLedgerOffloadDriver` | Offloader driver name | azureblob + `offloadersDirectory` | Offloader directory | offloaders + `managedLedgerOffloadBucket` | Bucket | pulsar-topic-offload + +- **Optional** configurations are as below. + + Optional | Description | Example value + |---|---|--- + `managedLedgerOffloadReadBufferSizeInBytes`|Size of block read|1 MB + `managedLedgerOffloadMaxBlockSizeInBytes`|Size of block write|64 MB + `managedLedgerMinLedgerRolloverTimeMinutes`|Minimum time between ledger rollover for a topic

    **Note**: it is not recommended that you set this configuration in the production environment.|2 + `managedLedgerMaxEntriesPerLedger`|Maximum number of entries to append to a ledger before triggering a rollover.

    **Note**: it is not recommended that you set this configuration in the production environment.|5000 + +#### Bucket (required) + +A bucket is a basic container that holds your data. Everything you store in Azure BlobStore must be contained in a bucket. You can use a bucket to organize your data and control access to your data, but unlike directory and folder, you cannot nest a bucket. + +##### Example + +This example names the bucket as _pulsar-topic-offload_. + +```conf + +managedLedgerOffloadBucket=pulsar-topic-offload + +``` + +#### Authentication (required) + +To be able to access Azure BlobStore, you need to authenticate with Azure BlobStore. + +* Set the environment variables `AZURE_STORAGE_ACCOUNT` and `AZURE_STORAGE_ACCESS_KEY` in `conf/pulsar_env.sh`. + + "export" is important so that the variables are made available in the environment of spawned processes. + + ```bash + + export AZURE_STORAGE_ACCOUNT=ABC123456789 + export AZURE_STORAGE_ACCESS_KEY=ded7db27a4558e2ea8bbf0bf37ae0e8521618f366c + + ``` + +#### Size of block read/write + +You can configure the size of a request sent to or read from Azure BlobStore in the configuration file `broker.conf` or `standalone.conf`. + +Configuration|Description|Default value +|---|---|--- +`managedLedgerOffloadReadBufferSizeInBytes`|Block size for each individual read when reading back data from Azure BlobStore store.|1 MB +`managedLedgerOffloadMaxBlockSizeInBytes`|Maximum size of a "part" sent during a multipart upload to Azure BlobStore store. It **cannot** be smaller than 5 MB. |64 MB + +### Configure Azure BlobStore offloader to run automatically + +Namespace policy can be configured to offload data automatically once a threshold is reached. The threshold is based on the size of data that a topic has stored on a Pulsar cluster. Once the topic reaches the threshold, an offloading operation is triggered automatically. + +Threshold value|Action +|---|--- +> 0 | It triggers the offloading operation if the topic storage reaches its threshold. += 0|It causes a broker to offload data as soon as possible. +< 0 |It disables automatic offloading operation. + +Automatic offloading runs when a new segment is added to a topic log. If you set the threshold on a namespace, but few messages are being produced to the topic, offloader does not work until the current segment is full. + +You can configure the threshold size using CLI tools, such as pulsar-admin. + +The offload configurations in `broker.conf` and `standalone.conf` are used for the namespaces that do not have namespace level offload policies. Each namespace can have its own offload policy. If you want to set offload policy for each namespace, use the command [`pulsar-admin namespaces set-offload-policies options`](https://pulsar.apache.org/tools/pulsar-admin/2.6.0-SNAPSHOT/#-em-set-offload-policies-em-) command. + +#### Example + +This example sets the Azure BlobStore offloader threshold size to 10 MB using pulsar-admin. + +```bash + +bin/pulsar-admin namespaces set-offload-threshold --size 10M my-tenant/my-namespace + +``` + +:::tip + +For more information about the `pulsar-admin namespaces set-offload-threshold options` command, including flags, descriptions, and default values, see [here](https://pulsar.apache.org/tools/pulsar-admin/2.6.0-SNAPSHOT/#-em-set-offload-threshold-em-). + +::: + +### Configure Azure BlobStore offloader to run manually + +For individual topics, you can trigger Azure BlobStore offloader manually using one of the following methods: + +- Use REST endpoint. + +- Use CLI tools (such as pulsar-admin). + + To trigger it via CLI tools, you need to specify the maximum amount of data (threshold) that should be retained on a Pulsar cluster for a topic. If the size of the topic data on the Pulsar cluster exceeds this threshold, segments from the topic are moved to Azure BlobStore until the threshold is no longer exceeded. Older segments are moved first. + +#### Example + +- This example triggers the Azure BlobStore offloader to run manually using pulsar-admin. + + ```bash + + bin/pulsar-admin topics offload --size-threshold 10M my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ```bash + + Offload triggered for persistent://my-tenant/my-namespace/topic1 for messages before 2:0:-1 + + ``` + + :::tip + + For more information about the `pulsar-admin topics offload options` command, including flags, descriptions, and default values, see [here](https://pulsar.apache.org/tools/pulsar-admin/2.6.0-SNAPSHOT/#-em-offload-em-). + + ::: + +- This example checks the Azure BlobStore offloader status using pulsar-admin. + + ```bash + + bin/pulsar-admin topics offload-status persistent://my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ```bash + + Offload is currently running + + ``` + + To wait for the Azure BlobStore offloader to complete the job, add the `-w` flag. + + ```bash + + bin/pulsar-admin topics offload-status -w persistent://my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ``` + + Offload was a success + + ``` + + If there is an error in offloading, the error is propagated to the `pulsar-admin topics offload-status` command. + + ```bash + + bin/pulsar-admin topics offload-status persistent://my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ``` + + Error in offload + null + + Reason: Error offloading: org.apache.bookkeeper.mledger.ManagedLedgerException: + + ``` + + :::tip + + For more information about the `pulsar-admin topics offload-status options` command, including flags, descriptions, and default values, see [here](https://pulsar.apache.org/tools/pulsar-admin/2.6.0-SNAPSHOT/#-em-offload-status-em-). + + ::: + diff --git a/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-filesystem.md b/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-filesystem.md new file mode 100644 index 0000000000000..c988c98b78fe4 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-filesystem.md @@ -0,0 +1,319 @@ +--- +id: tiered-storage-filesystem +title: Use filesystem offloader with Pulsar +sidebar_label: "Filesystem offloader" +original_id: tiered-storage-filesystem +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +This chapter guides you through every step of installing and configuring the filesystem offloader and using it with Pulsar. + +## Installation + +Follow the steps below to install the filesystem offloader. + +### Prerequisite + +- Pulsar: 2.4.2 or later versions + +- Hadoop: 3.x.x + +### Step + +This example uses Pulsar 2.5.1. + +1. Download the Pulsar tarball using one of the following ways: + + * Download from the [Apache mirror](https://archive.apache.org/dist/pulsar/pulsar-2.5.1/apache-pulsar-2.5.1-bin.tar.gz) + + * Download from the Pulsar [download page](https://pulsar.apache.org/download) + + * Use [wget](https://www.gnu.org/software/wget) + + ```shell + + wget https://archive.apache.org/dist/pulsar/pulsar-2.5.1/apache-pulsar-2.5.1-bin.tar.gz + + ``` + +2. Download and untar the Pulsar offloaders package. + + ```bash + + wget https://downloads.apache.org/pulsar/pulsar-2.5.1/apache-pulsar-offloaders-2.5.1-bin.tar.gz + + tar xvfz apache-pulsar-offloaders-2.5.1-bin.tar.gz + + ``` + + :::note + + * If you are running Pulsar in a bare metal cluster, make sure that `offloaders` tarball is unzipped in every broker's Pulsar directory. + * If you are running Pulsar in Docker or deploying Pulsar using a Docker image (such as K8S and DCOS), you can use the `apachepulsar/pulsar-all` image instead of the `apachepulsar/pulsar` image. `apachepulsar/pulsar-all` image has already bundled tiered storage offloaders. + + ::: + +3. Copy the Pulsar offloaders as `offloaders` in the Pulsar directory. + + ``` + + mv apache-pulsar-offloaders-2.5.1/offloaders apache-pulsar-2.5.1/offloaders + + ls offloaders + + ``` + + **Output** + + ``` + + tiered-storage-file-system-2.5.1.nar + tiered-storage-jcloud-2.5.1.nar + + ``` + + :::note + + * If you are running Pulsar in a bare metal cluster, make sure that `offloaders` tarball is unzipped in every broker's Pulsar directory. + * If you are running Pulsar in Docker or deploying Pulsar using a Docker image (such as K8s and DCOS), you can use the `apachepulsar/pulsar-all` image instead of the `apachepulsar/pulsar` image. `apachepulsar/pulsar-all` image has already bundled tiered storage offloaders. + + ::: + +## Configuration + +:::note + +Before offloading data from BookKeeper to filesystem, you need to configure some properties of the filesystem offloader driver. + +::: + +Besides, you can also configure the filesystem offloader to run it automatically or trigger it manually. + +### Configure filesystem offloader driver + +You can configure filesystem offloader driver in the configuration file `broker.conf` or `standalone.conf`. + +- **Required** configurations are as below. + + Required configuration | Description | Example value + |---|---|--- + `managedLedgerOffloadDriver` | Offloader driver name, which is case-insensitive. | filesystem + `fileSystemURI` | Connection address | hdfs://127.0.0.1:9000 + `fileSystemProfilePath` | Hadoop profile path | ../conf/filesystem_offload_core_site.xml + +- **Optional** configurations are as below. + + Optional configuration| Description | Example value + |---|---|--- + `managedLedgerMinLedgerRolloverTimeMinutes`|Minimum time between ledger rollover for a topic

    **Note**: it is not recommended that you set this configuration in the production environment.|2 + `managedLedgerMaxEntriesPerLedger`|Maximum number of entries to append to a ledger before triggering a rollover.

    **Note**: it is not recommended that you set this configuration in the production environment.|5000 + +#### Offloader driver (required) + +Offloader driver name, which is case-insensitive. + +This example sets the offloader driver name as _filesystem_. + +```conf + +managedLedgerOffloadDriver=filesystem + +``` + +#### Connection address (required) + +Connection address is the URI to access the default Hadoop distributed file system. + +##### Example + +This example sets the connection address as _hdfs://127.0.0.1:9000_. + +```conf + +fileSystemURI=hdfs://127.0.0.1:9000 + +``` + +#### Hadoop profile path (required) + +The configuration file is stored in the Hadoop profile path. It contains various settings for Hadoop performance tuning. + +##### Example + +This example sets the Hadoop profile path as _../conf/filesystem_offload_core_site.xml_. + +```conf + +fileSystemProfilePath=../conf/filesystem_offload_core_site.xml + +``` + +You can set the following configurations in the _filesystem_offload_core_site.xml_ file. + +``` + + + fs.defaultFS + + + + + hadoop.tmp.dir + pulsar + + + + io.file.buffer.size + 4096 + + + + io.seqfile.compress.blocksize + 1000000 + + + + io.seqfile.compression.type + BLOCK + + + + io.map.index.interval + 128 + + +``` + +:::tip + +For more information about the Hadoop HDFS, see [here](https://hadoop.apache.org/docs/current/). + +::: + +### Configure filesystem offloader to run automatically + +Namespace policy can be configured to offload data automatically once a threshold is reached. The threshold is based on the size of data that a topic has stored on a Pulsar cluster. Once the topic reaches the threshold, an offload operation is triggered automatically. + +Threshold value|Action +|---|--- +> 0 | It triggers the offloading operation if the topic storage reaches its threshold. += 0|It causes a broker to offload data as soon as possible. +< 0 |It disables automatic offloading operation. + +Automatic offload runs when a new segment is added to a topic log. If you set the threshold on a namespace, but few messages are being produced to the topic, offloader does not work until the current segment is full. + +You can configure the threshold size using CLI tools, such as pulsar-admin. + +#### Example + +This example sets the filesystem offloader threshold size to 10 MB using pulsar-admin. + +```bash + +pulsar-admin namespaces set-offload-threshold --size 10M my-tenant/my-namespace + +``` + +:::tip + +For more information about the `pulsar-admin namespaces set-offload-threshold options` command, including flags, descriptions, default values, and shorthands, see [here](reference-pulsar-admin.md#set-offload-threshold). + +::: + +### Configure filesystem offloader to run manually + +For individual topics, you can trigger filesystem offloader manually using one of the following methods: + +- Use REST endpoint. + +- Use CLI tools (such as pulsar-admin). + +To trigger via CLI tools, you need to specify the maximum amount of data (threshold) that should be retained on a Pulsar cluster for a topic. If the size of the topic data on the Pulsar cluster exceeds this threshold, segments from the topic are offloaded to the filesystem until the threshold is no longer exceeded. Older segments are offloaded first. + +#### Example + +- This example triggers the filesystem offloader to run manually using pulsar-admin. + + ```bash + + pulsar-admin topics offload --size-threshold 10M persistent://my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ```bash + + Offload triggered for persistent://my-tenant/my-namespace/topic1 for messages before 2:0:-1 + + ``` + + :::tip + + For more information about the `pulsar-admin topics offload options` command, including flags, descriptions, default values, and shorthands, see [here](reference-pulsar-admin.md#offload). + + ::: + +- This example checks filesystem offloader status using pulsar-admin. + + ```bash + + pulsar-admin topics offload-status persistent://my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ```bash + + Offload is currently running + + ``` + + To wait for the filesystem to complete the job, add the `-w` flag. + + ```bash + + pulsar-admin topics offload-status -w persistent://my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ``` + + Offload was a success + + ``` + + If there is an error in the offloading operation, the error is propagated to the `pulsar-admin topics offload-status` command. + + ```bash + + pulsar-admin topics offload-status persistent://my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ``` + + Error in offload + null + + Reason: Error offloading: org.apache.bookkeeper.mledger.ManagedLedgerException: java.util.concurrent.CompletionException: com.amazonaws.services.s3.model.AmazonS3Exception: Anonymous users cannot initiate multipart uploads. Please authenticate. (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied; Request ID: 798758DE3F1776DF; S3 Extended Request ID: dhBFz/lZm1oiG/oBEepeNlhrtsDlzoOhocuYMpKihQGXe6EG8puRGOkK6UwqzVrMXTWBxxHcS+g=), S3 Extended Request ID: dhBFz/lZm1oiG/oBEepeNlhrtsDlzoOhocuYMpKihQGXe6EG8puRGOkK6UwqzVrMXTWBxxHcS+g= + + ``` + + :::tip + + For more information about the `pulsar-admin topics offload-status options` command, including flags, descriptions, default values, and shorthands, see [here](reference-pulsar-admin.md#offload-status). + + ::: + +## Tutorial + +For the complete and step-by-step instructions on how to use the filesystem offloader with Pulsar, see [here](https://hub.streamnative.io/offloaders/filesystem/2.5.1). \ No newline at end of file diff --git a/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-gcs.md b/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-gcs.md new file mode 100644 index 0000000000000..5c70cd36cd0fd --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-gcs.md @@ -0,0 +1,323 @@ +--- +id: tiered-storage-gcs +title: Use GCS offloader with Pulsar +sidebar_label: "GCS offloader" +original_id: tiered-storage-gcs +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +This chapter guides you through every step of installing and configuring the GCS offloader and using it with Pulsar. + +## Installation + +Follow the steps below to install the GCS offloader. + +### Prerequisite + +- Pulsar: 2.4.2 or later versions + +### Step + +This example uses Pulsar 2.5.1. + +1. Download the Pulsar tarball using one of the following ways: + + * Download from the [Apache mirror](https://archive.apache.org/dist/pulsar/pulsar-2.5.1/apache-pulsar-2.5.1-bin.tar.gz) + + * Download from the Pulsar [download page](https://pulsar.apache.org/download) + + * Use [wget](https://www.gnu.org/software/wget) + + ```shell + + wget https://archive.apache.org/dist/pulsar/pulsar-2.5.1/apache-pulsar-2.5.1-bin.tar.gz + + ``` + +2. Download and untar the Pulsar offloaders package. + + ```bash + + wget https://downloads.apache.org/pulsar/pulsar-2.5.1/apache-pulsar-offloaders-2.5.1-bin.tar.gz + + tar xvfz apache-pulsar-offloaders-2.5.1-bin.tar.gz + + ``` + + :::note + + * If you are running Pulsar in a bare metal cluster, make sure that `offloaders` tarball is unzipped in every broker's Pulsar directory. + * If you are running Pulsar in Docker or deploying Pulsar using a Docker image (such as K8S and DCOS), you can use the `apachepulsar/pulsar-all` image instead of the `apachepulsar/pulsar` image. `apachepulsar/pulsar-all` image has already bundled tiered storage offloaders. + + ::: + +3. Copy the Pulsar offloaders as `offloaders` in the Pulsar directory. + + ``` + + mv apache-pulsar-offloaders-2.5.1/offloaders apache-pulsar-2.5.1/offloaders + + ls offloaders + + ``` + + **Output** + + As shown in the output, Pulsar uses [Apache jclouds](https://jclouds.apache.org) to support GCS and AWS S3 for long term storage. + + ``` + + tiered-storage-file-system-2.5.1.nar + tiered-storage-jcloud-2.5.1.nar + + ``` + +## Configuration + +:::note + +Before offloading data from BookKeeper to GCS, you need to configure some properties of the GCS offloader driver. + +::: + +Besides, you can also configure the GCS offloader to run it automatically or trigger it manually. + +### Configure GCS offloader driver + +You can configure GCS offloader driver in the configuration file `broker.conf` or `standalone.conf`. + +- **Required** configurations are as below. + + **Required** configuration | Description | Example value + |---|---|--- + `managedLedgerOffloadDriver`|Offloader driver name, which is case-insensitive.|google-cloud-storage + `offloadersDirectory`|Offloader directory|offloaders + `gcsManagedLedgerOffloadBucket`|Bucket|pulsar-topic-offload + `gcsManagedLedgerOffloadRegion`|Bucket region|europe-west3 + `gcsManagedLedgerOffloadServiceAccountKeyFile`|Authentication |/Users/user-name/Downloads/project-804d5e6a6f33.json + +- **Optional** configurations are as below. + + Optional configuration|Description|Example value + |---|---|--- + `gcsManagedLedgerOffloadReadBufferSizeInBytes`|Size of block read|1 MB + `gcsManagedLedgerOffloadMaxBlockSizeInBytes`|Size of block write|64 MB + `managedLedgerMinLedgerRolloverTimeMinutes`|Minimum time between ledger rollover for a topic.|2 + `managedLedgerMaxEntriesPerLedger`|The max number of entries to append to a ledger before triggering a rollover.|5000 + +#### Bucket (required) + +A bucket is a basic container that holds your data. Everything you store in GCS **must** be contained in a bucket. You can use a bucket to organize your data and control access to your data, but unlike directory and folder, you can not nest a bucket. + +##### Example + +This example names the bucket as _pulsar-topic-offload_. + +```conf + +gcsManagedLedgerOffloadBucket=pulsar-topic-offload + +``` + +#### Bucket region (required) + +Bucket region is the region where a bucket is located. If a bucket region is not specified, the **default** region (`us multi-regional location`) is used. + +:::tip + +For more information about bucket location, see [here](https://cloud.google.com/storage/docs/bucket-locations). + +::: + +##### Example + +This example sets the bucket region as _europe-west3_. + +``` + +gcsManagedLedgerOffloadRegion=europe-west3 + +``` + +#### Authentication (required) + +To enable a broker access GCS, you need to configure `gcsManagedLedgerOffloadServiceAccountKeyFile` in the configuration file `broker.conf`. + +`gcsManagedLedgerOffloadServiceAccountKeyFile` is +a JSON file, containing GCS credentials of a service account. + +##### Example + +To generate service account credentials or view the public credentials that you've already generated, follow the following steps. + +1. Navigate to the [Service accounts page](https://console.developers.google.com/iam-admin/serviceaccounts). + +2. Select a project or create a new one. + +3. Click **Create service account**. + +4. In the **Create service account** window, type a name for the service account and select **Furnish a new private key**. + + If you want to [grant G Suite domain-wide authority](https://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority) to the service account, select **Enable G Suite Domain-wide Delegation**. + +5. Click **Create**. + + :::note + + Make sure the service account you create has permission to operate GCS, you need to assign **Storage Admin** permission to your service account [here](https://cloud.google.com/storage/docs/access-control/iam). + + ::: + +6. You can get the following information and set this in `broker.conf`. + + ```conf + + gcsManagedLedgerOffloadServiceAccountKeyFile="/Users/user-name/Downloads/project-804d5e6a6f33.json" + + ``` + + :::tip + + - For more information about how to create `gcsManagedLedgerOffloadServiceAccountKeyFile`, see [here](https://support.google.com/googleapi/answer/6158849). + - For more information about Google Cloud IAM, see [here](https://cloud.google.com/storage/docs/access-control/iam). + + ::: + +#### Size of block read/write + +You can configure the size of a request sent to or read from GCS in the configuration file `broker.conf`. + +Configuration|Description +|---|--- +`gcsManagedLedgerOffloadReadBufferSizeInBytes`|Block size for each individual read when reading back data from GCS.

    The **default** value is 1 MB. +`gcsManagedLedgerOffloadMaxBlockSizeInBytes`|Maximum size of a "part" sent during a multipart upload to GCS.

    It **can not** be smaller than 5 MB.

    The **default** value is 64 MB. + +### Configure GCS offloader to run automatically + +Namespace policy can be configured to offload data automatically once a threshold is reached. The threshold is based on the size of data that a topic has stored on a Pulsar cluster. Once the topic reaches the threshold, an offload operation is triggered automatically. + +Threshold value|Action +|---|--- +> 0 | It triggers the offloading operation if the topic storage reaches its threshold. += 0|It causes a broker to offload data as soon as possible. +< 0 |It disables automatic offloading operation. + +Automatic offloading runs when a new segment is added to a topic log. If you set the threshold on a namespace, but few messages are being produced to the topic, offloader does not work until the current segment is full. + +You can configure the threshold size using CLI tools, such as pulsar-admin. + +The offload configurations in `broker.conf` and `standalone.conf` are used for the namespaces that do not have namespace level offload policies. Each namespace can have its own offload policy. If you want to set offload policy for each namespace, use the command [`pulsar-admin namespaces set-offload-policies options`](https://pulsar.apache.org/tools/pulsar-admin/2.6.0-SNAPSHOT/#-em-set-offload-policies-em-) command. + +#### Example + +This example sets the GCS offloader threshold size to 10 MB using pulsar-admin. + +```bash + +pulsar-admin namespaces set-offload-threshold --size 10M my-tenant/my-namespace + +``` + +:::tip + +For more information about the `pulsar-admin namespaces set-offload-threshold options` command, including flags, descriptions, default values, and shorthands, see [here](reference-pulsar-admin.md#set-offload-threshold). + +::: + +### Configure GCS offloader to run manually + +For individual topics, you can trigger GCS offloader manually using one of the following methods: + +- Use REST endpoint. + +- Use CLI tools (such as pulsar-admin). + + To trigger the GCS via CLI tools, you need to specify the maximum amount of data (threshold) that should be retained on a Pulsar cluster for a topic. If the size of the topic data on the Pulsar cluster exceeds this threshold, segments from the topic are moved to GCS until the threshold is no longer exceeded. Older segments are moved first. + +#### Example + +- This example triggers the GCS offloader to run manually using pulsar-admin with the command `pulsar-admin topics offload (topic-name) (threshold)`. + + ```bash + + pulsar-admin topics offload persistent://my-tenant/my-namespace/topic1 10M + + ``` + + **Output** + + ```bash + + Offload triggered for persistent://my-tenant/my-namespace/topic1 for messages before 2:0:-1 + + ``` + + :::tip + + For more information about the `pulsar-admin topics offload options` command, including flags, descriptions, default values, and shorthands, see [here](reference-pulsar-admin.md#offload). + + ::: + +- This example checks the GCS offloader status using pulsar-admin with the command `pulsar-admin topics offload-status options`. + + ```bash + + pulsar-admin topics offload-status persistent://my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ```bash + + Offload is currently running + + ``` + + To wait for GCS to complete the job, add the `-w` flag. + + ```bash + + pulsar-admin topics offload-status -w persistent://my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ``` + + Offload was a success + + ``` + + If there is an error in offloading, the error is propagated to the `pulsar-admin topics offload-status` command. + + ```bash + + pulsar-admin topics offload-status persistent://my-tenant/my-namespace/topic1 + + ``` + + **Output** + + ``` + + Error in offload + null + + Reason: Error offloading: org.apache.bookkeeper.mledger.ManagedLedgerException: java.util.concurrent.CompletionException: com.amazonaws.services.s3.model.AmazonS3Exception: Anonymous users cannot initiate multipart uploads. Please authenticate. (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied; Request ID: 798758DE3F1776DF; S3 Extended Request ID: dhBFz/lZm1oiG/oBEepeNlhrtsDlzoOhocuYMpKihQGXe6EG8puRGOkK6UwqzVrMXTWBxxHcS+g=), S3 Extended Request ID: dhBFz/lZm1oiG/oBEepeNlhrtsDlzoOhocuYMpKihQGXe6EG8puRGOkK6UwqzVrMXTWBxxHcS+g= + + ``` + + :::tip + + For more information about the `pulsar-admin topics offload-status options` command, including flags, descriptions, default values, and shorthands, see [here](reference-pulsar-admin.md#offload-status). + + ::: + +## Tutorial + +For the complete and step-by-step instructions on how to use the GCS offloader with Pulsar, see [here](https://hub.streamnative.io/offloaders/gcs/2.5.1#usage). \ No newline at end of file diff --git a/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-overview.md b/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-overview.md new file mode 100644 index 0000000000000..8536268c48bc8 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/tiered-storage-overview.md @@ -0,0 +1,55 @@ +--- +id: tiered-storage-overview +title: Overview of tiered storage +sidebar_label: "Overview" +original_id: tiered-storage-overview +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +Pulsar's **Tiered Storage** feature allows older backlog data to be moved from BookKeeper to long term and cheaper storage, while still allowing clients to access the backlog as if nothing has changed. + +* Tiered storage uses [Apache jclouds](https://jclouds.apache.org) to support [Amazon S3](https://aws.amazon.com/s3/) and [GCS (Google Cloud Storage)](https://cloud.google.com/storage/) for long term storage. + + With jclouds, it is easy to add support for more [cloud storage providers](https://jclouds.apache.org/reference/providers/#blobstore-providers) in the future. + + :::tip + + For more information about how to use the AWS S3 offloader with Pulsar, see [here](tiered-storage-aws). + For more information about how to use the GCS offloader with Pulsar, see [here](tiered-storage-gcs). + + ::: + +* Tiered storage uses [Apache Hadoop](http://hadoop.apache.org/) to support filesystems for long term storage. + + With Hadoop, it is easy to add support for more filesystems in the future. + + :::tip + + For more information about how to use the filesystem offloader with Pulsar, see [here](tiered-storage-filesystem). + + ::: + +## When to use tiered storage? + +Tiered storage should be used when you have a topic for which you want to keep a very long backlog for a long time. + +For example, if you have a topic containing user actions which you use to train your recommendation systems, you may want to keep that data for a long time, so that if you change your recommendation algorithm, you can rerun it against your full user history. + +## How does tiered storage work? + +A topic in Pulsar is backed by a **log**, known as a **managed ledger**. This log is composed of an ordered list of segments. Pulsar only writes to the final segment of the log. All previous segments are sealed. The data within the segment is immutable. This is known as a **segment oriented architecture**. + +![Tiered storage](/assets/pulsar-tiered-storage.png "Tiered Storage") + +The tiered storage offloading mechanism takes advantage of segment oriented architecture. When offloading is requested, the segments of the log are copied one-by-one to tiered storage. All segments of the log (apart from the current segment) written to tiered storage can be offloaded. + +Data written to BookKeeper is replicated to 3 physical machines by default. However, once a segment is sealed in BookKeeper, it becomes immutable and can be copied to long term storage. Long term storage can achieve cost savings by using mechanisms such as [Reed-Solomon error correction](https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction) to require fewer physical copies of data. + +Before offloading ledgers to long term storage, you need to configure buckets, credentials, and other properties for the cloud storage service. Additionally, Pulsar uses multi-part objects to upload the segment data and brokers may crash while uploading the data. It is recommended that you add a life cycle rule for your bucket to expire incomplete multi-part upload after a day or two days to avoid getting charged for incomplete uploads. Moreover, you can trigger the offloading operation manually (via REST API or CLI) or automatically (via CLI). + +After offloading ledgers to long term storage, you can still query data in the offloaded ledgers with Pulsar SQL. + +For more information about tiered storage for Pulsar topics, see [here](https://github.com/apache/pulsar/wiki/PIP-17:-Tiered-storage-for-Pulsar-topics). diff --git a/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json b/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json index 68614e5e6b3f2..aa9f4c87815af 100644 --- a/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json +++ b/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json @@ -187,6 +187,32 @@ "id": "version-2.7.1/sql-rest-api" } ] + }, + { + "type": "category", + "label": "Tiered Storage", + "items": [ + { + "type": "doc", + "id": "version-2.7.1/tiered-storage-overview" + }, + { + "type": "doc", + "id": "version-2.7.1/tiered-storage-aws" + }, + { + "type": "doc", + "id": "version-2.7.1/tiered-storage-gcs" + }, + { + "type": "doc", + "id": "version-2.7.1/tiered-storage-filesystem" + }, + { + "type": "doc", + "id": "version-2.7.1/tiered-storage-azure" + } + ] } ] } \ No newline at end of file From 6072e812b1aaf3ac80e69533b2020dfac4dadd27 Mon Sep 17 00:00:00 2001 From: LiLi Date: Tue, 2 Nov 2021 10:10:30 +0800 Subject: [PATCH 4/5] [website][upgrade]feat: docs migration - 2.7.1 / transaction Signed-off-by: LiLi --- .../version-2.7.1/concepts-transactions.md | 34 ++++ .../version-2.7.1/transaction-api.md | 178 ++++++++++++++++++ .../version-2.7.1/transaction-guarantee.md | 21 +++ .../version-2.7.1-sidebars.json | 18 ++ 4 files changed, 251 insertions(+) create mode 100644 site2/website-next/versioned_docs/version-2.7.1/concepts-transactions.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/transaction-api.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/transaction-guarantee.md diff --git a/site2/website-next/versioned_docs/version-2.7.1/concepts-transactions.md b/site2/website-next/versioned_docs/version-2.7.1/concepts-transactions.md new file mode 100644 index 0000000000000..9e44d4f4e0258 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/concepts-transactions.md @@ -0,0 +1,34 @@ +--- +id: transactions +title: Transactions +sidebar_label: "Overview" +original_id: transactions +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +Transactional semantics enable event streaming applications to consume, process, and produce messages in one atomic operation. In Pulsar, a producer or consumer can work with messages across multiple topics and partitions and ensure those messages are processed as a single unit. + +The following concepts help you understand Pulsar transactions. + +## Transaction coordinator and transaction log +The transaction coordinator maintains the topics and subscriptions that interact in a transaction. When a transaction is committed, the transaction coordinator interacts with the topic owner broker to complete the transaction. + +The transaction coordinator maintains the entire life cycle of transactions, and prevents a transaction from incorrect status. + +The transaction coordinator handles transaction timeout, and ensures that the transaction is aborted after a transaction timeout. + +All the transaction metadata is persisted in the transaction log. The transaction log is backed by a Pulsar topic. After the transaction coordinator crashes, it can restore the transaction metadata from the transaction log. + +## Transaction ID +The transaction ID (TxnID) identifies a unique transaction in Pulsar. The transaction ID is 128-bit. The highest 16 bits are reserved for the ID of the transaction coordinator, and the remaining bits are used for monotonically increasing numbers in each transaction coordinator. It is easy to locate the transaction crash with the TxnID. + +## Transaction buffer +Messages produced within a transaction are stored in the transaction buffer. The messages in transaction buffer are not materialized (visible) to consumers until the transactions are committed. The messages in the transaction buffer are discarded when the transactions are aborted. + +## Pending acknowledge state +Message acknowledges within a transaction are maintained by the pending acknowledge state before the transaction completes. If a message is in the pending acknowledge state, the message cannot be acknowledged by other transactions until the message is removed from the pending acknowledge state. + +The pending acknowledge state is persisted to the pending acknowledge log. The pending acknowledge log is backed by a Pulsar topic. A new broker can restore the state from the pending acknowledge log to ensure the acknowledgement is not lost. diff --git a/site2/website-next/versioned_docs/version-2.7.1/transaction-api.md b/site2/website-next/versioned_docs/version-2.7.1/transaction-api.md new file mode 100644 index 0000000000000..ba8dedab3befe --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/transaction-api.md @@ -0,0 +1,178 @@ +--- +id: transactions-api +title: Transactions API (Developer Preview) +sidebar_label: "Transactions API" +original_id: transactions-api +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +All messages in a transaction is available only to consumers after the transaction is committed. If a transaction is aborted, all the writes and acknowledgments in this transaction rollback. + +Currently, Pulsar transaction is a developer preview feature. It is disabled by default. You can enable the feature and use transactions in your application in development environment. + +## Prerequisites +1. To enable transactions in Pulsar, you need to configure the parameter in the `broker.conf` file. + +``` + +transactionCoordinatorEnabled=true + +``` + +2. Initialize transaction coordinator metadata, so the transaction coordinators can leverage advantages of the partitioned topic, such as load balance. + +``` + +bin/pulsar initialize-transaction-coordinator-metadata -cs 127.0.0.1:2181 -c standalone + +``` + +After initializing transaction coordinator metadata, you can use the transactions API. The following APIs are available. + +## Initialize Pulsar client + +You can enable transaction for transaction client and initialize transaction coordinator client. + +``` + +PulsarClient pulsarClient = PulsarClient.builder() + .serviceUrl("pulsar://localhost:6650") + .enableTransaction(true) + .build(); + +``` + +## Start transactions +You can start transaction in the following way. + +``` + +Transaction txn = pulsarClient + .newTransaction() + .withTransactionTimeout(5, TimeUnit.MINUTES) + .build() + .get(); + +``` + +## Produce transaction messages + +A transaction parameter is required when producing new transaction messages. The semantic of the transaction messages in Pulsar is `read-committed`, so the consumer cannot receive the ongoing transaction messages before the transaction is committed. + +``` + +producer.newMessage(txn).value("Hello Pulsar Transaction".getBytes()).sendAsync(); + +``` + +## Acknowledge the messages with the transaction + +The transaction acknowledgement requires a transaction parameter. The transaction acknowledgement marks the messages state to pending-ack state. When the transaction is committed, the pending-ack state becomes ack state. If the transaction is aborted, the pending-ack state becomes unack state. + +``` + +Message message = consumer.receive(); +consumer.acknowledgeAsync(message.getMessageId(), txn); + +``` + +## Commit transactions + +When the transaction is committed, consumers receive the transaction messages and the pending-ack state becomes ack state. + +``` + +txn.commit().get(); + +``` + +## Abort transaction + +When the transaction is aborted, the transaction acknowledgement is canceled and the pending-ack messages are redelivered. + +``` + +txn.abort().get(); + +``` + +### Example +The following example shows how messages are processed in transaction. + +``` + +PulsarClient pulsarClient = PulsarClient.builder() + .serviceUrl(getPulsarServiceList().get(0).getBrokerServiceUrl()) + .statsInterval(0, TimeUnit.SECONDS) + .enableTransaction(true) + .build(); + +String sourceTopic = "public/default/source-topic"; +String sinkTopic = "public/default/sink-topic"; + +Producer sourceProducer = pulsarClient + .newProducer(Schema.STRING) + .topic(sourceTopic) + .create(); +sourceProducer.newMessage().value("hello pulsar transaction").sendAsync(); + +Consumer sourceConsumer = pulsarClient + .newConsumer(Schema.STRING) + .topic(sourceTopic) + .subscriptionName("test") + .subscriptionType(SubscriptionType.Shared) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + +Producer sinkProducer = pulsarClient + .newProducer(Schema.STRING) + .topic(sinkTopic) + .sendTimeout(0, TimeUnit.MILLISECONDS) + .create(); + +Transaction txn = pulsarClient + .newTransaction() + .withTransactionTimeout(5, TimeUnit.MINUTES) + .build() + .get(); + +// source message acknowledgement and sink message produce belong to one transaction, +// they are combined into an atomic operation. +Message message = sourceConsumer.receive(); +sourceConsumer.acknowledgeAsync(message.getMessageId(), txn); +sinkProducer.newMessage(txn).value("sink data").sendAsync(); + +txn.commit().get(); + +``` + +## Enable batch messages in transactions + +To enable batch messages in transactions, you need to enable the batch index acknowledgement feature. The transaction acks check whether the batch index acknowledgement conflicts. + +To enable batch index acknowledgement, you need to set `acknowledgmentAtBatchIndexLevelEnabled` to `true` in the `broker.conf` or `standalone.conf` file. + +``` + +acknowledgmentAtBatchIndexLevelEnabled=true + +``` + +And then you need to call the `enableBatchIndexAcknowledgment(true)` method in the consumer builder. + +``` + +Consumer sinkConsumer = pulsarClient + .newConsumer() + .topic(transferTopic) + .subscriptionName("sink-topic") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionType(SubscriptionType.Shared) + .enableBatchIndexAcknowledgment(true) // enable batch index acknowledgement + .subscribe(); + +``` + diff --git a/site2/website-next/versioned_docs/version-2.7.1/transaction-guarantee.md b/site2/website-next/versioned_docs/version-2.7.1/transaction-guarantee.md new file mode 100644 index 0000000000000..b75c94a1625e8 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/transaction-guarantee.md @@ -0,0 +1,21 @@ +--- +id: transactions-guarantee +title: Transactions Guarantee +sidebar_label: "Transactions Guarantee" +original_id: transactions-guarantee +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +Pulsar transactions support the following guarantee. + +## Atomic multi-partition writes and multi-subscription acknowledges +Transactions enable atomic writes to multiple topics and partitions. A batch of messages in a transaction can be received from, produced to, and acknowledged by many partitions. All the operations involved in a transaction succeed or fail as a single unit. + +## Read transactional message +All the messages in a transaction are available only for consumers until the transaction is committed. + +## Acknowledge transactional message +A message is acknowledged successfully only once by a consumer under the subscription when acknowledging the message with the transaction ID. \ No newline at end of file diff --git a/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json b/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json index aa9f4c87815af..2ac7003bfd77b 100644 --- a/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json +++ b/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json @@ -213,6 +213,24 @@ "id": "version-2.7.1/tiered-storage-azure" } ] + }, + { + "type": "category", + "label": "Transactions", + "items": [ + { + "type": "doc", + "id": "version-2.7.1/transactions" + }, + { + "type": "doc", + "id": "version-2.7.1/transactions-guarantee" + }, + { + "type": "doc", + "id": "version-2.7.1/transactions-api" + } + ] } ] } \ No newline at end of file From 283d490f102c9e965b97abbe1c47c27f90ca5d06 Mon Sep 17 00:00:00 2001 From: LiLi Date: Tue, 2 Nov 2021 10:14:00 +0800 Subject: [PATCH 5/5] [website][upgrade]feat: docs migration - 2.7.1 / helm Signed-off-by: LiLi --- .../version-2.7.1/helm-deploy.md | 438 ++++++++++++++++++ .../version-2.7.1/helm-install.md | 48 ++ .../version-2.7.1/helm-overview.md | 108 +++++ .../version-2.7.1/helm-prepare.md | 96 ++++ .../version-2.7.1/helm-tools.md | 47 ++ .../version-2.7.1/helm-upgrade.md | 49 ++ .../version-2.7.1-sidebars.json | 30 ++ 7 files changed, 816 insertions(+) create mode 100644 site2/website-next/versioned_docs/version-2.7.1/helm-deploy.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/helm-install.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/helm-overview.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/helm-prepare.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/helm-tools.md create mode 100644 site2/website-next/versioned_docs/version-2.7.1/helm-upgrade.md diff --git a/site2/website-next/versioned_docs/version-2.7.1/helm-deploy.md b/site2/website-next/versioned_docs/version-2.7.1/helm-deploy.md new file mode 100644 index 0000000000000..e64ca2de9b1cf --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/helm-deploy.md @@ -0,0 +1,438 @@ +--- +id: helm-deploy +title: Deploy Pulsar cluster using Helm +sidebar_label: "Deployment" +original_id: helm-deploy +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +Before running `helm install`, you need to decide how to run Pulsar. +Options can be specified using Helm's `--set option.name=value` command line option. + +## Select configuration options + +In each section, collect the options that are combined to use with the `helm install` command. + +### Kubernetes namespace + +By default, the Pulsar Helm chart is installed to a namespace called `pulsar`. + +```yaml + +namespace: pulsar + +``` + +To install the Pulsar Helm chart into a different Kubernetes namespace, you can include this option in the `helm install` command. + +```bash + +--set namespace= + +``` + +By default, the Pulsar Helm chart doesn't create the namespace. + +```yaml + +namespaceCreate: false + +``` + +To use the Pulsar Helm chart to create the Kubernetes namespace automatically, you can include this option in the `helm install` command. + +```bash + +--set namespaceCreate=true + +``` + +### Persistence + +By default, the Pulsar Helm chart creates Volume Claims with the expectation that a dynamic provisioner creates the underlying Persistent Volumes. + +```yaml + +volumes: + persistence: true + # configure the components to use local persistent volume + # the local provisioner should be installed prior to enable local persistent volume + local_storage: false + +``` + +To use local persistent volumes as the persistent storage for Helm release, you can install the [local storage provisioner](#install-local-storage-provisioner) and include the following option in the `helm install` command. + +```bash + +--set volumes.local_storage=true + +``` + +:::note + +Before installing the production instance of Pulsar, ensure to plan the storage settings to avoid extra storage migration work. Because after initial installation, you must edit Kubernetes objects manually if you want to change storage settings. + +::: + +The Pulsar Helm chart is designed for production use. To use the Pulsar Helm chart in a development environment (such as Minikube), you can disable persistence by including this option in your `helm install` command. + +```bash + +--set volumes.persistence=false + +``` + +### Affinity + +By default, `anti-affinity` is enabled to ensure pods of the same component can run on different nodes. + +```yaml + +affinity: + anti_affinity: true + +``` + +To use the Pulsar Helm chart in a development environment (such as Minikue), you can disable `anti-affinity` by including this option in your `helm install` command. + +```bash + +--set affinity.anti_affinity=false + +``` + +### Components + +The Pulsar Helm chart is designed for production usage. It deploys a production-ready Pulsar cluster, including Pulsar core components and monitoring components. + +You can customize the components to be deployed by turning on/off individual components. + +```yaml + +## Components +## +## Control what components of Apache Pulsar to deploy for the cluster +components: + # zookeeper + zookeeper: true + # bookkeeper + bookkeeper: true + # bookkeeper - autorecovery + autorecovery: true + # broker + broker: true + # functions + functions: true + # proxy + proxy: true + # toolset + toolset: true + # pulsar manager + pulsar_manager: true + +## Monitoring Components +## +## Control what components of the monitoring stack to deploy for the cluster +monitoring: + # monitoring - prometheus + prometheus: true + # monitoring - grafana + grafana: true + +``` + +### Docker images + +The Pulsar Helm chart is designed to enable controlled upgrades. So it can configure independent image versions for components. You can customize the images by setting individual component. + +```yaml + +## Images +## +## Control what images to use for each component +images: + zookeeper: + repository: apachepulsar/pulsar-all + tag: 2.5.0 + pullPolicy: IfNotPresent + bookie: + repository: apachepulsar/pulsar-all + tag: 2.5.0 + pullPolicy: IfNotPresent + autorecovery: + repository: apachepulsar/pulsar-all + tag: 2.5.0 + pullPolicy: IfNotPresent + broker: + repository: apachepulsar/pulsar-all + tag: 2.5.0 + pullPolicy: IfNotPresent + proxy: + repository: apachepulsar/pulsar-all + tag: 2.5.0 + pullPolicy: IfNotPresent + functions: + repository: apachepulsar/pulsar-all + tag: 2.5.0 + prometheus: + repository: prom/prometheus + tag: v1.6.3 + pullPolicy: IfNotPresent + grafana: + repository: streamnative/apache-pulsar-grafana-dashboard-k8s + tag: 0.0.4 + pullPolicy: IfNotPresent + pulsar_manager: + repository: apachepulsar/pulsar-manager + tag: v0.1.0 + pullPolicy: IfNotPresent + hasCommand: false + +``` + +### TLS + +The Pulsar Helm chart can be configured to enable TLS (Transport Layer Security) to protect all the traffic between components. Before enabling TLS, you have to provision TLS certificates for the required components. + +#### Provision TLS certificates using cert-manager + +To use the `cert-manager` to provision the TLS certificates, you have to install the [cert-manager](#install-cert-manager) before installing the Pulsar Helm chart. After successfully installing the cert-manager, you can set `certs.internal_issuer.enabled` to `true`. Therefore, the Pulsar Helm chart can use the `cert-manager` to generate `selfsigning` TLS certificates for the configured components. + +```yaml + +certs: + internal_issuer: + enabled: false + component: internal-cert-issuer + type: selfsigning + +``` + +You can also customize the generated TLS certificates by configuring the fields as the following. + +```yaml + +tls: + # common settings for generating certs + common: + # 90d + duration: 2160h + # 15d + renewBefore: 360h + organization: + - pulsar + keySize: 4096 + keyAlgorithm: rsa + keyEncoding: pkcs8 + +``` + +#### Enable TLS + +After installing the `cert-manager`, you can set `tls.enabled` to `true` to enable TLS encryption for the entire cluster. + +```yaml + +tls: + enabled: false + +``` + +You can also configure whether to enable TLS encryption for individual component. + +```yaml + +tls: + # settings for generating certs for proxy + proxy: + enabled: false + cert_name: tls-proxy + # settings for generating certs for broker + broker: + enabled: false + cert_name: tls-broker + # settings for generating certs for bookies + bookie: + enabled: false + cert_name: tls-bookie + # settings for generating certs for zookeeper + zookeeper: + enabled: false + cert_name: tls-zookeeper + # settings for generating certs for recovery + autorecovery: + cert_name: tls-recovery + # settings for generating certs for toolset + toolset: + cert_name: tls-toolset + +``` + +### Authentication + +By default, authentication is disabled. You can set `auth.authentication.enabled` to `true` to enable authentication. +Currently, the Pulsar Helm chart only supports JWT authentication provider. You can set `auth.authentication.provider` to `jwt` to use the JWT authentication provider. + +```yaml + +# Enable or disable broker authentication and authorization. +auth: + authentication: + enabled: false + provider: "jwt" + jwt: + # Enable JWT authentication + # If the token is generated by a secret key, set the usingSecretKey as true. + # If the token is generated by a private key, set the usingSecretKey as false. + usingSecretKey: false + superUsers: + # broker to broker communication + broker: "broker-admin" + # proxy to broker communication + proxy: "proxy-admin" + # pulsar-admin client to broker/proxy communication + client: "admin" + +``` + +To enable authentication, you can run [prepare helm release](#prepare-the-helm-release) to generate token secret keys and tokens for three super users specified in the `auth.superUsers` field. The generated token keys and super user tokens are uploaded and stored as Kubernetes secrets prefixed with `-token-`. You can use the following command to find those secrets. + +```bash + +kubectl get secrets -n + +``` + +### Authorization + +By default, authorization is disabled. Authorization can be enabled only when authentication is enabled. + +```yaml + +auth: + authorization: + enabled: false + +``` + +To enable authorization, you can include this option in the `helm install` command. + +```bash + +--set auth.authorization.enabled=true + +``` + +### CPU and RAM resource requirements + +By default, the resource requests and the number of replicas for the Pulsar components in the Pulsar Helm chart are adequate for a small production deployment. If you deploy a non-production instance, you can reduce the defaults to fit into a smaller cluster. + +Once you have all of your configuration options collected, you can install dependent charts before installing the Pulsar Helm chart. + +## Install dependent charts + +### Install local storage provisioner + +To use local persistent volumes as the persistent storage, you need to install a storage provisioner for [local persistent volumes](https://kubernetes.io/blog/2019/04/04/kubernetes-1.14-local-persistent-volumes-ga/). + +One of the easiest way to get started is to use the local storage provisioner provided along with the Pulsar Helm chart. + +``` + +helm repo add streamnative https://charts.streamnative.io +helm repo update +helm install pulsar-storage-provisioner streamnative/local-storage-provisioner + +``` + +### Install cert-manager + +The Pulsar Helm chart uses the [cert-manager](https://github.com/jetstack/cert-manager) to provision and manage TLS certificates automatically. To enable TLS encryption for brokers or proxies, you need to install the cert-manager in advance. + +For details about how to install the cert-manager, follow the [official instructions](https://cert-manager.io/docs/installation/kubernetes/#installing-with-helm). + +Alternatively, we provide a bash script [install-cert-manager.sh](https://github.com/apache/pulsar-helm-chart/blob/master/scripts/cert-manager/install-cert-manager.sh) to install a cert-manager release to the namespace `cert-manager`. + +```bash + +git clone https://github.com/apache/pulsar-helm-chart +cd pulsar-helm-chart +./scripts/cert-manager/install-cert-manager.sh + +``` + +## Prepare Helm release + +Once you have install all the dependent charts and collected all of your configuration options, you can run [prepare_helm_release.sh](https://github.com/apache/pulsar-helm-chart/blob/master/scripts/pulsar/prepare_helm_release.sh) to prepare the Helm release. + +```bash + +git clone https://github.com/apache/pulsar-helm-chart +cd pulsar-helm-chart +./scripts/pulsar/prepare_helm_release.sh -n -k + +``` + +The `prepare_helm_release` creates the following resources: + +- A Kubernetes namespace for installing the Pulsar release +- JWT secret keys and tokens for three super users: `broker-admin`, `proxy-admin`, and `admin`. By default, it generates an asymmetric pubic/private key pair. You can choose to generate a symmetric secret key by specifying `--symmetric`. + - `proxy-admin` role is used for proxies to communicate to brokers. + - `broker-admin` role is used for inter-broker communications. + - `admin` role is used by the admin tools. + +## Deploy Pulsar cluster using Helm + +Once you have finished the following three things, you can install a Helm release. + +- Collect all of your configuration options. +- Install dependent charts. +- Prepare the Helm release. + +In this example, we name our Helm release `pulsar`. + +```bash + +helm repo add apache https://pulsar.apache.org/charts +helm repo update +helm install pulsar apache/pulsar \ + --timeout 10m \ + --set initialize=true \ + --set [your configuration options] + +``` + +:::note + +For the first deployment, add `--set initialize=true` option to initialize bookie and Pulsar cluster metadata. + +::: + +You can also use the `--version ` option if you want to install a specific version of Pulsar Helm chart. + +## Monitor deployment + +A list of installed resources are output once the Pulsar cluster is deployed. This may take 5-10 minutes. + +The status of the deployment can be checked by running the `helm status pulsar` command, which can also be done while the deployment is taking place if you run the command in another terminal. + +## Access Pulsar cluster + +The default values will create a `ClusterIP` for the following resources, which you can use to interact with the cluster. + +- Proxy: You can use the IP address to produce and consume messages to the installed Pulsar cluster. +- Pulsar Manager: You can access the Pulsar Manager UI at `http://:9527`. +- Grafana Dashboard: You can access the Grafana dashboard at `http://:3000`. + +To find the IP addresses of those components, run the following command: + +```bash + +kubectl get service -n + +``` + diff --git a/site2/website-next/versioned_docs/version-2.7.1/helm-install.md b/site2/website-next/versioned_docs/version-2.7.1/helm-install.md new file mode 100644 index 0000000000000..7db5d915fe216 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/helm-install.md @@ -0,0 +1,48 @@ +--- +id: helm-install +title: Install Apache Pulsar using Helm +sidebar_label: "Install" +original_id: helm-install +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +Install Apache Pulsar on Kubernetes with the official Pulsar Helm chart. + +## Requirements + +To deploy Apache Pulsar on Kubernetes, the followings are required. + +- kubectl 1.14 or higher, compatible with your cluster ([+/- 1 minor release from your cluster](https://kubernetes.io/docs/tasks/tools/install-kubectl/#before-you-begin)) +- Helm v3 (3.0.2 or higher) +- A Kubernetes cluster, version 1.14 or higher + +## Environment setup + +Before deploying Pulsar, you need to prepare your environment. + +### Tools + +Install [`helm`](helm-tools.md) and [`kubectl`](helm-tools) on your computer. + +## Cloud cluster preparation + +:::note + +Kubernetes 1.14 or higher is required. + +::: + +To create and connect to the Kubernetes cluster, follow the instructions: + +- [Google Kubernetes Engine](helm-prepare.md#google-kubernetes-engine) + +## Pulsar deployment + +Once the environment is set up and configuration is generated, you can now proceed to the [deployment of Pulsar](helm-deploy). + +## Pulsar upgrade + +To upgrade an existing Kubernetes installation, follow the [upgrade documentation](helm-upgrade). diff --git a/site2/website-next/versioned_docs/version-2.7.1/helm-overview.md b/site2/website-next/versioned_docs/version-2.7.1/helm-overview.md new file mode 100644 index 0000000000000..1f9d2b066205f --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/helm-overview.md @@ -0,0 +1,108 @@ +--- +id: helm-overview +title: Apache Pulsar Helm Chart +sidebar_label: "Overview" +original_id: helm-overview +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +This is the official supported Helm chart to install Apache Pulsar on a cloud-native environment. It was enhanced based on StreamNative's [Helm Chart](https://github.com/streamnative/charts). + +## Introduction + +The Apache Pulsar Helm chart is one of the most convenient ways to operate Pulsar on Kubernetes. This Pulsar Helm chart contains all the required components to get started and can scale to large deployments. + +This chart includes all the components for a complete experience, but each part can be configured to be installed separately. + +- Pulsar core components: + - ZooKeeper + - Bookies + - Brokers + - Function workers + - Proxies +- Control Center: + - Pulsar Manager + - Prometheus + - Grafana + +It includes support for: + +- Security + - Automatically provisioned TLS certificates, using [Jetstack](https://www.jetstack.io/)'s [cert-manager](https://cert-manager.io/docs/) + - self-signed + - [Let's Encrypt](https://letsencrypt.org/) + - TLS Encryption + - Proxy + - Broker + - Toolset + - Bookie + - ZooKeeper + - Authentication + - JWT + - Authorization +- Storage + - Non-persistence storage + - Persistence volume + - Local persistent volumes +- Functions + - Kubernetes Runtime + - Process Runtime + - Thread Runtime +- Operations + - Independent image versions for all components, enabling controlled upgrades + +## Pulsar Helm chart quick start + +To get up and run with these charts as fast as possible, in a **non-production** use case, we provide a [quick start guide](getting-started-helm) for Proof of Concept (PoC) deployments. + +This guide walks the user through deploying these charts with default values and features, but *does not* meet production ready requirements. To deploy these charts into production under sustained load, follow the complete [Installation Guide](helm-install). + +## Troubleshooting + +We have done our best to make these charts as seamless as possible. Occasionally, troubles do go outside of our control. We have collected tips and tricks for troubleshooting common issues. Please check them first before raising an [issue](https://github.com/apache/pulsar/issues/new/choose), and feel free to add to them by raising a [Pull Request](https://github.com/apache/pulsar/compare). + +## Installation + +The Apache Pulsar Helm chart contains all required dependencies. + +If you deploy a PoC for testing, we strongly suggest you follow our [Quick Start Guide](getting-started-helm) for your first iteration. + +1. [Preparation](helm-prepare) +2. [Deployment](helm-deploy) + +## Upgrading + +Once the Pulsar Helm chart is installed, use the `helm upgrade` to complete configuration changes and chart updates. + +```bash + +helm repo add apache https://pulsar.apache.org/charts +helm repo update +helm get values > pulsar.yaml +helm upgrade apache/pulsar -f pulsar.yaml + +``` + +For more detailed information, see [Upgrading](helm-upgrade). + +## Uninstallation + +To uninstall the Pulsar Helm chart, run the following command: + +```bash + +helm delete + +``` + +For the purposes of continuity, these charts have some Kubernetes objects that cannot be removed when performing `helm delete`. +It is recommended to *consciously* remove these items, as they affect re-deployment. + +* PVCs for stateful data: *consciously* remove these items. + - ZooKeeper: This is your metadata. + - BookKeeper: This is your data. + - Prometheus: This is your metrics data, which can be safely removed. +* Secrets: if the secrets are generated by the [prepare release script](https://github.com/apache/pulsar-helm-chart/blob/master/scripts/pulsar/prepare_helm_release.sh), they contain secret keys and tokens. You can use the [cleanup release script](https://github.com/apache/pulsar-helm-chart/blob/master/scripts/pulsar/cleanup_helm_release.sh) to remove these secrets and tokens as needed. diff --git a/site2/website-next/versioned_docs/version-2.7.1/helm-prepare.md b/site2/website-next/versioned_docs/version-2.7.1/helm-prepare.md new file mode 100644 index 0000000000000..705e35794a39f --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/helm-prepare.md @@ -0,0 +1,96 @@ +--- +id: helm-prepare +title: Prepare Kubernetes resources +sidebar_label: "Prepare" +original_id: helm-prepare +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +For a fully functional Pulsar cluster, you need a few resources before deploying the Apache Pulsar Helm chart. The following provides instructions to prepare the Kubernetes cluster before deploying the Pulsar Helm chart. + +- [Google Kubernetes Engine](#google-kubernetes-engine) + - [Manual cluster creation](#manual-cluster-creation) + - [Scripted cluster creation](#scripted-cluster-creation) + - [Create cluster with local SSDs](#create-cluster-with-local-ssds) +- [Next Steps](#next-steps) + +## Google Kubernetes Engine + +To get started easier, a script is provided to create the cluster automatically. Alternatively, a cluster can be created manually as well. + +- [Google Kubernetes Engine](#google-kubernetes-engine) + - [Manual cluster creation](#manual-cluster-creation) + - [Scripted cluster creation](#scripted-cluster-creation) + - [Create cluster with local SSDs](#create-cluster-with-local-ssds) +- [Next Steps](#next-steps) + +### Manual cluster creation + +To provision a Kubernetes cluster manually, follow the [GKE instructions](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster). + +Alternatively, you can use the [instructions](#scripted-cluster-creation) below to provision a GKE cluster as needed. + +### Scripted cluster creation + +A [bootstrap script](https://github.com/streamnative/charts/tree/master/scripts/pulsar/gke_bootstrap_script.sh) has been created to automate much of the setup process for users on GCP/GKE. + +The script can: + +1. Create a new GKE cluster. +2. Allow the cluster to modify DNS (Domain Name Server) records. +3. Setup `kubectl`, and connect it to the cluster. + +Google Cloud SDK is a dependency of this script, so ensure it is [set up correctly](helm-tools.md#connect-to-a-gke-cluster) for the script to work. + +The script reads various parameters from environment variables and an argument `up` or `down` for bootstrap and clean-up respectively. + +The following table describes all variables. + +| **Variable** | **Description** | **Default value** | +| ------------ | --------------- | ----------------- | +| PROJECT | ID of your GCP project | No default value. It requires to be set. | +| CLUSTER_NAME | Name of the GKE cluster | `pulsar-dev` | +| CONFDIR | Configuration directory to store Kubernetes configuration | ${HOME}/.config/streamnative | +| INT_NETWORK | IP space to use within this cluster | `default` | +| LOCAL_SSD_COUNT | Number of local SSD counts | 4 | +| MACHINE_TYPE | Type of machine to use for nodes | `n1-standard-4` | +| NUM_NODES | Number of nodes to be created in each of the cluster's zones | 4 | +| PREEMPTIBLE | Create nodes using preemptible VM instances in the new cluster. | false | +| REGION | Compute region for the cluster | `us-east1` | +| USE_LOCAL_SSD | Flag to create a cluster with local SSDs | false | +| ZONE | Compute zone for the cluster | `us-east1-b` | +| ZONE_EXTENSION | The extension (`a`, `b`, `c`) of the zone name of the cluster | `b` | +| EXTRA_CREATE_ARGS | Extra arguments passed to create command | | + +Run the script, by passing in your desired parameters. It can work with the default parameters except for `PROJECT` which is required: + +```bash + +PROJECT= scripts/pulsar/gke_bootstrap_script.sh up + +``` + +The script can also be used to clean up the created GKE resources. + +```bash + +PROJECT= scripts/pulsar/gke_bootstrap_script.sh down + +``` + +#### Create cluster with local SSDs + +To install a Pulsar Helm chart using local persistent volumes, you need to create a GKE cluster with local SSDs. You can do so Specifying the `USE_LOCAL_SSD` to be `true` in the following command to create a Pulsar cluster with local SSDs. + +``` + +PROJECT= USE_LOCAL_SSD=true LOCAL_SSD_COUNT= scripts/pulsar/gke_bootstrap_script.sh up + +``` + +## Next Steps + +Continue with the [installation of the chart](helm-deploy) once you have the cluster up and running. diff --git a/site2/website-next/versioned_docs/version-2.7.1/helm-tools.md b/site2/website-next/versioned_docs/version-2.7.1/helm-tools.md new file mode 100644 index 0000000000000..773c681f7c331 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/helm-tools.md @@ -0,0 +1,47 @@ +--- +id: helm-tools +title: Required tools for deploying Pulsar Helm Chart +sidebar_label: "Required Tools" +original_id: helm-tools +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +Before deploying Pulsar to your Kubernetes cluster, there are some tools you must have installed locally. + +## kubectl + +kubectl is the tool that talks to the Kubernetes API. kubectl 1.14 or higher is required and it needs to be compatible with your cluster ([+/- 1 minor release from your cluster](https://kubernetes.io/docs/tasks/tools/install-kubectl/#before-you-begin)). + +To Install kubectl locally, follow the [Kubernetes documentation](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl). + +The server version of kubectl cannot be obtained until we connect to a cluster. + +## Helm + +Helm is the package manager for Kubernetes. The Apache Pulsar Helm Chart is tested and supported with Helm v3. + +### Get Helm + +You can get Helm from the project's [releases page](https://github.com/helm/helm/releases), or follow other options under the official documentation of [installing Helm](https://helm.sh/docs/intro/install/). + +### Next steps + +Once kubectl and Helm are configured, you can configure your [Kubernetes cluster](helm-prepare). + +## Additional information + +### Templates + +Templating in Helm is done through Golang's [text/template](https://golang.org/pkg/text/template/) and [sprig](https://godoc.org/github.com/Masterminds/sprig). + +For more information about how all the inner workings behave, check these documents: + +- [Functions and Pipelines](https://helm.sh/docs/chart_template_guide/functions_and_pipelines/) +- [Subcharts and Globals](https://helm.sh/docs/chart_template_guide/subcharts_and_globals/) + +### Tips and tricks + +For additional information on developing with Helm, check [tips and tricks section](https://helm.sh/docs/howto/charts_tips_and_tricks/) in the Helm repository. \ No newline at end of file diff --git a/site2/website-next/versioned_docs/version-2.7.1/helm-upgrade.md b/site2/website-next/versioned_docs/version-2.7.1/helm-upgrade.md new file mode 100644 index 0000000000000..8b00b8ed37553 --- /dev/null +++ b/site2/website-next/versioned_docs/version-2.7.1/helm-upgrade.md @@ -0,0 +1,49 @@ +--- +id: helm-upgrade +title: Upgrade Pulsar Helm release +sidebar_label: "Upgrade" +original_id: helm-upgrade +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +Before upgrading your Pulsar installation, you need to check the change log corresponding to the specific release you want to upgrade to and look for any release notes that might pertain to the new Pulsar helm chart version. + +We also recommend that you need to provide all values using the `helm upgrade --set key=value` syntax or the `-f values.yml` instead of using `--reuse-values`, because some of the current values might be deprecated. + +:::note + +You can retrieve your previous `--set` arguments cleanly, with `helm get values `. If you direct this into a file (`helm get values > pulsar.yml`), you can safely + +::: + +pass this file through `-f`. Thus `helm upgrade apache/pulsar -f pulsar.yaml`. This safely replaces the behavior of `--reuse-values`. + +## Steps + +To upgrade Apache Pulsar to a newer version, follow these steps: + +1. Check the change log for the specific version you would like to upgrade to. +2. Go through [deployment documentation](helm-deploy) step by step. +3. Extract your previous `--set` arguments with the following command. + + ```bash + + helm get values > pulsar.yaml + + ``` + +4. Decide all the values you need to set. +5. Perform the upgrade, with all `--set` arguments extracted in step 4. + + ```bash + + helm upgrade apache/pulsar \ + --version \ + -f pulsar.yaml \ + --set ... + + ``` + diff --git a/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json b/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json index 2ac7003bfd77b..2900041919a67 100644 --- a/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json +++ b/site2/website-next/versioned_sidebars/version-2.7.1-sidebars.json @@ -231,6 +231,36 @@ "id": "version-2.7.1/transactions-api" } ] + }, + { + "type": "category", + "label": "Kubernetes (Helm)", + "items": [ + { + "type": "doc", + "id": "version-2.7.1/helm-overview" + }, + { + "type": "doc", + "id": "version-2.7.1/helm-prepare" + }, + { + "type": "doc", + "id": "version-2.7.1/helm-install" + }, + { + "type": "doc", + "id": "version-2.7.1/helm-deploy" + }, + { + "type": "doc", + "id": "version-2.7.1/helm-upgrade" + }, + { + "type": "doc", + "id": "version-2.7.1/helm-tools" + } + ] } ] } \ No newline at end of file