From a2328dee0a1a5763dce973ab6ae1787d2f50203a Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Mon, 27 Sep 2021 11:35:38 -0700 Subject: [PATCH 1/4] Documentation for Oracle Debezium connector --- site2/docs/io-connectors.md | 8 +++++ site2/docs/io-debezium-source.md | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/site2/docs/io-connectors.md b/site2/docs/io-connectors.md index 238b20c02b5a3..3a6b557270e99 100644 --- a/site2/docs/io-connectors.md +++ b/site2/docs/io-connectors.md @@ -45,6 +45,14 @@ Pulsar has various source connectors, which are sorted alphabetically as below. * [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/debezium/mongodb/src/main/java/org/apache/pulsar/io/debezium/mongodb/DebeziumMongoDbSource.java) +### Debezium Oracle + +* [Configuration](io-debezium-source.md#configuration) + +* [Example](io-debezium-source.md#example-of-oracle) + +* [Java class](https://github.com/apache/pulsar/blob/master/pulsar-io/debezium/oracle/src/main/java/org/apache/pulsar/io/debezium/oracle/DebeziumOracleSource.java) + ### DynamoDB * [Configuration](io-dynamodb-source.md#configuration) diff --git a/site2/docs/io-debezium-source.md b/site2/docs/io-debezium-source.md index 632a59ebd83e4..0fbe6ad1ac5cb 100644 --- a/site2/docs/io-debezium-source.md +++ b/site2/docs/io-debezium-source.md @@ -458,6 +458,58 @@ This example shows how to change the data of a MongoDB table using the Pulsar De {"schema":{"type":"struct","fields":[{"type":"string","optional":false,"field":"id"}],"optional":false,"name":"dbserver1.inventory.products.Key"},"payload":{"id":"104"}}, value = {"schema":{"type":"struct","fields":[{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"after"},{"type":"string","optional":true,"name":"io.debezium.data.Json","version":1,"field":"patch"},{"type":"struct","fields":[{"type":"string","optional":false,"field":"version"},{"type":"string","optional":false,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"int64","optional":false,"field":"ts_ms"},{"type":"string","optional":true,"name":"io.debezium.data.Enum","version":1,"parameters":{"allowed":"true,last,false"},"default":"false","field":"snapshot"},{"type":"string","optional":false,"field":"db"},{"type":"string","optional":false,"field":"rs"},{"type":"string","optional":false,"field":"collection"},{"type":"int32","optional":false,"field":"ord"},{"type":"int64","optional":true,"field":"h"}],"optional":false,"name":"io.debezium.connector.mongo.Source","field":"source"},{"type":"string","optional":true,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"}],"optional":false,"name":"dbserver1.inventory.products.Envelope"},"payload":{"after":"{\"_id\": {\"$numberLong\": \"104\"},\"name\": \"hammer\",\"description\": \"12oz carpenter's hammer\",\"weight\": 1.25,\"quantity\": 4}","patch":null,"source":{"version":"0.10.0.Final","connector":"mongodb","name":"dbserver1","ts_ms":1573541905000,"snapshot":"true","db":"inventory","rs":"rs0","collection":"products","ord":1,"h":4983083486544392763},"op":"r","ts_ms":1573541909761}}. ``` +## Example of Oracle + +### Packaging + +Oracle connector does not include Oracle JDBC driver and the end user will need to package it with the connector. +Major reasons for not including the drivers are the variety of versions and Oracle licensing. It is recommended using the driver provided with your Oracle Db installation, or you can [download](https://www.oracle.com/database/technologies/appdev/jdbc.html) one. +Integration test have an [example](https://github.com/apache/pulsar/blob/e2bc52d40450fa00af258c4432a5b71d50a5c6e0/tests/docker-images/latest-version-image/Dockerfile#L110-L122) of packaging the driver into the connector nar file. + +### Configuration + +Debezium [requires](https://debezium.io/documentation/reference/1.5/connectors/oracle.html#oracle-overview) Oracle DB with LogMiner or XStream API enabled. +Supported options and steps for enabling them vary from version to version of Oracle DB. +Steps outlined in the [documentation](https://debezium.io/documentation/reference/1.5/connectors/oracle.html#oracle-overview) and used in the [integration test](https://github.com/apache/pulsar/blob/master/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumOracleDbSourceTester.java) may or may not work for the version and edition of Oracle DB you are using. +Please refer to the documentation for Oracle DB as needed. + +Similarly to other connectors, you can use JSON or YAMl to configure the connector. +Using yaml as an example, you can create a debezium-oracle-source-config.yaml file like: + +```yaml +tenant: "public" +namespace: "default" +name: "debezium-oracle-source" +topicName: "debezium-oracle-topic" +parallelism: 1 + +className: "org.apache.pulsar.io.debezium.oracle.DebeziumOracleSource" +database.dbname: "XE" + +configs: + database.hostname: "localhost" + database.port: "1521" + database.user: "dbzuser" + database.password: "dbz" + database.dbname: "XE" + database.server.name: "XE" + schema.exclude.list: "system,dbzuser" + snapshot.mode: "initial" + topic.namespace: "public/default" + task.class: "io.debezium.connector.oracle.OracleConnectorTask" + value.converter: "org.apache.kafka.connect.json.JsonConverter" + key.converter: "org.apache.kafka.connect.json.JsonConverter" + typeClassName: "org.apache.pulsar.common.schema.KeyValue" + database.history: "org.apache.pulsar.io.debezium.PulsarDatabaseHistory" + database.tcpKeepAlive: "true" + decimal.handling.mode: "double" + database.history.pulsar.topic: "debezium-oracle-source-history-topic" + database.history.pulsar.service.url: "pulsar://127.0.0.1:6650" +``` + +Full list of configuration properties supported by Debezium is available in the [documentation](https://debezium.io/documentation/reference/1.5/connectors/oracle.html#oracle-connector-properties). + + ## FAQ ### Debezium postgres connector will hang when create snap From bb9c85abed6118ab526fa2d0263bf99d7a4a28d2 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Mon, 27 Sep 2021 12:05:46 -0700 Subject: [PATCH 2/4] Update site2/docs/io-debezium-source.md Co-authored-by: Michael Marshall --- site2/docs/io-debezium-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site2/docs/io-debezium-source.md b/site2/docs/io-debezium-source.md index 0fbe6ad1ac5cb..2bfede4022379 100644 --- a/site2/docs/io-debezium-source.md +++ b/site2/docs/io-debezium-source.md @@ -463,7 +463,7 @@ This example shows how to change the data of a MongoDB table using the Pulsar De ### Packaging Oracle connector does not include Oracle JDBC driver and the end user will need to package it with the connector. -Major reasons for not including the drivers are the variety of versions and Oracle licensing. It is recommended using the driver provided with your Oracle Db installation, or you can [download](https://www.oracle.com/database/technologies/appdev/jdbc.html) one. +Major reasons for not including the drivers are the variety of versions and Oracle licensing. It is recommended using the driver provided with your Oracle DB installation, or you can [download](https://www.oracle.com/database/technologies/appdev/jdbc.html) one. Integration test have an [example](https://github.com/apache/pulsar/blob/e2bc52d40450fa00af258c4432a5b71d50a5c6e0/tests/docker-images/latest-version-image/Dockerfile#L110-L122) of packaging the driver into the connector nar file. ### Configuration From ded31ded61acc8edaa53fb00c59e9f572b24e9f0 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Tue, 28 Sep 2021 09:20:12 -0700 Subject: [PATCH 3/4] Apply suggestions from code review CR feedback Co-authored-by: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> --- site2/docs/io-debezium-source.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site2/docs/io-debezium-source.md b/site2/docs/io-debezium-source.md index 2bfede4022379..bdbc9663f0c8d 100644 --- a/site2/docs/io-debezium-source.md +++ b/site2/docs/io-debezium-source.md @@ -462,8 +462,8 @@ This example shows how to change the data of a MongoDB table using the Pulsar De ### Packaging -Oracle connector does not include Oracle JDBC driver and the end user will need to package it with the connector. -Major reasons for not including the drivers are the variety of versions and Oracle licensing. It is recommended using the driver provided with your Oracle DB installation, or you can [download](https://www.oracle.com/database/technologies/appdev/jdbc.html) one. +Oracle connector does not include Oracle JDBC driver and you need to package it with the connector. +Major reasons for not including the drivers are the variety of versions and Oracle licensing. It is recommended to use the driver provided with your Oracle DB installation, or you can [download](https://www.oracle.com/database/technologies/appdev/jdbc.html) one. Integration test have an [example](https://github.com/apache/pulsar/blob/e2bc52d40450fa00af258c4432a5b71d50a5c6e0/tests/docker-images/latest-version-image/Dockerfile#L110-L122) of packaging the driver into the connector nar file. ### Configuration @@ -507,7 +507,7 @@ configs: database.history.pulsar.service.url: "pulsar://127.0.0.1:6650" ``` -Full list of configuration properties supported by Debezium is available in the [documentation](https://debezium.io/documentation/reference/1.5/connectors/oracle.html#oracle-connector-properties). +For the full list of configuration properties supported by Debezium, see [Debezium Connector for Oracle](https://debezium.io/documentation/reference/1.5/connectors/oracle.html#oracle-connector-properties). ## FAQ From dec87afbcc27dae871a8bbd626051da56839dfdb Mon Sep 17 00:00:00 2001 From: Andrey Yegorov Date: Tue, 28 Sep 2021 09:28:45 -0700 Subject: [PATCH 4/4] CR feedback --- site2/docs/io-debezium-source.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/site2/docs/io-debezium-source.md b/site2/docs/io-debezium-source.md index bdbc9663f0c8d..4683f6ebbb701 100644 --- a/site2/docs/io-debezium-source.md +++ b/site2/docs/io-debezium-source.md @@ -471,11 +471,37 @@ Integration test have an [example](https://github.com/apache/pulsar/blob/e2bc52d Debezium [requires](https://debezium.io/documentation/reference/1.5/connectors/oracle.html#oracle-overview) Oracle DB with LogMiner or XStream API enabled. Supported options and steps for enabling them vary from version to version of Oracle DB. Steps outlined in the [documentation](https://debezium.io/documentation/reference/1.5/connectors/oracle.html#oracle-overview) and used in the [integration test](https://github.com/apache/pulsar/blob/master/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumOracleDbSourceTester.java) may or may not work for the version and edition of Oracle DB you are using. -Please refer to the documentation for Oracle DB as needed. +Please refer to the [documentation for Oracle DB](https://docs.oracle.com/en/database/oracle/oracle-database/) as needed. Similarly to other connectors, you can use JSON or YAMl to configure the connector. Using yaml as an example, you can create a debezium-oracle-source-config.yaml file like: +* JSON + +```json +{ + "database.hostname": "localhost", + "database.port": "1521", + "database.user": "dbzuser", + "database.password": "dbz", + "database.dbname": "XE", + "database.server.name": "XE", + "schema.exclude.list": "system,dbzuser", + "snapshot.mode": "initial", + "topic.namespace": "public/default", + "task.class": "io.debezium.connector.oracle.OracleConnectorTask", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "typeClassName": "org.apache.pulsar.common.schema.KeyValue", + "database.history": "org.apache.pulsar.io.debezium.PulsarDatabaseHistory", + "database.tcpKeepAlive": "true", + "decimal.handling.mode": "double", + "database.history.pulsar.topic": "debezium-oracle-source-history-topic", + "database.history.pulsar.service.url": "pulsar://127.0.0.1:6650" +} +``` +* YAML + ```yaml tenant: "public" namespace: "default"