From 6c4fba64dd290634115110b077198405eb01b988 Mon Sep 17 00:00:00 2001 From: David Kjerrumgaard Date: Wed, 5 May 2021 16:32:09 -0700 Subject: [PATCH 1/6] Rebased to master --- .../functions/PulsarFunctionsTest.java | 1222 +---------------- .../functions/PulsarFunctionsTestBase.java | 99 +- .../PulsarFunctionsGoProcessTest.java} | 15 +- .../functions/go/PulsarFunctionsGoTest.java | 37 + .../PulsarFunctionsGoThreadTest.java} | 14 +- .../java/PulsarFunctionsJavaProcessTest.java | 29 + .../java/PulsarFunctionsJavaTest.java | 164 +++ .../java/PulsarFunctionsJavaThreadTest.java | 28 + .../PulsarFunctionsPythonProcessTest.java | 28 + .../python/PulsarFunctionsPythonTest.java | 67 + .../PulsarFunctionsPythonThreadTest.java | 28 + .../integration/io/PulsarIOTestBase.java | 119 ++ .../integration/io/RabbitMQSinkTester.java | 4 +- .../integration/io/RabbitMQSourceTester.java | 5 +- .../io/{ => sinks}/CassandraSinkTester.java | 2 +- .../{ => sinks}/ElasticSearchSinkTester.java | 2 +- .../io/{ => sinks}/HdfsSinkTester.java | 2 +- .../{ => sinks}/JdbcPostgresSinkTester.java | 2 +- .../io/{ => sinks}/KafkaSinkTester.java | 3 +- .../integration/io/sinks/PulsarSinksTest.java | 518 +++++++ .../io/{ => sinks}/SinkTester.java | 2 +- .../io/{ => sources}/AvroKafkaSourceTest.java | 2 +- .../DebeziumMongoDbSourceTester.java | 2 +- .../DebeziumMySqlSourceTester.java | 2 +- .../DebeziumPostgreSqlSourceTester.java | 2 +- .../GenericRecordSourceTest.java | 2 +- .../io/{ => sources}/KafkaSourceTester.java | 2 +- .../PulsarSourcePropertyTest.java | 2 +- .../io/sources/PulsarSourcesTest.java | 588 ++++++++ .../io/{ => sources}/SourceTester.java | 2 +- .../src/test/resources/pulsar-process.xml | 4 +- .../src/test/resources/pulsar-thread.xml | 4 +- .../integration/src/test/resources/pulsar.xml | 4 + 33 files changed, 1702 insertions(+), 1304 deletions(-) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/{PulsarFunctionsThreadTest.java => go/PulsarFunctionsGoProcessTest.java} (78%) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/{PulsarFunctionsProcessTest.java => go/PulsarFunctionsGoThreadTest.java} (78%) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaProcessTest.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaThreadTest.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonProcessTest.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonThreadTest.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestBase.java rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sinks}/CassandraSinkTester.java (98%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sinks}/ElasticSearchSinkTester.java (98%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sinks}/HdfsSinkTester.java (97%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sinks}/JdbcPostgresSinkTester.java (98%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sinks}/KafkaSinkTester.java (97%) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sinks}/SinkTester.java (98%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sources}/AvroKafkaSourceTest.java (99%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sources}/DebeziumMongoDbSourceTester.java (98%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sources}/DebeziumMySqlSourceTester.java (98%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sources}/DebeziumPostgreSqlSourceTester.java (98%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sources}/GenericRecordSourceTest.java (99%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sources}/KafkaSourceTester.java (98%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sources}/PulsarSourcePropertyTest.java (99%) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarSourcesTest.java rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sources}/SourceTester.java (99%) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index c273ebc7573cf..9ecc6a4021a80 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -33,12 +33,8 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import net.jodah.failsafe.Failsafe; -import net.jodah.failsafe.RetryPolicy; -import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.BatcherBuilder; @@ -49,364 +45,36 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.PulsarClientImpl; -import org.apache.pulsar.client.impl.schema.AvroSchema; -import org.apache.pulsar.client.impl.schema.KeyValueSchema; -import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.FunctionStats; import org.apache.pulsar.common.policies.data.FunctionStatus; -import org.apache.pulsar.common.policies.data.SinkStatus; -import org.apache.pulsar.common.policies.data.SourceStatus; -import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TopicStats; -import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.functions.api.examples.AutoSchemaFunction; import org.apache.pulsar.functions.api.examples.AvroSchemaTestFunction; import org.apache.pulsar.functions.api.examples.pojo.AvroTestObject; import org.apache.pulsar.functions.api.examples.serde.CustomObject; -import org.apache.pulsar.tests.integration.containers.DebeziumMongoDbContainer; -import org.apache.pulsar.tests.integration.containers.DebeziumMySQLContainer; -import org.apache.pulsar.tests.integration.containers.DebeziumPostgreSqlContainer; import org.apache.pulsar.tests.integration.docker.ContainerExecException; import org.apache.pulsar.tests.integration.docker.ContainerExecResult; import org.apache.pulsar.tests.integration.functions.utils.CommandGenerator; import org.apache.pulsar.tests.integration.functions.utils.CommandGenerator.Runtime; -import org.apache.pulsar.tests.integration.io.CassandraSinkTester; -import org.apache.pulsar.tests.integration.io.DebeziumMongoDbSourceTester; -import org.apache.pulsar.tests.integration.io.DebeziumMySqlSourceTester; -import org.apache.pulsar.tests.integration.io.DebeziumPostgreSqlSourceTester; -import org.apache.pulsar.tests.integration.io.ElasticSearchSinkTester; -import org.apache.pulsar.tests.integration.io.HdfsSinkTester; -import org.apache.pulsar.tests.integration.io.JdbcPostgresSinkTester; -import org.apache.pulsar.tests.integration.io.JdbcPostgresSinkTester.Foo; -import org.apache.pulsar.tests.integration.io.KafkaSinkTester; -import org.apache.pulsar.tests.integration.io.KafkaSourceTester; -import org.apache.pulsar.tests.integration.io.RabbitMQSinkTester; -import org.apache.pulsar.tests.integration.io.RabbitMQSourceTester; -import org.apache.pulsar.tests.integration.io.SinkTester; -import org.apache.pulsar.tests.integration.io.SourceTester; import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.assertj.core.api.Assertions; import org.awaitility.Awaitility; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.shaded.com.google.common.collect.Sets; import org.testng.annotations.Test; -import org.testng.collections.Maps; /** - * A test base for testing sink. + * A test base for testing functions. */ @Slf4j public abstract class PulsarFunctionsTest extends PulsarFunctionsTestBase { - final Duration ONE_MINUTE = Duration.ofMinutes(1); - final Duration TEN_SECONDS = Duration.ofSeconds(10); - - final RetryPolicy statusRetryPolicy = new RetryPolicy() - .withMaxDuration(ONE_MINUTE) - .withDelay(TEN_SECONDS) - .onRetry(e -> log.error("Retry ... ")); - private final AtomicInteger testId = new AtomicInteger(0); - - PulsarFunctionsTest(FunctionRuntimeType functionRuntimeType) { + public PulsarFunctionsTest(FunctionRuntimeType functionRuntimeType) { super(functionRuntimeType); } - @Test(groups = "sink") - public void testKafkaSink() throws Exception { - final String kafkaContainerName = "kafka-" + randomName(8); - testSink(new KafkaSinkTester(kafkaContainerName), true, new KafkaSourceTester(kafkaContainerName)); - } - - @Test(enabled = false, groups = "sink") - public void testCassandraSink() throws Exception { - testSink(CassandraSinkTester.createTester(true), true); - } - - @Test(enabled = false, groups = "sink") - public void testCassandraArchiveSink() throws Exception { - testSink(CassandraSinkTester.createTester(false), false); - } - - @Test(enabled = false, groups = "sink") - public void testHdfsSink() throws Exception { - testSink(new HdfsSinkTester(), false); - } - - @Test(groups = "sink") - public void testJdbcSink() throws Exception { - testSink(new JdbcPostgresSinkTester(), true); - } - - @Test(enabled = false, groups = "sink") - public void testElasticSearchSink() throws Exception { - testSink(new ElasticSearchSinkTester(), true); - } - - @Test(groups = "sink") - public void testRabbitMQSink() throws Exception { - final String containerName = "rabbitmq-" + randomName(8); - testSink(new RabbitMQSinkTester(containerName), true, new RabbitMQSourceTester(containerName)); - } - - @Test(groups = "source") - public void testDebeziumMySqlSourceJson() throws Exception { - testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter", true); - } - - @Test(groups = "source") - public void testDebeziumMySqlSourceAvro() throws Exception { - testDebeziumMySqlConnect( - "org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter", false); - } - - @Test(groups = "source") - public void testDebeziumPostgreSqlSource() throws Exception { - testDebeziumPostgreSqlConnect("org.apache.kafka.connect.json.JsonConverter", true); - } - - @Test(groups = "source") - public void testDebeziumMongoDbSource() throws Exception{ - testDebeziumMongoDbConnect("org.apache.kafka.connect.json.JsonConverter", true); - } - - private void testSink(SinkTester tester, boolean builtin) throws Exception { - tester.startServiceContainer(pulsarCluster); - try { - runSinkTester(tester, builtin); - } finally { - tester.stopServiceContainer(pulsarCluster); - } - } - - - private void testSink(SinkTester sinkTester, - boolean builtinSink, - SourceTester sourceTester) - throws Exception { - ServiceContainerT serviceContainer = sinkTester.startServiceContainer(pulsarCluster); - try { - runSinkTester(sinkTester, builtinSink); - if (null != sourceTester) { - sourceTester.setServiceContainer(serviceContainer); - testSource(sourceTester); - } - } finally { - sinkTester.stopServiceContainer(pulsarCluster); - } - } - private void runSinkTester(SinkTester tester, boolean builtin) throws Exception { - final String tenant = TopicName.PUBLIC_TENANT; - final String namespace = TopicName.DEFAULT_NAMESPACE; - final String inputTopicName = "test-sink-connector-" - + tester.getSinkType() + "-" + functionRuntimeType + "-input-topic-" + randomName(8); - final String sinkName = "test-sink-connector-" - + tester.getSinkType().name().toLowerCase() + "-" + functionRuntimeType + "-name-" + randomName(8); - final int numMessages = 20; - - // prepare the testing environment for sink - prepareSink(tester); - - ensureSubscriptionCreated( - inputTopicName, - String.format("public/default/%s", sinkName), - tester.getInputTopicSchema()); - - // submit the sink connector - submitSinkConnector(tester, tenant, namespace, sinkName, inputTopicName); - - // get sink info - getSinkInfoSuccess(tester, tenant, namespace, sinkName, builtin); - - // get sink status - Failsafe.with(statusRetryPolicy).run(() -> getSinkStatus(tenant, namespace, sinkName)); - - // produce messages - Map kvs; - if (tester instanceof JdbcPostgresSinkTester) { - kvs = produceSchemaInsertMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcPostgresSinkTester.Foo.class)); - // wait for sink to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages)); - - // validate the sink result - tester.validateSinkResult(kvs); - - kvs = produceSchemaUpdateMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcPostgresSinkTester.Foo.class)); - - // wait for sink to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages + 20)); - - // validate the sink result - tester.validateSinkResult(kvs); - - kvs = produceSchemaDeleteMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcPostgresSinkTester.Foo.class)); - - // wait for sink to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages + 20 + 20)); - - // validate the sink result - tester.validateSinkResult(kvs); - - } else { - kvs = produceMessagesToInputTopic(inputTopicName, numMessages); - // wait for sink to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages)); - // validate the sink result - tester.validateSinkResult(kvs); - } - - // update the sink - updateSinkConnector(tester, tenant, namespace, sinkName, inputTopicName); - - // delete the sink - deleteSink(tenant, namespace, sinkName); - - // get sink info (sink should be deleted) - getSinkInfoNotFound(tenant, namespace, sinkName); - } - - protected void prepareSink(SinkTester tester) throws Exception { - tester.prepareSink(); - } - - protected void submitSinkConnector(SinkTester tester, - String tenant, - String namespace, - String sinkName, - String inputTopicName) throws Exception { - String[] commands; - if (tester.getSinkType() != SinkTester.SinkType.UNDEFINED) { - commands = new String[] { - PulsarCluster.ADMIN_SCRIPT, - "sink", "create", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName, - "--sink-type", tester.sinkType().getValue().toLowerCase(), - "--sinkConfig", new Gson().toJson(tester.sinkConfig()), - "--inputs", inputTopicName - }; - } else { - commands = new String[] { - PulsarCluster.ADMIN_SCRIPT, - "sink", "create", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName, - "--archive", tester.getSinkArchive(), - "--classname", tester.getSinkClassName(), - "--sinkConfig", new Gson().toJson(tester.sinkConfig()), - "--inputs", inputTopicName - }; - } - log.info("Run command : {}", StringUtils.join(commands, ' ')); - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - assertTrue( - result.getStdout().contains("\"Created successfully\""), - result.getStdout()); - } - - protected void updateSinkConnector(SinkTester tester, - String tenant, - String namespace, - String sinkName, - String inputTopicName) throws Exception { - String[] commands; - if (tester.getSinkType() != SinkTester.SinkType.UNDEFINED) { - commands = new String[] { - PulsarCluster.ADMIN_SCRIPT, - "sink", "update", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName, - "--sink-type", tester.sinkType().getValue().toLowerCase(), - "--sinkConfig", new Gson().toJson(tester.sinkConfig()), - "--inputs", inputTopicName, - "--parallelism", "2" - }; - } else { - commands = new String[] { - PulsarCluster.ADMIN_SCRIPT, - "sink", "create", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName, - "--archive", tester.getSinkArchive(), - "--classname", tester.getSinkClassName(), - "--sinkConfig", new Gson().toJson(tester.sinkConfig()), - "--inputs", inputTopicName, - "--parallelism", "2" - }; - } - log.info("Run command : {}", StringUtils.join(commands, ' ')); - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - assertTrue( - result.getStdout().contains("\"Updated successfully\""), - result.getStdout()); - } - - protected void getSinkInfoSuccess(SinkTester tester, - String tenant, - String namespace, - String sinkName, - boolean builtin) throws Exception { - String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "sink", - "get", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName - }; - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - log.info("Get sink info : {}", result.getStdout()); - if (builtin) { - assertTrue( - result.getStdout().contains("\"archive\": \"builtin://" + tester.getSinkType().getValue().toLowerCase() + "\""), - result.getStdout() - ); - } else { - assertTrue( - result.getStdout().contains("\"className\": \"" + tester.getSinkClassName() + "\""), - result.getStdout() - ); - } - } - - protected void getSinkStatus(String tenant, String namespace, String sinkName) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "sink", - "status", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName - }; - - final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - log.info("Get sink status : {}", result.getStdout()); - - assertEquals(result.getExitCode(), 0); - - SinkStatus sinkStatus = SinkStatus.decode(result.getStdout()); - - assertEquals(sinkStatus.getNumInstances(), 1); - assertEquals(sinkStatus.getNumRunning(), 1); - assertEquals(sinkStatus.getInstances().size(), 1); - assertEquals(sinkStatus.getInstances().get(0).getInstanceId(), 0); - assertEquals(sinkStatus.getInstances().get(0).getStatus().isRunning(), true); - assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); - assertEquals(sinkStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); - } - protected Map produceMessagesToInputTopic(String inputTopicName, int numMessages) throws Exception { @Cleanup @@ -432,442 +100,6 @@ protected Map produceMessagesToInputTopic(String inputTopicName, return kvs; } - // This for JdbcPostgresSinkTester - protected Map produceSchemaInsertMessagesToInputTopic(String inputTopicName, - int numMessages, - Schema schema) throws Exception { - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - Producer producer = client.newProducer(schema) - .topic(inputTopicName) - .create(); - - LinkedHashMap kvs = new LinkedHashMap<>(); - for (int i = 0; i < numMessages; i++) { - String key = "key-" + i; - - JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo(); - obj.setField1("field1_insert_" + i); - obj.setField2("field2_insert_" + i); - obj.setField3(i); - String value = new String(schema.encode(obj)); - Map properties = Maps.newHashMap(); - properties.put("ACTION", "INSERT"); - - kvs.put(key, value); - kvs.put("ACTION", "INSERT"); - producer.newMessage() - .properties(properties) - .key(key) - .value(obj) - .send(); - } - return kvs; - } - - // This for JdbcPostgresSinkTester - protected Map produceSchemaUpdateMessagesToInputTopic(String inputTopicName, - int numMessages, - Schema schema) throws Exception { - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - Producer producer = client.newProducer(schema) - .topic(inputTopicName) - .create(); - - LinkedHashMap kvs = new LinkedHashMap<>(); - log.info("update start"); - for (int i = 0; i < numMessages; i++) { - String key = "key-" + i; - - JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo(); - obj.setField1("field1_insert_" + i); - obj.setField2("field2_update_" + i); - obj.setField3(i); - String value = new String(schema.encode(obj)); - Map properties = Maps.newHashMap(); - properties.put("ACTION", "UPDATE"); - - kvs.put(key, value); - kvs.put("ACTION", "UPDATE"); - producer.newMessage() - .properties(properties) - .key(key) - .value(obj) - .send(); - } - log.info("update end"); - return kvs; - } - - // This for JdbcPostgresSinkTester - protected Map produceSchemaDeleteMessagesToInputTopic(String inputTopicName, - int numMessages, - Schema schema) throws Exception { - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - Producer producer = client.newProducer(schema) - .topic(inputTopicName) - .create(); - - LinkedHashMap kvs = new LinkedHashMap<>(); - for (int i = 0; i < numMessages; i++) { - String key = "key-" + i; - - JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo(); - obj.setField1("field1_insert_" + i); - obj.setField2("field2_update_" + i); - obj.setField3(i); - String value = new String(schema.encode(obj)); - Map properties = Maps.newHashMap(); - properties.put("ACTION", "DELETE"); - - kvs.put(key, value); - kvs.put("ACTION", "DELETE"); - producer.newMessage() - .properties(properties) - .key(key) - .value(obj) - .send(); - } - return kvs; - } - - protected void deleteSink(String tenant, String namespace, String sinkName) throws Exception { - - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "sink", - "delete", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName - }; - - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - assertTrue( - result.getStdout().contains("Deleted successfully"), - result.getStdout() - ); - result.assertNoStderr(); - } - - protected void getSinkInfoNotFound(String tenant, String namespace, String sinkName) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "sink", - "get", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName - }; - try { - pulsarCluster.getAnyWorker().execCmd(commands); - fail("Command should have exited with non-zero"); - } catch (ContainerExecException e) { - assertTrue(e.getResult().getStderr().contains("Reason: Sink " + sinkName + " doesn't exist")); - } - } - - // - // Source Test - // - - private void testSource(SourceTester tester) throws Exception { - final String tenant = TopicName.PUBLIC_TENANT; - final String namespace = TopicName.DEFAULT_NAMESPACE; - final String outputTopicName = "test-source-connector-" - + functionRuntimeType + "-output-topic-" + randomName(8); - final String sourceName = "test-source-connector-" - + functionRuntimeType + "-name-" + randomName(8); - final int numMessages = 20; - - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); - admin.topics().createNonPartitionedTopic(outputTopicName); - - @Cleanup - Consumer consumer = client.newConsumer(Schema.STRING) - .topic(outputTopicName) - .subscriptionName("source-tester") - .subscriptionType(SubscriptionType.Exclusive) - .subscribe(); - - // prepare the testing environment for source - prepareSource(tester); - - // submit the source connector - submitSourceConnector(tester, tenant, namespace, sourceName, outputTopicName); - - // get source info - getSourceInfoSuccess(tester, tenant, namespace, sourceName); - - // get source status - Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); - - // produce messages - Map kvs = tester.produceSourceMessages(numMessages); - - // wait for source to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); - - // validate the source result - validateSourceResult(consumer, kvs); - - // update the source connector - updateSourceConnector(tester, tenant, namespace, sourceName, outputTopicName); - - // delete the source - deleteSource(tenant, namespace, sourceName); - - // get source info (source should be deleted) - getSourceInfoNotFound(tenant, namespace, sourceName); - } - - protected void prepareSource(SourceTester tester) throws Exception { - tester.prepareSource(); - } - - protected void submitSourceConnector(SourceTester tester, - String tenant, - String namespace, - String sourceName, - String outputTopicName) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", "create", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName, - "--source-type", tester.sourceType(), - "--sourceConfig", new Gson().toJson(tester.sourceConfig()), - "--destinationTopicName", outputTopicName - }; - - log.info("Run command : {}", StringUtils.join(commands, ' ')); - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - assertTrue( - result.getStdout().contains("\"Created successfully\""), - result.getStdout()); - } - - protected void updateSourceConnector(SourceTester tester, - String tenant, - String namespace, - String sourceName, - String outputTopicName) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", "update", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName, - "--source-type", tester.sourceType(), - "--sourceConfig", new Gson().toJson(tester.sourceConfig()), - "--destinationTopicName", outputTopicName, - "--parallelism", "2" - }; - - log.info("Run command : {}", StringUtils.join(commands, ' ')); - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - assertTrue( - result.getStdout().contains("\"Updated successfully\""), - result.getStdout()); - } - - protected void getSourceInfoSuccess(SourceTester tester, - String tenant, - String namespace, - String sourceName) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", - "get", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName - }; - - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - log.info("Get source info : {}", result.getStdout()); - assertTrue( - result.getStdout().contains("\"archive\": \"builtin://" + tester.getSourceType() + "\""), - result.getStdout() - ); - } - - protected void getSourceStatus(String tenant, String namespace, String sourceName) throws Exception { - - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", - "status", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName - }; - - final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - log.info("Get source status : {}", result.getStdout()); - - assertEquals(result.getExitCode(), 0); - - final SourceStatus sourceStatus = SourceStatus.decode(result.getStdout()); - - assertEquals(sourceStatus.getNumInstances(), 1); - assertEquals(sourceStatus.getNumRunning(), 1); - assertEquals(sourceStatus.getInstances().size(), 1); - assertEquals(sourceStatus.getInstances().get(0).getStatus().isRunning(), true); - assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); - assertEquals(sourceStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); - - assertTrue(result.getStdout().contains("\"running\" : true")); - - } - - protected void validateSourceResult(Consumer consumer, - Map kvs) throws Exception { - for (Map.Entry kv : kvs.entrySet()) { - Message msg = consumer.receive(); - assertEquals(kv.getKey(), msg.getKey()); - assertEquals(kv.getValue(), msg.getValue()); - } - } - - protected void waitForProcessingSourceMessages(String tenant, - String namespace, - String sourceName, - int numMessages) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", - "status", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName - }; - - final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - log.info("Get source status : {}", result.getStdout()); - - assertEquals(result.getExitCode(), 0); - - SourceStatus sourceStatus = SourceStatus.decode(result.getStdout()); - assertEquals(sourceStatus.getNumInstances(), 1); - assertEquals(sourceStatus.getNumRunning(), 1); - assertEquals(sourceStatus.getInstances().size(), 1); - assertEquals(sourceStatus.getInstances().get(0).getInstanceId(), 0); - assertEquals(sourceStatus.getInstances().get(0).getStatus().isRunning(), true); - assertTrue(sourceStatus.getInstances().get(0).getStatus().getLastReceivedTime() > 0); - assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumReceivedFromSource(), numMessages); - assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumWritten(), numMessages); - assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); - assertEquals(sourceStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); - } - - protected void waitForProcessingSinkMessages(String tenant, - String namespace, - String sinkName, - int numMessages) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "sink", - "status", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName - }; - - final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - log.info("Get sink status : {}", result.getStdout()); - - assertEquals(result.getExitCode(), 0); - - final SinkStatus sinkStatus = SinkStatus.decode(result.getStdout()); - - assertEquals(sinkStatus.getNumInstances(), 1); - assertEquals(sinkStatus.getNumRunning(), 1); - assertEquals(sinkStatus.getInstances().size(), 1); - assertEquals(sinkStatus.getInstances().get(0).getInstanceId(), 0); - assertEquals(sinkStatus.getInstances().get(0).getStatus().isRunning(), true); - assertTrue(sinkStatus.getInstances().get(0).getStatus().getLastReceivedTime() > 0); - assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumReadFromPulsar(), numMessages); - assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumWrittenToSink(), numMessages); - assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); - assertEquals(sinkStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); - } - - protected void deleteSource(String tenant, String namespace, String sourceName) throws Exception { - - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", - "delete", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName - }; - - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - assertTrue( - result.getStdout().contains("Delete source successfully"), - result.getStdout() - ); - result.assertNoStderr(); - } - - protected void getSourceInfoNotFound(String tenant, String namespace, String sourceName) throws Exception { - - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", - "get", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName - }; - - try { - pulsarCluster.getAnyWorker().execCmd(commands); - fail("Command should have exited with non-zero"); - } catch (ContainerExecException e) { - assertTrue(e.getResult().getStderr().contains("Reason: Source " + sourceName + " doesn't exist")); - } - } - - @Test(groups = {"python_function", "function"}) - public void testPythonFunctionLocalRun() throws Exception { - testFunctionLocalRun(Runtime.PYTHON); - } - - @Test(enabled = false, groups = {"go_function", "function"}) - public void testGoFunctionLocalRun() throws Exception { - testFunctionLocalRun(Runtime.GO); - } - - @Test(groups = {"java_function", "function"}) - public void testJavaFunctionLocalRun() throws Exception { - testFunctionLocalRun(Runtime.JAVA); - } - public void testFunctionLocalRun(Runtime runtime) throws Exception { if (functionRuntimeType == FunctionRuntimeType.THREAD) { return; @@ -1091,68 +323,8 @@ public void testWindowFunction(String type, String[] expectedResults) throws Exc getFunctionInfoNotFound(functionName); } - - @Test(groups = {"java_function", "function"}) - public void testSlidingCountWindowTest() throws Exception { - String[] EXPECTED_RESULTS = { - "0,1,2,3,4", - "0,1,2,3,4,5,6,7,8,9", - "5,6,7,8,9,10,11,12,13,14", - "10,11,12,13,14,15,16,17,18,19", - "15,16,17,18,19,20,21,22,23,24", - "20,21,22,23,24,25,26,27,28,29", - "25,26,27,28,29,30,31,32,33,34", - "30,31,32,33,34,35,36,37,38,39", - "35,36,37,38,39,40,41,42,43,44", - "40,41,42,43,44,45,46,47,48,49", - "45,46,47,48,49,50,51,52,53,54", - "50,51,52,53,54,55,56,57,58,59", - "55,56,57,58,59,60,61,62,63,64", - "60,61,62,63,64,65,66,67,68,69", - "65,66,67,68,69,70,71,72,73,74", - "70,71,72,73,74,75,76,77,78,79", - "75,76,77,78,79,80,81,82,83,84", - "80,81,82,83,84,85,86,87,88,89", - "85,86,87,88,89,90,91,92,93,94", - "90,91,92,93,94,95,96,97,98,99", - }; - - testWindowFunction("sliding", EXPECTED_RESULTS); - } - - @Test(groups = {"java_function", "function"}) - public void testTumblingCountWindowTest() throws Exception { - String[] EXPECTED_RESULTS = { - "0,1,2,3,4,5,6,7,8,9", - "10,11,12,13,14,15,16,17,18,19", - "20,21,22,23,24,25,26,27,28,29", - "30,31,32,33,34,35,36,37,38,39", - "40,41,42,43,44,45,46,47,48,49", - "50,51,52,53,54,55,56,57,58,59", - "60,61,62,63,64,65,66,67,68,69", - "70,71,72,73,74,75,76,77,78,79", - "80,81,82,83,84,85,86,87,88,89", - "90,91,92,93,94,95,96,97,98,99", - }; - - testWindowFunction("tumbling", EXPECTED_RESULTS); - } - - // - // Test CRUD functions on different runtimes. - // - - @Test(groups = {"python_function", "function"}) - public void testPythonFunctionNegAck() throws Exception { - testFunctionNegAck(Runtime.PYTHON); - } - - @Test(groups = {"java_function", "function"}) - public void testJavaFunctionNegAck() throws Exception { - testFunctionNegAck(Runtime.JAVA); - } - - private void testFunctionNegAck(Runtime runtime) throws Exception { + + protected void testFunctionNegAck(Runtime runtime) throws Exception { if (functionRuntimeType == FunctionRuntimeType.THREAD) { return; } @@ -1324,21 +496,11 @@ private void testFunctionNegAck(Runtime runtime) throws Exception { checkSubscriptionsCleanup(inputTopicName); } - @Test(groups = {"python_function", "function"}) - public void testPythonPublishFunction() throws Exception { - testPublishFunction(Runtime.PYTHON); - } - - @Test(groups = {"java_function", "function"}) - public void testJavaPublishFunction() throws Exception { - testPublishFunction(Runtime.JAVA); - } - public void testGoPublishFunction() throws Exception { testPublishFunction(Runtime.GO); } - private void testPublishFunction(Runtime runtime) throws Exception { + protected void testPublishFunction(Runtime runtime) throws Exception { if (functionRuntimeType == FunctionRuntimeType.THREAD) { return; } @@ -1463,80 +625,7 @@ private void testPublishFunction(Runtime runtime) throws Exception { checkSubscriptionsCleanup(inputTopicName); } - @Test(groups = {"java_function", "function"}) - public void testSerdeFunction() throws Exception { - testCustomSerdeFunction(); - } - - private void testCustomSerdeFunction() throws Exception { - if (functionRuntimeType == FunctionRuntimeType.THREAD) { - return; - } - - - String inputTopicName = "persistent://public/default/test-serde-java-input-" + randomName(8); - String outputTopicName = "test-publish-serde-output-" + randomName(8); - try (PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build()) { - admin.topics().createNonPartitionedTopic(inputTopicName); - admin.topics().createNonPartitionedTopic(outputTopicName); - } - - String functionName = "test-serde-fn-" + randomName(8); - submitFunction( - Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS, - SERDE_OUTPUT_CLASS, Collections.singletonMap("serde-topic", outputTopicName) - ); - - // get function info - getFunctionInfoSuccess(functionName); - // get function stats - getFunctionStatsEmpty(functionName); - - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd( - PulsarCluster.ADMIN_SCRIPT, - "functions", - "status", - "--tenant", "public", - "--namespace", "default", - "--name", functionName - ); - - FunctionStatus functionStatus = FunctionStatus.decode(result.getStdout()); - assertEquals(functionStatus.getNumInstances(), 1); - assertEquals(functionStatus.getInstances().get(0).getStatus().isRunning(), true); - } - - @Test(groups = {"python_function", "function"}) - public void testPythonExclamationFunction() throws Exception { - testExclamationFunction(Runtime.PYTHON, false, false, false); - } - - @Test(groups = {"python_function", "function"}) - public void testPythonExclamationFunctionWithExtraDeps() throws Exception { - testExclamationFunction(Runtime.PYTHON, false, false, true); - } - - @Test(groups = {"python_function", "function"}) - public void testPythonExclamationZipFunction() throws Exception { - testExclamationFunction(Runtime.PYTHON, false, true, false); - } - - @Test(groups = {"python_function", "function"}) - public void testPythonExclamationTopicPatternFunction() throws Exception { - testExclamationFunction(Runtime.PYTHON, true, false, false); - } - - @Test(groups = {"java_function", "function"}) - public void testJavaExclamationFunction() throws Exception { - testExclamationFunction(Runtime.JAVA, false, false, false); - } - - @Test(groups = {"java_function", "function"}) - public void testJavaExclamationTopicPatternFunction() throws Exception { - testExclamationFunction(Runtime.JAVA, true, false, false); - } - - private void testExclamationFunction(Runtime runtime, + protected void testExclamationFunction(Runtime runtime, boolean isTopicPattern, boolean pyZip, boolean withExtraDeps) throws Exception { @@ -1744,7 +833,7 @@ private void updateFunctionParallelism(String functionName, int parallelism) thr assertTrue(result.getStdout().contains("\"Updated successfully\"")); } - private void submitFunction(Runtime runtime, + protected void submitFunction(Runtime runtime, String inputTopicName, String outputTopicName, String functionName, @@ -1804,7 +893,7 @@ private void ensureSubscriptionCreated(String inputTopicName, } } - private void getFunctionInfoSuccess(String functionName) throws Exception { + protected void getFunctionInfoSuccess(String functionName) throws Exception { ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd( PulsarCluster.ADMIN_SCRIPT, "functions", @@ -1818,7 +907,7 @@ private void getFunctionInfoSuccess(String functionName) throws Exception { assertTrue(result.getStdout().contains("\"name\": \"" + functionName + "\"")); } - private void getFunctionStatsEmpty(String functionName) throws Exception { + protected void getFunctionStatsEmpty(String functionName) throws Exception { ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd( PulsarCluster.ADMIN_SCRIPT, "functions", @@ -2303,298 +1392,7 @@ public void testAvroSchemaFunction() throws Exception { getFunctionInfoNotFound(functionName); } - private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { - - final String tenant = TopicName.PUBLIC_TENANT; - final String namespace = TopicName.DEFAULT_NAMESPACE; - final String outputTopicName = "debe-output-topic-name-" + testId.getAndIncrement(); - boolean isJsonConverter = converterClassName.endsWith("JsonConverter"); - final String consumeTopicName = "debezium/mysql-" - + (isJsonConverter ? "json" : "avro") - + "/dbserver1.inventory.products"; - final String sourceName = "test-source-debezium-mysql" + (isJsonConverter ? "json" : "avro") - + "-" + functionRuntimeType + "-" + randomName(8); - - // This is the binlog count that contained in mysql container. - final int numMessages = 47; - - - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); - initNamespace(admin); - - try { - SchemaInfo lastSchemaInfo = admin.schemas().getSchemaInfo(consumeTopicName); - log.info("lastSchemaInfo: {}", lastSchemaInfo == null ? "null" : lastSchemaInfo.toString()); - } catch (Exception e) { - log.warn("failed to get schemaInfo for topic: {}, exceptions message: {}", - consumeTopicName, e.getMessage()); - } - - admin.topics().createNonPartitionedTopic(outputTopicName); - - @Cleanup - DebeziumMySqlSourceTester sourceTester = new DebeziumMySqlSourceTester(pulsarCluster, converterClassName); - sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); - - // setup debezium mysql server - DebeziumMySQLContainer mySQLContainer = new DebeziumMySQLContainer(pulsarCluster.getClusterName()); - sourceTester.setServiceContainer(mySQLContainer); - - // prepare the testing environment for source - prepareSource(sourceTester); - - // submit the source connector - submitSourceConnector(sourceTester, tenant, namespace, sourceName, outputTopicName); - - // get source info - getSourceInfoSuccess(sourceTester, tenant, namespace, sourceName); - - // get source status - Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); - - // wait for source to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); - - @Cleanup - Consumer consumer = client.newConsumer(getSchema(jsonWithEnvelope)) - .topic(consumeTopicName) - .subscriptionName("debezium-source-tester") - .subscriptionType(SubscriptionType.Exclusive) - .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) - .subscribe(); - log.info("[debezium mysql test] create consumer finish. converterName: {}", converterClassName); - - // validate the source result - sourceTester.validateSourceResult(consumer, 9, null, converterClassName); - - // prepare insert event - sourceTester.prepareInsertEvent(); - - // validate the source insert event - sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT, converterClassName); - - // prepare update event - sourceTester.prepareUpdateEvent(); - - // validate the source update event - sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE, converterClassName); - - // prepare delete event - sourceTester.prepareDeleteEvent(); - - // validate the source delete event - sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE, converterClassName); - - // delete the source - deleteSource(tenant, namespace, sourceName); - - // get source info (source should be deleted) - getSourceInfoNotFound(tenant, namespace, sourceName); - } - - private void testDebeziumPostgreSqlConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { - - final String tenant = TopicName.PUBLIC_TENANT; - final String namespace = TopicName.DEFAULT_NAMESPACE; - final String outputTopicName = "debe-output-topic-name-" + testId.getAndIncrement(); - final String consumeTopicName = "debezium/postgresql/dbserver1.inventory.products"; - final String sourceName = "test-source-debezium-postgersql-" + functionRuntimeType + "-" + randomName(8); - - - // This is the binlog count that contained in postgresql container. - final int numMessages = 26; - - - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); - initNamespace(admin); - - admin.topics().createNonPartitionedTopic(consumeTopicName); - admin.topics().createNonPartitionedTopic(outputTopicName); - - @Cleanup - Consumer consumer = client.newConsumer(getSchema(jsonWithEnvelope)) - .topic(consumeTopicName) - .subscriptionName("debezium-source-tester") - .subscriptionType(SubscriptionType.Exclusive) - .subscribe(); - - @Cleanup - DebeziumPostgreSqlSourceTester sourceTester = new DebeziumPostgreSqlSourceTester(pulsarCluster); - sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); - - // setup debezium postgresql server - DebeziumPostgreSqlContainer postgreSqlContainer = new DebeziumPostgreSqlContainer(pulsarCluster.getClusterName()); - sourceTester.setServiceContainer(postgreSqlContainer); - - // prepare the testing environment for source - prepareSource(sourceTester); - - // submit the source connector - submitSourceConnector(sourceTester, tenant, namespace, sourceName, outputTopicName); - - // get source info - getSourceInfoSuccess(sourceTester, tenant, namespace, sourceName); - - // get source status - Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); - - // wait for source to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); - - // validate the source result - sourceTester.validateSourceResult(consumer, 9, null, converterClassName); - - // prepare insert event - sourceTester.prepareInsertEvent(); - - // validate the source insert event - sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT, converterClassName); - - // prepare update event - sourceTester.prepareUpdateEvent(); - - // validate the source update event - sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE, converterClassName); - - // prepare delete event - sourceTester.prepareDeleteEvent(); - - // validate the source delete event - sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE, converterClassName); - - // delete the source - deleteSource(tenant, namespace, sourceName); - - // get source info (source should be deleted) - getSourceInfoNotFound(tenant, namespace, sourceName); - } - - private void testDebeziumMongoDbConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { - - final String tenant = TopicName.PUBLIC_TENANT; - final String namespace = TopicName.DEFAULT_NAMESPACE; - final String outputTopicName = "debe-output-topic-name"; - final String consumeTopicName = "debezium/mongodb/dbserver1.inventory.products"; - final String sourceName = "test-source-connector-" - + functionRuntimeType + "-name-" + randomName(8); - - // This is the binlog count that contained in mongodb container. - final int numMessages = 17; - - - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); - initNamespace(admin); - - admin.topics().createNonPartitionedTopic(consumeTopicName); - admin.topics().createNonPartitionedTopic(outputTopicName); - - @Cleanup - Consumer consumer = client.newConsumer(getSchema(jsonWithEnvelope)) - .topic(consumeTopicName) - .subscriptionName("debezium-source-tester") - .subscriptionType(SubscriptionType.Exclusive) - .subscribe(); - - @Cleanup - DebeziumMongoDbSourceTester sourceTester = new DebeziumMongoDbSourceTester(pulsarCluster); - sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); - - // setup debezium mongodb server - DebeziumMongoDbContainer mongoDbContainer = new DebeziumMongoDbContainer(pulsarCluster.getClusterName()); - sourceTester.setServiceContainer(mongoDbContainer); - // prepare the testing environment for source - prepareSource(sourceTester); - - // submit the source connector - submitSourceConnector(sourceTester, tenant, namespace, sourceName, outputTopicName); - - // get source info - getSourceInfoSuccess(sourceTester, tenant, namespace, sourceName); - - // get source status - Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); - - // wait for source to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); - - // validate the source result - sourceTester.validateSourceResult(consumer, 9, null, converterClassName); - - // prepare insert event - sourceTester.prepareInsertEvent(); - - // validate the source insert event - sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT, converterClassName); - - // prepare update event - sourceTester.prepareUpdateEvent(); - - // validate the source update event - sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE, converterClassName); - - // prepare delete event - sourceTester.prepareDeleteEvent(); - - // validate the source delete event - sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE, converterClassName); - - // delete the source - deleteSource(tenant, namespace, sourceName); - - // get source info (source should be deleted) - getSourceInfoNotFound(tenant, namespace, sourceName); - } - - private void initNamespace(PulsarAdmin admin) { - log.info("[initNamespace] start."); - try { - admin.tenants().createTenant("debezium", new TenantInfo(Sets.newHashSet(), - Sets.newHashSet(pulsarCluster.getClusterName()))); - admin.namespaces().createNamespace("debezium/mysql-json"); - admin.namespaces().createNamespace("debezium/mysql-avro"); - admin.namespaces().createNamespace("debezium/mongodb"); - admin.namespaces().createNamespace("debezium/postgresql"); - } catch (Exception e) { - log.info("[initNamespace] msg: {}", e.getMessage()); - } - log.info("[initNamespace] finish."); - } - - private Schema getSchema(boolean jsonWithEnvelope) { - if (jsonWithEnvelope) { - return KeyValueSchema.kvBytes(); - } else { - return KeyValueSchema.of(Schema.AUTO_CONSUME(), Schema.AUTO_CONSUME(), KeyValueEncodingType.SEPARATED); - } - } - - @Test(groups = {"java_function", "function"}) - public void testJavaLoggingFunction() throws Exception { - testLoggingFunction(Runtime.JAVA); - } - - private void testLoggingFunction(Runtime runtime) throws Exception { + protected void testLoggingFunction(Runtime runtime) throws Exception { if (functionRuntimeType == FunctionRuntimeType.THREAD && runtime == Runtime.PYTHON) { // python can only run on process mode return; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java index 01e8fce8ca102..75a3ae07190e2 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java @@ -32,54 +32,9 @@ @Slf4j public abstract class PulsarFunctionsTestBase extends PulsarTestSuite { - @DataProvider(name = "FunctionRuntimeTypes") - public static Object[][] getData() { - return new Object[][] { - { FunctionRuntimeType.PROCESS }, - { FunctionRuntimeType.THREAD } - }; - } - - protected final FunctionRuntimeType functionRuntimeType; - - public PulsarFunctionsTestBase() { - this(FunctionRuntimeType.PROCESS); - } - - protected PulsarFunctionsTestBase(FunctionRuntimeType functionRuntimeType) { - this.functionRuntimeType = functionRuntimeType; - } - - private void setupFunctionWorkers() { - final int numFunctionWorkers = 2; - log.info("Setting up {} function workers : function runtime type = {}", - numFunctionWorkers, functionRuntimeType); - pulsarCluster.setupFunctionWorkers(randomName(5), functionRuntimeType, numFunctionWorkers); - log.info("{} function workers has started", numFunctionWorkers); - } - - private void teardownFunctionWorkers() { - log.info("Tearing down function workers ..."); - pulsarCluster.stopWorkers(); - log.info("All functions workers are stopped."); - } - - @Override - public void setupCluster() throws Exception { - super.setupCluster(); - setupFunctionWorkers(); - } - - @Override - public void tearDownCluster() throws Exception { - teardownFunctionWorkers(); - super.tearDownCluster(); - } - // // Common Variables used by functions test // - public static final String EXCLAMATION_JAVA_CLASS = "org.apache.pulsar.functions.api.examples.ExclamationFunction"; @@ -106,7 +61,6 @@ public void tearDownCluster() throws Exception { public static final String PUBLISH_PYTHON_CLASS = "typed_message_builder_publish.TypedMessageBuilderPublish"; public static final String EXCEPTION_PYTHON_CLASS = "exception_function"; - public static final String EXCLAMATION_PYTHON_FILE = "exclamation_function.py"; public static final String EXCLAMATION_WITH_DEPS_PYTHON_FILE = "exclamation_with_extra_deps.py"; public static final String EXCLAMATION_PYTHON_ZIP_FILE = "exclamation.zip"; @@ -119,6 +73,49 @@ public void tearDownCluster() throws Exception { public static final String LOGGING_JAVA_CLASS = "org.apache.pulsar.functions.api.examples.LoggingFunction"; + @DataProvider(name = "FunctionRuntimeTypes") + public static Object[][] getData() { + return new Object[][] { + { FunctionRuntimeType.PROCESS }, + { FunctionRuntimeType.THREAD } + }; + } + + @DataProvider(name = "FunctionRuntimes") + public static Object[][] functionRuntimes() { + return new Object[][] { + new Object[] { Runtime.JAVA }, + new Object[] { Runtime.PYTHON }, + new Object[] { Runtime.GO } + }; + } + + protected final FunctionRuntimeType functionRuntimeType; + + public PulsarFunctionsTestBase() { + this(FunctionRuntimeType.PROCESS); + } + + protected PulsarFunctionsTestBase(FunctionRuntimeType functionRuntimeType) { + this.functionRuntimeType = functionRuntimeType; + } + + @BeforeClass + public void setupFunctionWorkers() { + final int numFunctionWorkers = 2; + log.info("Setting up {} function workers : function runtime type = {}", + numFunctionWorkers, functionRuntimeType); + pulsarCluster.setupFunctionWorkers(randomName(5), functionRuntimeType, numFunctionWorkers); + log.info("{} function workers has started", numFunctionWorkers); + } + + @AfterClass(alwaysRun = true) + public void teardownFunctionWorkers() { + log.info("Tearing down function workers ..."); + pulsarCluster.stopWorkers(); + log.info("All functions workers are stopped."); + } + protected static String getExclamationClass(Runtime runtime, boolean pyZip, boolean extraDeps) { @@ -136,14 +133,4 @@ protected static String getExclamationClass(Runtime runtime, throw new IllegalArgumentException("Unsupported runtime : " + runtime); } } - - @DataProvider(name = "FunctionRuntimes") - public static Object[][] functionRuntimes() { - return new Object[][] { - new Object[] { Runtime.JAVA }, - new Object[] { Runtime.PYTHON }, - new Object[] { Runtime.GO } - }; - } - } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsThreadTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoProcessTest.java similarity index 78% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsThreadTest.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoProcessTest.java index d1c132b3b3c5f..348cf50682bff 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsThreadTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoProcessTest.java @@ -16,15 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.functions; +package org.apache.pulsar.tests.integration.functions.go; import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType; -/** - * Thread based test. - */ -public class PulsarFunctionsThreadTest extends PulsarFunctionsTest { - public PulsarFunctionsThreadTest() { - super(FunctionRuntimeType.THREAD); - } +public class PulsarFunctionsGoProcessTest extends PulsarFunctionsGoTest { + + PulsarFunctionsGoProcessTest() { + super(FunctionRuntimeType.PROCESS); + } + } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java new file mode 100644 index 0000000000000..5aae69226a662 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.functions.go; + +import org.apache.pulsar.tests.integration.functions.PulsarFunctionsTest; +import org.apache.pulsar.tests.integration.functions.utils.CommandGenerator.Runtime; +import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType; +import org.testng.annotations.Test; + +public class PulsarFunctionsGoTest extends PulsarFunctionsTest { + + PulsarFunctionsGoTest(FunctionRuntimeType functionRuntimeType) { + super(functionRuntimeType); + } + + @Test(enabled = false, groups = {"go_function", "function"}) + public void testGoFunctionLocalRun() throws Exception { + testFunctionLocalRun(Runtime.GO); + } + +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsProcessTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoThreadTest.java similarity index 78% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsProcessTest.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoThreadTest.java index ec36d528f50a9..6d70707eedf03 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsProcessTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoThreadTest.java @@ -16,15 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.functions; +package org.apache.pulsar.tests.integration.functions.go; import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType; -/** - * Process based test. - */ -public class PulsarFunctionsProcessTest extends PulsarFunctionsTest { - public PulsarFunctionsProcessTest() { - super(FunctionRuntimeType.PROCESS); - } +public class PulsarFunctionsGoThreadTest extends PulsarFunctionsGoTest { + + PulsarFunctionsGoThreadTest() { + super(FunctionRuntimeType.THREAD); + } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaProcessTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaProcessTest.java new file mode 100644 index 0000000000000..71c6cb07fada6 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaProcessTest.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.functions.java; + +import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType; + +public class PulsarFunctionsJavaProcessTest extends PulsarFunctionsJavaTest { + + public PulsarFunctionsJavaProcessTest() { + super(FunctionRuntimeType.PROCESS); + } + +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java new file mode 100644 index 0000000000000..28c7d0de6c85a --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.functions.java; + +import static org.testng.Assert.assertEquals; + +import java.util.Collections; + +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.common.policies.data.FunctionStatus; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.functions.PulsarFunctionsTest; +import org.apache.pulsar.tests.integration.functions.utils.CommandGenerator.Runtime; +import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.testng.annotations.Test; + +public class PulsarFunctionsJavaTest extends PulsarFunctionsTest { + + PulsarFunctionsJavaTest(FunctionRuntimeType functionRuntimeType) { + super(functionRuntimeType); + } + + @Test(groups = {"java_function", "function"}) + public void testJavaFunctionLocalRun() throws Exception { + testFunctionLocalRun(Runtime.JAVA); + } + + @Test(groups = {"java_function", "function"}) + public void testJavaFunctionNegAck() throws Exception { + testFunctionNegAck(Runtime.JAVA); + } + + @Test(groups = {"java_function", "function"}) + public void testJavaPublishFunction() throws Exception { + testPublishFunction(Runtime.JAVA); + } + + @Test(groups = {"java_function", "function"}) + public void testSerdeFunction() throws Exception { + testCustomSerdeFunction(); + } + + private void testCustomSerdeFunction() throws Exception { + if (functionRuntimeType == FunctionRuntimeType.THREAD) { + return; + } + + if (pulsarCluster == null) { + super.setupCluster(); + super.setupFunctionWorkers(); + } + + String inputTopicName = "persistent://public/default/test-serde-java-input-" + randomName(8); + String outputTopicName = "test-publish-serde-output-" + randomName(8); + try (PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build()) { + admin.topics().createNonPartitionedTopic(inputTopicName); + admin.topics().createNonPartitionedTopic(outputTopicName); + } + + String functionName = "test-serde-fn-" + randomName(8); + submitFunction( + Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS, + SERDE_OUTPUT_CLASS, Collections.singletonMap("serde-topic", outputTopicName) + ); + + // get function info + getFunctionInfoSuccess(functionName); + // get function stats + getFunctionStatsEmpty(functionName); + + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd( + PulsarCluster.ADMIN_SCRIPT, + "functions", + "status", + "--tenant", "public", + "--namespace", "default", + "--name", functionName + ); + + FunctionStatus functionStatus = FunctionStatus.decode(result.getStdout()); + assertEquals(functionStatus.getNumInstances(), 1); + assertEquals(functionStatus.getInstances().get(0).getStatus().isRunning(), true); + } + + + @Test(groups = {"java_function", "function"}) + public void testJavaExclamationFunction() throws Exception { + testExclamationFunction(Runtime.JAVA, false, false, false); + } + + @Test(groups = {"java_function", "function"}) + public void testJavaExclamationTopicPatternFunction() throws Exception { + testExclamationFunction(Runtime.JAVA, true, false, false); + } + + @Test(groups = {"java_function", "function"}) + public void testJavaLoggingFunction() throws Exception { + testLoggingFunction(Runtime.JAVA); + } + + @Test(groups = {"java_function", "function"}) + public void testTumblingCountWindowTest() throws Exception { + String[] EXPECTED_RESULTS = { + "0,1,2,3,4,5,6,7,8,9", + "10,11,12,13,14,15,16,17,18,19", + "20,21,22,23,24,25,26,27,28,29", + "30,31,32,33,34,35,36,37,38,39", + "40,41,42,43,44,45,46,47,48,49", + "50,51,52,53,54,55,56,57,58,59", + "60,61,62,63,64,65,66,67,68,69", + "70,71,72,73,74,75,76,77,78,79", + "80,81,82,83,84,85,86,87,88,89", + "90,91,92,93,94,95,96,97,98,99", + }; + + testWindowFunction("tumbling", EXPECTED_RESULTS); + } + + @Test(groups = {"java_function", "function"}) + public void testSlidingCountWindowTest() throws Exception { + String[] EXPECTED_RESULTS = { + "0,1,2,3,4", + "0,1,2,3,4,5,6,7,8,9", + "5,6,7,8,9,10,11,12,13,14", + "10,11,12,13,14,15,16,17,18,19", + "15,16,17,18,19,20,21,22,23,24", + "20,21,22,23,24,25,26,27,28,29", + "25,26,27,28,29,30,31,32,33,34", + "30,31,32,33,34,35,36,37,38,39", + "35,36,37,38,39,40,41,42,43,44", + "40,41,42,43,44,45,46,47,48,49", + "45,46,47,48,49,50,51,52,53,54", + "50,51,52,53,54,55,56,57,58,59", + "55,56,57,58,59,60,61,62,63,64", + "60,61,62,63,64,65,66,67,68,69", + "65,66,67,68,69,70,71,72,73,74", + "70,71,72,73,74,75,76,77,78,79", + "75,76,77,78,79,80,81,82,83,84", + "80,81,82,83,84,85,86,87,88,89", + "85,86,87,88,89,90,91,92,93,94", + "90,91,92,93,94,95,96,97,98,99", + }; + + testWindowFunction("sliding", EXPECTED_RESULTS); + } + +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaThreadTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaThreadTest.java new file mode 100644 index 0000000000000..60bd520a6bc71 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaThreadTest.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.functions.java; + +import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType; + +public class PulsarFunctionsJavaThreadTest extends PulsarFunctionsJavaTest { + + public PulsarFunctionsJavaThreadTest() { + super(FunctionRuntimeType.THREAD); + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonProcessTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonProcessTest.java new file mode 100644 index 0000000000000..aff9df59709cc --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonProcessTest.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.functions.python; + +import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType; + +public class PulsarFunctionsPythonProcessTest extends PulsarFunctionsPythonTest { + + public PulsarFunctionsPythonProcessTest() { + super(FunctionRuntimeType.PROCESS); + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java new file mode 100644 index 0000000000000..bfdf4ebbb6092 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.functions.python; + +import org.apache.pulsar.tests.integration.functions.PulsarFunctionsTest; +import org.apache.pulsar.tests.integration.functions.utils.CommandGenerator.Runtime; +import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType; +import org.testng.annotations.Test; + +public class PulsarFunctionsPythonTest extends PulsarFunctionsTest { + + PulsarFunctionsPythonTest(FunctionRuntimeType functionRuntimeType) { + super(functionRuntimeType); + } + + @Test(groups = {"python_function", "function"}) + public void testPythonFunctionLocalRun() throws Exception { + testFunctionLocalRun(Runtime.PYTHON); + } + + @Test(groups = {"python_function", "function"}) + public void testPythonFunctionNegAck() throws Exception { + testFunctionNegAck(Runtime.PYTHON); + } + + @Test(groups = {"python_function", "function"}) + public void testPythonPublishFunction() throws Exception { + testPublishFunction(Runtime.PYTHON); + } + + @Test(groups = {"python_function", "function"}) + public void testPythonExclamationFunction() throws Exception { + testExclamationFunction(Runtime.PYTHON, false, false, false); + } + + @Test(groups = {"python_function", "function"}) + public void testPythonExclamationFunctionWithExtraDeps() throws Exception { + testExclamationFunction(Runtime.PYTHON, false, false, true); + } + + @Test(groups = {"python_function", "function"}) + public void testPythonExclamationZipFunction() throws Exception { + testExclamationFunction(Runtime.PYTHON, false, true, false); + } + + @Test(groups = {"python_function", "function"}) + public void testPythonExclamationTopicPatternFunction() throws Exception { + testExclamationFunction(Runtime.PYTHON, true, false, false); + } + +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonThreadTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonThreadTest.java new file mode 100644 index 0000000000000..9f28a136cd0f9 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonThreadTest.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.functions.python; + +import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType; + +public class PulsarFunctionsPythonThreadTest extends PulsarFunctionsPythonTest { + + public PulsarFunctionsPythonThreadTest() { + super(FunctionRuntimeType.THREAD); + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestBase.java new file mode 100644 index 0000000000000..5adbcaf53cbc4 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestBase.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.io; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.schema.KeyValueSchema; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.apache.pulsar.tests.integration.functions.PulsarFunctionsTestBase; +import org.testcontainers.shaded.com.google.common.collect.Sets; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import net.jodah.failsafe.RetryPolicy; + +@Slf4j +public abstract class PulsarIOTestBase extends PulsarFunctionsTestBase { + + final Duration ONE_MINUTE = Duration.ofMinutes(1); + final Duration TEN_SECONDS = Duration.ofSeconds(10); + + protected final RetryPolicy statusRetryPolicy = new RetryPolicy() + .withMaxDuration(ONE_MINUTE) + .withDelay(TEN_SECONDS) + .onRetry(e -> log.error("Retry ... ")); + + protected final AtomicInteger testId = new AtomicInteger(0); + + protected void initNamespace(PulsarAdmin admin) { + log.info("[initNamespace] start."); + try { + admin.tenants().createTenant("debezium", new TenantInfo(Sets.newHashSet(), + Sets.newHashSet(pulsarCluster.getClusterName()))); + admin.namespaces().createNamespace("debezium/mysql-json"); + admin.namespaces().createNamespace("debezium/mysql-avro"); + admin.namespaces().createNamespace("debezium/mongodb"); + admin.namespaces().createNamespace("debezium/postgresql"); + } catch (Exception e) { + log.info("[initNamespace] msg: {}", e.getMessage()); + } + log.info("[initNamespace] finish."); + } + + protected Schema getSchema(boolean jsonWithEnvelope) { + if (jsonWithEnvelope) { + return KeyValueSchema.kvBytes(); + } else { + return KeyValueSchema.of(Schema.AUTO_CONSUME(), Schema.AUTO_CONSUME(), KeyValueEncodingType.SEPARATED); + } + } + + protected void ensureSubscriptionCreated(String inputTopicName, + String subscriptionName, + Schema inputTopicSchema) + throws Exception { + // ensure the function subscription exists before we start producing messages + try (PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build()) { + try (Consumer ignored = client.newConsumer(inputTopicSchema) + .topic(inputTopicName) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName(subscriptionName) + .subscribe()) { + } + } + } + + protected Map produceMessagesToInputTopic(String inputTopicName, + int numMessages) throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + Producer producer = client.newProducer(Schema.STRING) + .topic(inputTopicName) + .create(); + + LinkedHashMap kvs = new LinkedHashMap<>(); + for (int i = 0; i < numMessages; i++) { + String key = "key-" + i; + String value = "value-" + i; + kvs.put(key, value); + producer.newMessage() + .key(key) + .value(value) + .send(); + } + return kvs; + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSinkTester.java index eff6932998f8a..3b698dc22d118 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSinkTester.java @@ -25,8 +25,9 @@ import com.rabbitmq.client.DefaultConsumer; import com.rabbitmq.client.Envelope; import lombok.Data; -import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.tests.integration.containers.RabbitMQContainer; +import org.apache.pulsar.tests.integration.io.sinks.SinkTester; +import org.apache.pulsar.tests.integration.io.sinks.SinkTester.SinkType; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import java.io.IOException; @@ -38,7 +39,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; -@Slf4j public class RabbitMQSinkTester extends SinkTester { private final String exchangeName = "test-sink-exchange"; private final String queueName = "test-sink-queue"; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSourceTester.java index 7161644e1e966..12d1bbabb9ffc 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSourceTester.java @@ -23,12 +23,13 @@ import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; import org.apache.pulsar.tests.integration.containers.RabbitMQContainer; +import org.apache.pulsar.tests.integration.io.sources.SourceTester; + +import static org.apache.pulsar.tests.integration.io.RabbitMQSinkTester.createConnectionFactory; import java.util.LinkedHashMap; import java.util.Map; -import static org.apache.pulsar.tests.integration.io.RabbitMQSinkTester.createConnectionFactory; - public class RabbitMQSourceTester extends SourceTester { private RabbitMQContainer serviceContainer; private final String exchangeName = "test-src-exchange"; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/CassandraSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/CassandraSinkTester.java similarity index 98% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/CassandraSinkTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/CassandraSinkTester.java index 72650725c7ece..97f492bc76241 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/CassandraSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/CassandraSinkTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sinks; import com.datastax.driver.core.Cluster; import com.datastax.driver.core.ResultSet; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/ElasticSearchSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearchSinkTester.java similarity index 98% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/ElasticSearchSinkTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearchSinkTester.java index 0f1cc8abfa348..5d272364a7502 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/ElasticSearchSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearchSinkTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sinks; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/HdfsSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/HdfsSinkTester.java similarity index 97% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/HdfsSinkTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/HdfsSinkTester.java index 957b93acbd965..670200cd0b6ba 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/HdfsSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/HdfsSinkTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sinks; import java.util.Map; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcPostgresSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/JdbcPostgresSinkTester.java similarity index 98% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcPostgresSinkTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/JdbcPostgresSinkTester.java index 1470cbe99d1a0..61d0e73f26a0f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/JdbcPostgresSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/JdbcPostgresSinkTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sinks; import java.sql.Connection; import java.sql.DriverManager; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/KafkaSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/KafkaSinkTester.java similarity index 97% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/KafkaSinkTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/KafkaSinkTester.java index 90771467974af..21479c2d216aa 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/KafkaSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/KafkaSinkTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sinks; import static org.apache.pulsar.tests.integration.topologies.PulsarClusterTestBase.randomName; import static org.apache.pulsar.tests.integration.topologies.PulsarTestBase.randomName; @@ -32,6 +32,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.pulsar.tests.integration.io.sinks.SinkTester.SinkType; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.testcontainers.containers.Container.ExecResult; import org.testcontainers.containers.KafkaContainer; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java new file mode 100644 index 0000000000000..7a60e2d9e48f2 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java @@ -0,0 +1,518 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.io.sinks; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.schema.AvroSchema; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.SinkStatus; +import org.apache.pulsar.tests.integration.docker.ContainerExecException; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.io.PulsarIOTestBase; +import org.apache.pulsar.tests.integration.io.RabbitMQSinkTester; +import org.apache.pulsar.tests.integration.io.RabbitMQSourceTester; +import org.apache.pulsar.tests.integration.io.sinks.JdbcPostgresSinkTester.Foo; +import org.apache.pulsar.tests.integration.io.sources.KafkaSourceTester; +import org.apache.pulsar.tests.integration.io.sources.SourceTester; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.testcontainers.containers.GenericContainer; +import org.testng.annotations.Test; +import org.testng.collections.Maps; + +import com.google.gson.Gson; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import net.jodah.failsafe.Failsafe; + +@Slf4j +public class PulsarSinksTest extends PulsarIOTestBase { + + @Test(groups = "sink") + public void testKafkaSink() throws Exception { + final String kafkaContainerName = "kafka-" + randomName(8); + testSink(new KafkaSinkTester(kafkaContainerName), true, new KafkaSourceTester(kafkaContainerName)); + } + + @Test(enabled = false, groups = "sink") + public void testCassandraSink() throws Exception { + testSink(CassandraSinkTester.createTester(true), true); + } + + @Test(enabled = false, groups = "sink") + public void testCassandraArchiveSink() throws Exception { + testSink(CassandraSinkTester.createTester(false), false); + } + + @Test(enabled = false, groups = "sink") + public void testHdfsSink() throws Exception { + testSink(new HdfsSinkTester(), false); + } + + @Test(groups = "sink") + public void testJdbcSink() throws Exception { + testSink(new JdbcPostgresSinkTester(), true); + } + + @Test(enabled = false, groups = "sink") + public void testElasticSearchSink() throws Exception { + testSink(new ElasticSearchSinkTester(), true); + } + + @Test(groups = "sink") + public void testRabbitMQSink() throws Exception { + final String containerName = "rabbitmq-" + randomName(8); + testSink(new RabbitMQSinkTester(containerName), true, new RabbitMQSourceTester(containerName)); + } + + private void testSink(SinkTester tester, boolean builtin) throws Exception { + if (pulsarCluster == null) { + super.setupCluster(); + super.setupFunctionWorkers(); + } + tester.startServiceContainer(pulsarCluster); + try { + runSinkTester(tester, builtin); + } finally { + tester.stopServiceContainer(pulsarCluster); + } + } + + private void testSink(SinkTester sinkTester, + boolean builtinSink, + SourceTester sourceTester) + throws Exception { + if (pulsarCluster == null) { + super.setupCluster(); + super.setupFunctionWorkers(); + } + ServiceContainerT serviceContainer = sinkTester.startServiceContainer(pulsarCluster); + try { + runSinkTester(sinkTester, builtinSink); + if (null != sourceTester) { + sourceTester.setServiceContainer(serviceContainer); +// testSource(sourceTester); + } + } finally { + sinkTester.stopServiceContainer(pulsarCluster); + } + } + + private void runSinkTester(SinkTester tester, boolean builtin) throws Exception { + final String tenant = TopicName.PUBLIC_TENANT; + final String namespace = TopicName.DEFAULT_NAMESPACE; + final String inputTopicName = "test-sink-connector-" + + tester.getSinkType() + "-" + functionRuntimeType + "-input-topic-" + randomName(8); + final String sinkName = "test-sink-connector-" + + tester.getSinkType().name().toLowerCase() + "-" + functionRuntimeType + "-name-" + randomName(8); + final int numMessages = 20; + + // prepare the testing environment for sink + prepareSink(tester); + + ensureSubscriptionCreated( + inputTopicName, + String.format("public/default/%s", sinkName), + tester.getInputTopicSchema()); + + // submit the sink connector + submitSinkConnector(tester, tenant, namespace, sinkName, inputTopicName); + + // get sink info + getSinkInfoSuccess(tester, tenant, namespace, sinkName, builtin); + + // get sink status + Failsafe.with(statusRetryPolicy).run(() -> getSinkStatus(tenant, namespace, sinkName)); + + // produce messages + Map kvs; + if (tester instanceof JdbcPostgresSinkTester) { + kvs = produceSchemaInsertMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcPostgresSinkTester.Foo.class)); + // wait for sink to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages)); + + // validate the sink result + tester.validateSinkResult(kvs); + + kvs = produceSchemaUpdateMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcPostgresSinkTester.Foo.class)); + + // wait for sink to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages + 20)); + + // validate the sink result + tester.validateSinkResult(kvs); + + kvs = produceSchemaDeleteMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcPostgresSinkTester.Foo.class)); + + // wait for sink to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages + 20 + 20)); + + // validate the sink result + tester.validateSinkResult(kvs); + + } else { + kvs = produceMessagesToInputTopic(inputTopicName, numMessages); + // wait for sink to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages)); + // validate the sink result + tester.validateSinkResult(kvs); + } + + // update the sink + updateSinkConnector(tester, tenant, namespace, sinkName, inputTopicName); + + // delete the sink + deleteSink(tenant, namespace, sinkName); + + // get sink info (sink should be deleted) + getSinkInfoNotFound(tenant, namespace, sinkName); + } + + protected void prepareSink(SinkTester tester) throws Exception { + tester.prepareSink(); + } + + protected void submitSinkConnector(SinkTester tester, + String tenant, + String namespace, + String sinkName, + String inputTopicName) throws Exception { + String[] commands; + if (tester.getSinkType() != SinkTester.SinkType.UNDEFINED) { + commands = new String[] { + PulsarCluster.ADMIN_SCRIPT, + "sink", "create", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName, + "--sink-type", tester.sinkType().getValue().toLowerCase(), + "--sinkConfig", new Gson().toJson(tester.sinkConfig()), + "--inputs", inputTopicName + }; + } else { + commands = new String[] { + PulsarCluster.ADMIN_SCRIPT, + "sink", "create", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName, + "--archive", tester.getSinkArchive(), + "--classname", tester.getSinkClassName(), + "--sinkConfig", new Gson().toJson(tester.sinkConfig()), + "--inputs", inputTopicName + }; + } + log.info("Run command : {}", StringUtils.join(commands, ' ')); + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + assertTrue( + result.getStdout().contains("\"Created successfully\""), + result.getStdout()); + } + + protected void updateSinkConnector(SinkTester tester, + String tenant, + String namespace, + String sinkName, + String inputTopicName) throws Exception { + String[] commands; + if (tester.getSinkType() != SinkTester.SinkType.UNDEFINED) { + commands = new String[] { + PulsarCluster.ADMIN_SCRIPT, + "sink", "update", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName, + "--sink-type", tester.sinkType().getValue().toLowerCase(), + "--sinkConfig", new Gson().toJson(tester.sinkConfig()), + "--inputs", inputTopicName, + "--parallelism", "2" + }; + } else { + commands = new String[] { + PulsarCluster.ADMIN_SCRIPT, + "sink", "create", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName, + "--archive", tester.getSinkArchive(), + "--classname", tester.getSinkClassName(), + "--sinkConfig", new Gson().toJson(tester.sinkConfig()), + "--inputs", inputTopicName, + "--parallelism", "2" + }; + } + log.info("Run command : {}", StringUtils.join(commands, ' ')); + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + assertTrue( + result.getStdout().contains("\"Updated successfully\""), + result.getStdout()); + } + + protected void getSinkInfoSuccess(SinkTester tester, + String tenant, + String namespace, + String sinkName, + boolean builtin) throws Exception { + String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "sink", + "get", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName + }; + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + log.info("Get sink info : {}", result.getStdout()); + if (builtin) { + assertTrue( + result.getStdout().contains("\"archive\": \"builtin://" + tester.getSinkType().getValue().toLowerCase() + "\""), + result.getStdout() + ); + } else { + assertTrue( + result.getStdout().contains("\"className\": \"" + tester.getSinkClassName() + "\""), + result.getStdout() + ); + } + } + + protected void getSinkStatus(String tenant, String namespace, String sinkName) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "sink", + "status", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName + }; + + final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + log.info("Get sink status : {}", result.getStdout()); + + assertEquals(result.getExitCode(), 0); + + SinkStatus sinkStatus = SinkStatus.decode(result.getStdout()); + + assertEquals(sinkStatus.getNumInstances(), 1); + assertEquals(sinkStatus.getNumRunning(), 1); + assertEquals(sinkStatus.getInstances().size(), 1); + assertEquals(sinkStatus.getInstances().get(0).getInstanceId(), 0); + assertEquals(sinkStatus.getInstances().get(0).getStatus().isRunning(), true); + assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); + assertEquals(sinkStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); + } + + protected void deleteSink(String tenant, String namespace, String sinkName) throws Exception { + + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "sink", + "delete", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName + }; + + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + assertTrue( + result.getStdout().contains("Deleted successfully"), + result.getStdout() + ); + result.assertNoStderr(); + } + + protected void getSinkInfoNotFound(String tenant, String namespace, String sinkName) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "sink", + "get", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName + }; + try { + pulsarCluster.getAnyWorker().execCmd(commands); + fail("Command should have exited with non-zero"); + } catch (ContainerExecException e) { + assertTrue(e.getResult().getStderr().contains("Reason: Sink " + sinkName + " doesn't exist")); + } + } + + + protected void waitForProcessingSinkMessages(String tenant, + String namespace, + String sinkName, + int numMessages) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "sink", + "status", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName + }; + + final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + log.info("Get sink status : {}", result.getStdout()); + + assertEquals(result.getExitCode(), 0); + + final SinkStatus sinkStatus = SinkStatus.decode(result.getStdout()); + + assertEquals(sinkStatus.getNumInstances(), 1); + assertEquals(sinkStatus.getNumRunning(), 1); + assertEquals(sinkStatus.getInstances().size(), 1); + assertEquals(sinkStatus.getInstances().get(0).getInstanceId(), 0); + assertEquals(sinkStatus.getInstances().get(0).getStatus().isRunning(), true); + assertTrue(sinkStatus.getInstances().get(0).getStatus().getLastReceivedTime() > 0); + assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumReadFromPulsar(), numMessages); + assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumWrittenToSink(), numMessages); + assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); + assertEquals(sinkStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); + } + + // This for JdbcPostgresSinkTester + protected Map produceSchemaInsertMessagesToInputTopic(String inputTopicName, + int numMessages, + Schema schema) throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + Producer producer = client.newProducer(schema) + .topic(inputTopicName) + .create(); + + LinkedHashMap kvs = new LinkedHashMap<>(); + for (int i = 0; i < numMessages; i++) { + String key = "key-" + i; + + JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo(); + obj.setField1("field1_insert_" + i); + obj.setField2("field2_insert_" + i); + obj.setField3(i); + String value = new String(schema.encode(obj)); + Map properties = Maps.newHashMap(); + properties.put("ACTION", "INSERT"); + + kvs.put(key, value); + kvs.put("ACTION", "INSERT"); + producer.newMessage() + .properties(properties) + .key(key) + .value(obj) + .send(); + } + return kvs; + } + + // This for JdbcPostgresSinkTester + protected Map produceSchemaUpdateMessagesToInputTopic(String inputTopicName, + int numMessages, + Schema schema) throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + Producer producer = client.newProducer(schema) + .topic(inputTopicName) + .create(); + + LinkedHashMap kvs = new LinkedHashMap<>(); + log.info("update start"); + for (int i = 0; i < numMessages; i++) { + String key = "key-" + i; + + JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo(); + obj.setField1("field1_insert_" + i); + obj.setField2("field2_update_" + i); + obj.setField3(i); + String value = new String(schema.encode(obj)); + Map properties = Maps.newHashMap(); + properties.put("ACTION", "UPDATE"); + + kvs.put(key, value); + kvs.put("ACTION", "UPDATE"); + producer.newMessage() + .properties(properties) + .key(key) + .value(obj) + .send(); + } + log.info("update end"); + return kvs; + } + + // This for JdbcPostgresSinkTester + protected Map produceSchemaDeleteMessagesToInputTopic(String inputTopicName, + int numMessages, + Schema schema) throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + Producer producer = client.newProducer(schema) + .topic(inputTopicName) + .create(); + + LinkedHashMap kvs = new LinkedHashMap<>(); + for (int i = 0; i < numMessages; i++) { + String key = "key-" + i; + + JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo(); + obj.setField1("field1_insert_" + i); + obj.setField2("field2_update_" + i); + obj.setField3(i); + String value = new String(schema.encode(obj)); + Map properties = Maps.newHashMap(); + properties.put("ACTION", "DELETE"); + + kvs.put(key, value); + kvs.put("ACTION", "DELETE"); + producer.newMessage() + .properties(properties) + .key(key) + .value(obj) + .send(); + } + return kvs; + } + + +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/SinkTester.java similarity index 98% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/SinkTester.java index 5426db94effd6..2224799553c29 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/SinkTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sinks; import java.util.Map; import lombok.Getter; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/AvroKafkaSourceTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/AvroKafkaSourceTest.java similarity index 99% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/AvroKafkaSourceTest.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/AvroKafkaSourceTest.java index b8b962dad9fdb..52cade8500f23 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/AvroKafkaSourceTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/AvroKafkaSourceTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sources; import com.google.gson.Gson; import lombok.Cleanup; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMongoDbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumMongoDbSourceTester.java similarity index 98% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMongoDbSourceTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumMongoDbSourceTester.java index 6fa35ef918376..41f2b26eeab22 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMongoDbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumMongoDbSourceTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sources; import lombok.Getter; import lombok.extern.slf4j.Slf4j; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMySqlSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumMySqlSourceTester.java similarity index 98% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMySqlSourceTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumMySqlSourceTester.java index 4b665065e0e01..dc982862860eb 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumMySqlSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumMySqlSourceTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sources; import java.io.Closeable; import java.util.Map; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumPostgreSqlSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumPostgreSqlSourceTester.java similarity index 98% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumPostgreSqlSourceTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumPostgreSqlSourceTester.java index a8deb4ccc075d..7a876bdba7aa4 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/DebeziumPostgreSqlSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumPostgreSqlSourceTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sources; import lombok.Getter; import lombok.extern.slf4j.Slf4j; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/GenericRecordSourceTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/GenericRecordSourceTest.java similarity index 99% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/GenericRecordSourceTest.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/GenericRecordSourceTest.java index 1b591ec8e2ff4..e0cc328f42718 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/GenericRecordSourceTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/GenericRecordSourceTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sources; import static org.apache.pulsar.tests.integration.functions.utils.CommandGenerator.JAVAJAR; import static org.apache.pulsar.tests.integration.suites.PulsarTestSuite.retryStrategically; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/KafkaSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/KafkaSourceTester.java similarity index 98% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/KafkaSourceTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/KafkaSourceTester.java index 759af05ad898a..238fe7000b4be 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/KafkaSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/KafkaSourceTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sources; import static org.apache.pulsar.tests.integration.topologies.PulsarClusterTestBase.randomName; import static org.testng.Assert.assertTrue; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarSourcePropertyTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarSourcePropertyTest.java similarity index 99% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarSourcePropertyTest.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarSourcePropertyTest.java index 41c15b6887251..19da91ae86fe5 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarSourcePropertyTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarSourcePropertyTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sources; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarSourcesTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarSourcesTest.java new file mode 100644 index 0000000000000..6d8cf83e0e447 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarSourcesTest.java @@ -0,0 +1,588 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.io.sources; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.SourceStatus; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.tests.integration.containers.DebeziumMongoDbContainer; +import org.apache.pulsar.tests.integration.containers.DebeziumMySQLContainer; +import org.apache.pulsar.tests.integration.containers.DebeziumPostgreSqlContainer; +import org.apache.pulsar.tests.integration.docker.ContainerExecException; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.io.PulsarIOTestBase; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.testcontainers.containers.GenericContainer; +import org.testng.annotations.Test; + +import com.google.gson.Gson; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import net.jodah.failsafe.Failsafe; + +@Slf4j +public class PulsarSourcesTest extends PulsarIOTestBase { + + @Test(groups = "source") + public void testDebeziumMySqlSourceJson() throws Exception { + testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter", true); + } + + @Test(groups = "source") + public void testDebeziumMySqlSourceAvro() throws Exception { + testDebeziumMySqlConnect( + "org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter", false); + } + + @Test(groups = "source") + public void testDebeziumPostgreSqlSource() throws Exception { + testDebeziumPostgreSqlConnect("org.apache.kafka.connect.json.JsonConverter", true); + } + + @Test(groups = "source") + public void testDebeziumMongoDbSource() throws Exception{ + testDebeziumMongoDbConnect("org.apache.kafka.connect.json.JsonConverter", true); + } + + private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { + + final String tenant = TopicName.PUBLIC_TENANT; + final String namespace = TopicName.DEFAULT_NAMESPACE; + final String outputTopicName = "debe-output-topic-name-" + testId.getAndIncrement(); + boolean isJsonConverter = converterClassName.endsWith("JsonConverter"); + final String consumeTopicName = "debezium/mysql-" + + (isJsonConverter ? "json" : "avro") + + "/dbserver1.inventory.products"; + final String sourceName = "test-source-debezium-mysql" + (isJsonConverter ? "json" : "avro") + + "-" + functionRuntimeType + "-" + randomName(8); + + // This is the binlog count that contained in mysql container. + final int numMessages = 47; + + if (pulsarCluster == null) { + super.setupCluster(); + super.setupFunctionWorkers(); + } + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); + initNamespace(admin); + + try { + SchemaInfo lastSchemaInfo = admin.schemas().getSchemaInfo(consumeTopicName); + log.info("lastSchemaInfo: {}", lastSchemaInfo == null ? "null" : lastSchemaInfo.toString()); + } catch (Exception e) { + log.warn("failed to get schemaInfo for topic: {}, exceptions message: {}", + consumeTopicName, e.getMessage()); + } + + admin.topics().createNonPartitionedTopic(outputTopicName); + + @Cleanup + DebeziumMySqlSourceTester sourceTester = new DebeziumMySqlSourceTester(pulsarCluster, converterClassName); + sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); + + // setup debezium mysql server + DebeziumMySQLContainer mySQLContainer = new DebeziumMySQLContainer(pulsarCluster.getClusterName()); + sourceTester.setServiceContainer(mySQLContainer); + + // prepare the testing environment for source + prepareSource(sourceTester); + + // submit the source connector + submitSourceConnector(sourceTester, tenant, namespace, sourceName, outputTopicName); + + // get source info + getSourceInfoSuccess(sourceTester, tenant, namespace, sourceName); + + // get source status + Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); + + // wait for source to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); + + @Cleanup + Consumer consumer = client.newConsumer(getSchema(jsonWithEnvelope)) + .topic(consumeTopicName) + .subscriptionName("debezium-source-tester") + .subscriptionType(SubscriptionType.Exclusive) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + log.info("[debezium mysql test] create consumer finish. converterName: {}", converterClassName); + + // validate the source result + sourceTester.validateSourceResult(consumer, 9, null, converterClassName); + + // prepare insert event + sourceTester.prepareInsertEvent(); + + // validate the source insert event + sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT, converterClassName); + + // prepare update event + sourceTester.prepareUpdateEvent(); + + // validate the source update event + sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE, converterClassName); + + // prepare delete event + sourceTester.prepareDeleteEvent(); + + // validate the source delete event + sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE, converterClassName); + + // delete the source + deleteSource(tenant, namespace, sourceName); + + // get source info (source should be deleted) + getSourceInfoNotFound(tenant, namespace, sourceName); + } + + private void testDebeziumPostgreSqlConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { + + final String tenant = TopicName.PUBLIC_TENANT; + final String namespace = TopicName.DEFAULT_NAMESPACE; + final String outputTopicName = "debe-output-topic-name-" + testId.getAndIncrement(); + final String consumeTopicName = "debezium/postgresql/dbserver1.inventory.products"; + final String sourceName = "test-source-debezium-postgersql-" + functionRuntimeType + "-" + randomName(8); + + + // This is the binlog count that contained in postgresql container. + final int numMessages = 26; + + if (pulsarCluster == null) { + super.setupCluster(); + super.setupFunctionWorkers(); + } + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); + initNamespace(admin); + + admin.topics().createNonPartitionedTopic(consumeTopicName); + admin.topics().createNonPartitionedTopic(outputTopicName); + + @Cleanup + Consumer consumer = client.newConsumer(getSchema(jsonWithEnvelope)) + .topic(consumeTopicName) + .subscriptionName("debezium-source-tester") + .subscriptionType(SubscriptionType.Exclusive) + .subscribe(); + + @Cleanup + DebeziumPostgreSqlSourceTester sourceTester = new DebeziumPostgreSqlSourceTester(pulsarCluster); + sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); + + // setup debezium postgresql server + DebeziumPostgreSqlContainer postgreSqlContainer = new DebeziumPostgreSqlContainer(pulsarCluster.getClusterName()); + sourceTester.setServiceContainer(postgreSqlContainer); + + // prepare the testing environment for source + prepareSource(sourceTester); + + // submit the source connector + submitSourceConnector(sourceTester, tenant, namespace, sourceName, outputTopicName); + + // get source info + getSourceInfoSuccess(sourceTester, tenant, namespace, sourceName); + + // get source status + Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); + + // wait for source to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); + + // validate the source result + sourceTester.validateSourceResult(consumer, 9, null, converterClassName); + + // prepare insert event + sourceTester.prepareInsertEvent(); + + // validate the source insert event + sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT, converterClassName); + + // prepare update event + sourceTester.prepareUpdateEvent(); + + // validate the source update event + sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE, converterClassName); + + // prepare delete event + sourceTester.prepareDeleteEvent(); + + // validate the source delete event + sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE, converterClassName); + + // delete the source + deleteSource(tenant, namespace, sourceName); + + // get source info (source should be deleted) + getSourceInfoNotFound(tenant, namespace, sourceName); + } + + private void testDebeziumMongoDbConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { + + final String tenant = TopicName.PUBLIC_TENANT; + final String namespace = TopicName.DEFAULT_NAMESPACE; + final String outputTopicName = "debe-output-topic-name"; + final String consumeTopicName = "debezium/mongodb/dbserver1.inventory.products"; + final String sourceName = "test-source-connector-" + + functionRuntimeType + "-name-" + randomName(8); + + // This is the binlog count that contained in mongodb container. + final int numMessages = 17; + + if (pulsarCluster == null) { + super.setupCluster(); + super.setupFunctionWorkers(); + } + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); + initNamespace(admin); + + admin.topics().createNonPartitionedTopic(consumeTopicName); + admin.topics().createNonPartitionedTopic(outputTopicName); + + @Cleanup + Consumer consumer = client.newConsumer(getSchema(jsonWithEnvelope)) + .topic(consumeTopicName) + .subscriptionName("debezium-source-tester") + .subscriptionType(SubscriptionType.Exclusive) + .subscribe(); + + @Cleanup + DebeziumMongoDbSourceTester sourceTester = new DebeziumMongoDbSourceTester(pulsarCluster); + sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); + + // setup debezium mongodb server + DebeziumMongoDbContainer mongoDbContainer = new DebeziumMongoDbContainer(pulsarCluster.getClusterName()); + sourceTester.setServiceContainer(mongoDbContainer); + // prepare the testing environment for source + prepareSource(sourceTester); + + // submit the source connector + submitSourceConnector(sourceTester, tenant, namespace, sourceName, outputTopicName); + + // get source info + getSourceInfoSuccess(sourceTester, tenant, namespace, sourceName); + + // get source status + Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); + + // wait for source to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); + + // validate the source result + sourceTester.validateSourceResult(consumer, 9, null, converterClassName); + + // prepare insert event + sourceTester.prepareInsertEvent(); + + // validate the source insert event + sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT, converterClassName); + + // prepare update event + sourceTester.prepareUpdateEvent(); + + // validate the source update event + sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE, converterClassName); + + // prepare delete event + sourceTester.prepareDeleteEvent(); + + // validate the source delete event + sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE, converterClassName); + + // delete the source + deleteSource(tenant, namespace, sourceName); + + // get source info (source should be deleted) + getSourceInfoNotFound(tenant, namespace, sourceName); + } + + private void testSource(SourceTester tester) throws Exception { + final String tenant = TopicName.PUBLIC_TENANT; + final String namespace = TopicName.DEFAULT_NAMESPACE; + final String outputTopicName = "test-source-connector-" + + functionRuntimeType + "-output-topic-" + randomName(8); + final String sourceName = "test-source-connector-" + + functionRuntimeType + "-name-" + randomName(8); + final int numMessages = 20; + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); + admin.topics().createNonPartitionedTopic(outputTopicName); + + @Cleanup + Consumer consumer = client.newConsumer(Schema.STRING) + .topic(outputTopicName) + .subscriptionName("source-tester") + .subscriptionType(SubscriptionType.Exclusive) + .subscribe(); + + // prepare the testing environment for source + prepareSource(tester); + + // submit the source connector + submitSourceConnector(tester, tenant, namespace, sourceName, outputTopicName); + + // get source info + getSourceInfoSuccess(tester, tenant, namespace, sourceName); + + // get source status + Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); + + // produce messages + Map kvs = tester.produceSourceMessages(numMessages); + + // wait for source to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); + + // validate the source result + validateSourceResult(consumer, kvs); + + // update the source connector + updateSourceConnector(tester, tenant, namespace, sourceName, outputTopicName); + + // delete the source + deleteSource(tenant, namespace, sourceName); + + // get source info (source should be deleted) + getSourceInfoNotFound(tenant, namespace, sourceName); + } + + protected void prepareSource(SourceTester tester) throws Exception { + tester.prepareSource(); + } + + protected void submitSourceConnector(SourceTester tester, + String tenant, + String namespace, + String sourceName, + String outputTopicName) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", "create", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName, + "--source-type", tester.sourceType(), + "--sourceConfig", new Gson().toJson(tester.sourceConfig()), + "--destinationTopicName", outputTopicName + }; + + log.info("Run command : {}", StringUtils.join(commands, ' ')); + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + assertTrue( + result.getStdout().contains("\"Created successfully\""), + result.getStdout()); + } + + protected void updateSourceConnector(SourceTester tester, + String tenant, + String namespace, + String sourceName, + String outputTopicName) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", "update", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName, + "--source-type", tester.sourceType(), + "--sourceConfig", new Gson().toJson(tester.sourceConfig()), + "--destinationTopicName", outputTopicName, + "--parallelism", "2" + }; + + log.info("Run command : {}", StringUtils.join(commands, ' ')); + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + assertTrue( + result.getStdout().contains("\"Updated successfully\""), + result.getStdout()); + } + + protected void getSourceInfoSuccess(SourceTester tester, + String tenant, + String namespace, + String sourceName) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", + "get", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName + }; + + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + log.info("Get source info : {}", result.getStdout()); + assertTrue( + result.getStdout().contains("\"archive\": \"builtin://" + tester.getSourceType() + "\""), + result.getStdout() + ); + } + + protected void getSourceStatus(String tenant, String namespace, String sourceName) throws Exception { + + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", + "status", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName + }; + + final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + log.info("Get source status : {}", result.getStdout()); + + assertEquals(result.getExitCode(), 0); + + final SourceStatus sourceStatus = SourceStatus.decode(result.getStdout()); + + assertEquals(sourceStatus.getNumInstances(), 1); + assertEquals(sourceStatus.getNumRunning(), 1); + assertEquals(sourceStatus.getInstances().size(), 1); + assertEquals(sourceStatus.getInstances().get(0).getStatus().isRunning(), true); + assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); + assertEquals(sourceStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); + + assertTrue(result.getStdout().contains("\"running\" : true")); + + } + + protected void validateSourceResult(Consumer consumer, + Map kvs) throws Exception { + for (Map.Entry kv : kvs.entrySet()) { + Message msg = consumer.receive(); + assertEquals(kv.getKey(), msg.getKey()); + assertEquals(kv.getValue(), msg.getValue()); + } + } + + protected void waitForProcessingSourceMessages(String tenant, + String namespace, + String sourceName, + int numMessages) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", + "status", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName + }; + + final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + log.info("Get source status : {}", result.getStdout()); + + assertEquals(result.getExitCode(), 0); + + SourceStatus sourceStatus = SourceStatus.decode(result.getStdout()); + assertEquals(sourceStatus.getNumInstances(), 1); + assertEquals(sourceStatus.getNumRunning(), 1); + assertEquals(sourceStatus.getInstances().size(), 1); + assertEquals(sourceStatus.getInstances().get(0).getInstanceId(), 0); + assertEquals(sourceStatus.getInstances().get(0).getStatus().isRunning(), true); + assertTrue(sourceStatus.getInstances().get(0).getStatus().getLastReceivedTime() > 0); + assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumReceivedFromSource(), numMessages); + assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumWritten(), numMessages); + assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); + assertEquals(sourceStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); + } + + protected void deleteSource(String tenant, String namespace, String sourceName) throws Exception { + + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", + "delete", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName + }; + + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + assertTrue( + result.getStdout().contains("Delete source successfully"), + result.getStdout() + ); + result.assertNoStderr(); + } + + protected void getSourceInfoNotFound(String tenant, String namespace, String sourceName) throws Exception { + + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", + "get", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName + }; + + try { + pulsarCluster.getAnyWorker().execCmd(commands); + fail("Command should have exited with non-zero"); + } catch (ContainerExecException e) { + assertTrue(e.getResult().getStderr().contains("Reason: Source " + sourceName + " doesn't exist")); + } + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/SourceTester.java similarity index 99% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SourceTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/SourceTester.java index 27037ab302082..8019c10d229d0 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/SourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/SourceTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sources; import java.util.HashSet; import java.util.Map; diff --git a/tests/integration/src/test/resources/pulsar-process.xml b/tests/integration/src/test/resources/pulsar-process.xml index c3db70c48d65d..3043ecb3553f5 100644 --- a/tests/integration/src/test/resources/pulsar-process.xml +++ b/tests/integration/src/test/resources/pulsar-process.xml @@ -22,7 +22,9 @@ - + + + diff --git a/tests/integration/src/test/resources/pulsar-thread.xml b/tests/integration/src/test/resources/pulsar-thread.xml index 6f520dfbbf847..db5b1e935fc18 100644 --- a/tests/integration/src/test/resources/pulsar-thread.xml +++ b/tests/integration/src/test/resources/pulsar-thread.xml @@ -22,7 +22,9 @@ - + + + \ No newline at end of file diff --git a/tests/integration/src/test/resources/pulsar.xml b/tests/integration/src/test/resources/pulsar.xml index ac7de1f315c1e..e22a9c9f75f68 100644 --- a/tests/integration/src/test/resources/pulsar.xml +++ b/tests/integration/src/test/resources/pulsar.xml @@ -23,6 +23,9 @@ see {@link https://github.com/cbeust/testng/issues/508} --> + + + From c68010d18b8e0f6ba1dac2bc8ce665ffd33401a6 Mon Sep 17 00:00:00 2001 From: David Kjerrumgaard Date: Mon, 5 Apr 2021 14:49:24 -0700 Subject: [PATCH 2/6] Fixed test xml files --- tests/integration/src/test/resources/pulsar-function.xml | 4 ++-- tests/integration/src/test/resources/pulsar-messaging.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/src/test/resources/pulsar-function.xml b/tests/integration/src/test/resources/pulsar-function.xml index 71292a59c1f82..a18a65ff2e16d 100644 --- a/tests/integration/src/test/resources/pulsar-function.xml +++ b/tests/integration/src/test/resources/pulsar-function.xml @@ -23,9 +23,9 @@ - - + + diff --git a/tests/integration/src/test/resources/pulsar-messaging.xml b/tests/integration/src/test/resources/pulsar-messaging.xml index e0ac367e6e19b..aa31852fd6f62 100644 --- a/tests/integration/src/test/resources/pulsar-messaging.xml +++ b/tests/integration/src/test/resources/pulsar-messaging.xml @@ -25,8 +25,8 @@ - + \ No newline at end of file From 5d22c217da0971aee1b189b22acda578243b20f7 Mon Sep 17 00:00:00 2001 From: David Kjerrumgaard Date: Mon, 5 Apr 2021 16:09:59 -0700 Subject: [PATCH 3/6] Added Pulsar IO test suite --- .gitignore | 1 + .../src/test/resources/pulsar-io-sinks.xml | 28 ++++++++++++++++++ .../src/test/resources/pulsar-io-sources.xml | 28 ++++++++++++++++++ .../src/test/resources/pulsar-io-suite.xml | 29 +++++++++++++++++++ .../integration/src/test/resources/pulsar.xml | 6 ++-- 5 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 tests/integration/src/test/resources/pulsar-io-sinks.xml create mode 100644 tests/integration/src/test/resources/pulsar-io-sources.xml create mode 100644 tests/integration/src/test/resources/pulsar-io-suite.xml diff --git a/.gitignore b/.gitignore index 176d1d243af37..cfaf6c8ab075b 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ nb-configuration.xml # Mac **/.DS_Store +.java-version # VisualStudioCode artifacts .vscode/ diff --git a/tests/integration/src/test/resources/pulsar-io-sinks.xml b/tests/integration/src/test/resources/pulsar-io-sinks.xml new file mode 100644 index 0000000000000..614c371f3e52c --- /dev/null +++ b/tests/integration/src/test/resources/pulsar-io-sinks.xml @@ -0,0 +1,28 @@ + + + + + + + + + \ No newline at end of file diff --git a/tests/integration/src/test/resources/pulsar-io-sources.xml b/tests/integration/src/test/resources/pulsar-io-sources.xml new file mode 100644 index 0000000000000..4b034febc72b3 --- /dev/null +++ b/tests/integration/src/test/resources/pulsar-io-sources.xml @@ -0,0 +1,28 @@ + + + + + + + + + \ No newline at end of file diff --git a/tests/integration/src/test/resources/pulsar-io-suite.xml b/tests/integration/src/test/resources/pulsar-io-suite.xml new file mode 100644 index 0000000000000..33b681797649d --- /dev/null +++ b/tests/integration/src/test/resources/pulsar-io-suite.xml @@ -0,0 +1,29 @@ + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar.xml b/tests/integration/src/test/resources/pulsar.xml index e22a9c9f75f68..a2011e4c82144 100644 --- a/tests/integration/src/test/resources/pulsar.xml +++ b/tests/integration/src/test/resources/pulsar.xml @@ -23,9 +23,6 @@ see {@link https://github.com/cbeust/testng/issues/508} --> - - - + + From b581232033b19d4c71917676fff8f485aca9e23e Mon Sep 17 00:00:00 2001 From: David Kjerrumgaard Date: Tue, 6 Apr 2021 09:59:44 -0700 Subject: [PATCH 4/6] Moved RabbitMQ tests --- .../pulsar/tests/integration/io/sinks/PulsarSinksTest.java | 3 +-- .../tests/integration/io/{ => sinks}/RabbitMQSinkTester.java | 3 +-- .../integration/io/{ => sources}/RabbitMQSourceTester.java | 5 ++--- 3 files changed, 4 insertions(+), 7 deletions(-) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sinks}/RabbitMQSinkTester.java (97%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{ => sources}/RabbitMQSourceTester.java (93%) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java index 7a60e2d9e48f2..c133a0fe1a877 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java @@ -35,10 +35,9 @@ import org.apache.pulsar.tests.integration.docker.ContainerExecException; import org.apache.pulsar.tests.integration.docker.ContainerExecResult; import org.apache.pulsar.tests.integration.io.PulsarIOTestBase; -import org.apache.pulsar.tests.integration.io.RabbitMQSinkTester; -import org.apache.pulsar.tests.integration.io.RabbitMQSourceTester; import org.apache.pulsar.tests.integration.io.sinks.JdbcPostgresSinkTester.Foo; import org.apache.pulsar.tests.integration.io.sources.KafkaSourceTester; +import org.apache.pulsar.tests.integration.io.sources.RabbitMQSourceTester; import org.apache.pulsar.tests.integration.io.sources.SourceTester; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.testcontainers.containers.GenericContainer; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/RabbitMQSinkTester.java similarity index 97% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSinkTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/RabbitMQSinkTester.java index 3b698dc22d118..b3978bcd313ee 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/RabbitMQSinkTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sinks; import com.rabbitmq.client.AMQP; import com.rabbitmq.client.Channel; @@ -26,7 +26,6 @@ import com.rabbitmq.client.Envelope; import lombok.Data; import org.apache.pulsar.tests.integration.containers.RabbitMQContainer; -import org.apache.pulsar.tests.integration.io.sinks.SinkTester; import org.apache.pulsar.tests.integration.io.sinks.SinkTester.SinkType; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/RabbitMQSourceTester.java similarity index 93% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSourceTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/RabbitMQSourceTester.java index 12d1bbabb9ffc..b2adb023bbe29 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/RabbitMQSourceTester.java @@ -16,16 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io; +package org.apache.pulsar.tests.integration.io.sources; import com.rabbitmq.client.BuiltinExchangeType; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; import org.apache.pulsar.tests.integration.containers.RabbitMQContainer; -import org.apache.pulsar.tests.integration.io.sources.SourceTester; -import static org.apache.pulsar.tests.integration.io.RabbitMQSinkTester.createConnectionFactory; +import static org.apache.pulsar.tests.integration.io.sinks.RabbitMQSinkTester.createConnectionFactory; import java.util.LinkedHashMap; import java.util.Map; From 4d872cbd9bd13237137791a5edd6751902ba0f0e Mon Sep 17 00:00:00 2001 From: David Kjerrumgaard Date: Tue, 6 Apr 2021 12:36:46 -0700 Subject: [PATCH 5/6] Added PulsarIOTestRunner classes --- .../integration/io/PulsarIOTestBase.java | 128 ++-- .../integration/io/PulsarIOTestRunner.java | 109 ++++ .../io/{sinks => }/RabbitMQSinkTester.java | 3 +- .../{sources => }/RabbitMQSourceTester.java | 5 +- .../io/sinks/PulsarIOSinkRunner.java | 447 +++++++++++++ .../integration/io/sinks/PulsarSinksTest.java | 453 +------------- .../io/sources/PulsarIOSourceRunner.java | 296 +++++++++ .../io/sources/PulsarSourcesTest.java | 588 ------------------ .../DebeziumMongoDbSourceTester.java | 3 +- .../DebeziumMySqlSourceTester.java | 3 +- .../DebeziumPostgreSqlSourceTester.java | 3 +- .../debezium/PulsarDebeziumSourcesTest.java | 220 +++++++ .../PulsarIODebeziumSourceRunner.java | 117 ++++ .../src/test/resources/pulsar-io-sources.xml | 2 +- 14 files changed, 1242 insertions(+), 1135 deletions(-) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestRunner.java rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{sinks => }/RabbitMQSinkTester.java (97%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/{sources => }/RabbitMQSourceTester.java (93%) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarIOSinkRunner.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarIOSourceRunner.java delete mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarSourcesTest.java rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/{ => debezium}/DebeziumMongoDbSourceTester.java (97%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/{ => debezium}/DebeziumMySqlSourceTester.java (97%) rename tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/{ => debezium}/DebeziumPostgreSqlSourceTester.java (97%) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestBase.java index 5adbcaf53cbc4..334c7b4b38f40 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestBase.java @@ -18,102 +18,52 @@ */ package org.apache.pulsar.tests.integration.io; -import java.time.Duration; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.api.Consumer; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.impl.schema.KeyValueSchema; -import org.apache.pulsar.common.policies.data.TenantInfo; -import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.tests.integration.functions.PulsarFunctionsTestBase; -import org.testcontainers.shaded.com.google.common.collect.Sets; - -import lombok.Cleanup; -import lombok.extern.slf4j.Slf4j; -import net.jodah.failsafe.RetryPolicy; +import org.apache.pulsar.tests.integration.io.sinks.PulsarIOSinkRunner; +import org.apache.pulsar.tests.integration.io.sinks.SinkTester; +import org.apache.pulsar.tests.integration.io.sources.PulsarIOSourceRunner; +import org.apache.pulsar.tests.integration.io.sources.SourceTester; +import org.testcontainers.containers.GenericContainer; -@Slf4j public abstract class PulsarIOTestBase extends PulsarFunctionsTestBase { - final Duration ONE_MINUTE = Duration.ofMinutes(1); - final Duration TEN_SECONDS = Duration.ofSeconds(10); - - protected final RetryPolicy statusRetryPolicy = new RetryPolicy() - .withMaxDuration(ONE_MINUTE) - .withDelay(TEN_SECONDS) - .onRetry(e -> log.error("Retry ... ")); - - protected final AtomicInteger testId = new AtomicInteger(0); - - protected void initNamespace(PulsarAdmin admin) { - log.info("[initNamespace] start."); - try { - admin.tenants().createTenant("debezium", new TenantInfo(Sets.newHashSet(), - Sets.newHashSet(pulsarCluster.getClusterName()))); - admin.namespaces().createNamespace("debezium/mysql-json"); - admin.namespaces().createNamespace("debezium/mysql-avro"); - admin.namespaces().createNamespace("debezium/mongodb"); - admin.namespaces().createNamespace("debezium/postgresql"); - } catch (Exception e) { - log.info("[initNamespace] msg: {}", e.getMessage()); - } - log.info("[initNamespace] finish."); - } - - protected Schema getSchema(boolean jsonWithEnvelope) { - if (jsonWithEnvelope) { - return KeyValueSchema.kvBytes(); - } else { - return KeyValueSchema.of(Schema.AUTO_CONSUME(), Schema.AUTO_CONSUME(), KeyValueEncodingType.SEPARATED); + @SuppressWarnings({ "unchecked", "rawtypes" }) + protected void testSink(SinkTester tester, boolean builtin) throws Exception { + if (pulsarCluster == null) { + super.setupCluster(); + super.setupFunctionWorkers(); } - } - - protected void ensureSubscriptionCreated(String inputTopicName, - String subscriptionName, - Schema inputTopicSchema) - throws Exception { - // ensure the function subscription exists before we start producing messages - try (PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build()) { - try (Consumer ignored = client.newConsumer(inputTopicSchema) - .topic(inputTopicName) - .subscriptionType(SubscriptionType.Shared) - .subscriptionName(subscriptionName) - .subscribe()) { - } + tester.startServiceContainer(pulsarCluster); + try { + PulsarIOSinkRunner runner = new PulsarIOSinkRunner(pulsarCluster, functionRuntimeType.toString()); + runner.runSinkTester(tester, builtin); + } finally { + tester.stopServiceContainer(pulsarCluster); } } - - protected Map produceMessagesToInputTopic(String inputTopicName, - int numMessages) throws Exception { - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); + + @SuppressWarnings("rawtypes") + protected void testSink(SinkTester sinkTester, + boolean builtinSink, + SourceTester sourceTester) throws Exception { - @Cleanup - Producer producer = client.newProducer(Schema.STRING) - .topic(inputTopicName) - .create(); + if (pulsarCluster == null) { + super.setupCluster(); + super.setupFunctionWorkers(); + } - LinkedHashMap kvs = new LinkedHashMap<>(); - for (int i = 0; i < numMessages; i++) { - String key = "key-" + i; - String value = "value-" + i; - kvs.put(key, value); - producer.newMessage() - .key(key) - .value(value) - .send(); - } - return kvs; - } + ServiceContainerT serviceContainer = sinkTester.startServiceContainer(pulsarCluster); + + try { + PulsarIOSinkRunner runner = new PulsarIOSinkRunner(pulsarCluster, functionRuntimeType.toString()); + runner.runSinkTester(sinkTester, builtinSink); + if (null != sourceTester) { + PulsarIOSourceRunner sourceRunner = new PulsarIOSourceRunner(pulsarCluster, functionRuntimeType.toString()); + sourceTester.setServiceContainer(serviceContainer); + sourceRunner.testSource(sourceTester); + } + } finally { + sinkTester.stopServiceContainer(pulsarCluster); + } + } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestRunner.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestRunner.java new file mode 100644 index 0000000000000..cb9ffe47ce8c3 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestRunner.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.io; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.schema.KeyValueSchema; +import org.apache.pulsar.common.schema.KeyValueEncodingType; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import net.jodah.failsafe.RetryPolicy; + +@Slf4j +public abstract class PulsarIOTestRunner { + + final Duration ONE_MINUTE = Duration.ofMinutes(1); + final Duration TEN_SECONDS = Duration.ofSeconds(10); + + @SuppressWarnings({ "unchecked", "rawtypes" }) + protected final RetryPolicy statusRetryPolicy = new RetryPolicy() + .withMaxDuration(ONE_MINUTE) + .withDelay(TEN_SECONDS) + .onRetry(e -> log.error("Retry ... ")); + + protected PulsarCluster pulsarCluster; + protected String functionRuntimeType; + + protected PulsarIOTestRunner(PulsarCluster cluster, String functionRuntimeType) { + this.pulsarCluster = cluster; + this.functionRuntimeType = functionRuntimeType; + } + + @SuppressWarnings("rawtypes") + protected Schema getSchema(boolean jsonWithEnvelope) { + if (jsonWithEnvelope) { + return KeyValueSchema.kvBytes(); + } else { + return KeyValueSchema.of(Schema.AUTO_CONSUME(), Schema.AUTO_CONSUME(), KeyValueEncodingType.SEPARATED); + } + } + + protected void ensureSubscriptionCreated(String inputTopicName, + String subscriptionName, + Schema inputTopicSchema) + throws Exception { + // ensure the function subscription exists before we start producing messages + try (PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build()) { + try (Consumer ignored = client.newConsumer(inputTopicSchema) + .topic(inputTopicName) + .subscriptionType(SubscriptionType.Shared) + .subscriptionName(subscriptionName) + .subscribe()) { + } + } + } + + protected Map produceMessagesToInputTopic(String inputTopicName, + int numMessages) throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + Producer producer = client.newProducer(Schema.STRING) + .topic(inputTopicName) + .create(); + + LinkedHashMap kvs = new LinkedHashMap<>(); + for (int i = 0; i < numMessages; i++) { + String key = "key-" + i; + String value = "value-" + i; + kvs.put(key, value); + producer.newMessage() + .key(key) + .value(value) + .send(); + } + return kvs; + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/RabbitMQSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSinkTester.java similarity index 97% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/RabbitMQSinkTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSinkTester.java index b3978bcd313ee..3b698dc22d118 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/RabbitMQSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSinkTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io.sinks; +package org.apache.pulsar.tests.integration.io; import com.rabbitmq.client.AMQP; import com.rabbitmq.client.Channel; @@ -26,6 +26,7 @@ import com.rabbitmq.client.Envelope; import lombok.Data; import org.apache.pulsar.tests.integration.containers.RabbitMQContainer; +import org.apache.pulsar.tests.integration.io.sinks.SinkTester; import org.apache.pulsar.tests.integration.io.sinks.SinkTester.SinkType; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/RabbitMQSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSourceTester.java similarity index 93% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/RabbitMQSourceTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSourceTester.java index b2adb023bbe29..12d1bbabb9ffc 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/RabbitMQSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/RabbitMQSourceTester.java @@ -16,15 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io.sources; +package org.apache.pulsar.tests.integration.io; import com.rabbitmq.client.BuiltinExchangeType; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; import org.apache.pulsar.tests.integration.containers.RabbitMQContainer; +import org.apache.pulsar.tests.integration.io.sources.SourceTester; -import static org.apache.pulsar.tests.integration.io.sinks.RabbitMQSinkTester.createConnectionFactory; +import static org.apache.pulsar.tests.integration.io.RabbitMQSinkTester.createConnectionFactory; import java.util.LinkedHashMap; import java.util.Map; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarIOSinkRunner.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarIOSinkRunner.java new file mode 100644 index 0000000000000..bde324c46e494 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarIOSinkRunner.java @@ -0,0 +1,447 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.io.sinks; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.schema.AvroSchema; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.SinkStatus; +import org.apache.pulsar.tests.integration.docker.ContainerExecException; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.io.PulsarIOTestRunner; +import org.apache.pulsar.tests.integration.io.sinks.JdbcPostgresSinkTester.Foo; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.apache.pulsar.tests.integration.topologies.PulsarTestBase; +import org.testcontainers.containers.GenericContainer; +import org.testng.collections.Maps; + +import com.google.gson.Gson; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import net.jodah.failsafe.Failsafe; + +@Slf4j +public class PulsarIOSinkRunner extends PulsarIOTestRunner { + + public PulsarIOSinkRunner(PulsarCluster cluster, String functionRuntimeType) { + super(cluster, functionRuntimeType); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public void runSinkTester(SinkTester tester, boolean builtin) throws Exception { + final String tenant = TopicName.PUBLIC_TENANT; + final String namespace = TopicName.DEFAULT_NAMESPACE; + final String inputTopicName = "test-sink-connector-" + + tester.getSinkType() + "-" + functionRuntimeType + "-input-topic-" + PulsarTestBase.randomName(8); + final String sinkName = "test-sink-connector-" + + tester.getSinkType().name().toLowerCase() + "-" + functionRuntimeType + "-name-" + PulsarTestBase.randomName(8); + final int numMessages = 20; + + // prepare the testing environment for sink + prepareSink(tester); + + ensureSubscriptionCreated( + inputTopicName, + String.format("public/default/%s", sinkName), + tester.getInputTopicSchema()); + + // submit the sink connector + submitSinkConnector(tester, tenant, namespace, sinkName, inputTopicName); + + // get sink info + getSinkInfoSuccess(tester, tenant, namespace, sinkName, builtin); + + // get sink status + Failsafe.with(statusRetryPolicy).run(() -> getSinkStatus(tenant, namespace, sinkName)); + + // produce messages + Map kvs; + if (tester instanceof JdbcPostgresSinkTester) { + kvs = produceSchemaInsertMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcPostgresSinkTester.Foo.class)); + // wait for sink to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages)); + + // validate the sink result + tester.validateSinkResult(kvs); + + kvs = produceSchemaUpdateMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcPostgresSinkTester.Foo.class)); + + // wait for sink to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages + 20)); + + // validate the sink result + tester.validateSinkResult(kvs); + + kvs = produceSchemaDeleteMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcPostgresSinkTester.Foo.class)); + + // wait for sink to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages + 20 + 20)); + + // validate the sink result + tester.validateSinkResult(kvs); + + } else { + kvs = produceMessagesToInputTopic(inputTopicName, numMessages); + // wait for sink to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages)); + // validate the sink result + tester.validateSinkResult(kvs); + } + + // update the sink + updateSinkConnector(tester, tenant, namespace, sinkName, inputTopicName); + + // delete the sink + deleteSink(tenant, namespace, sinkName); + + // get sink info (sink should be deleted) + getSinkInfoNotFound(tenant, namespace, sinkName); + } + + @SuppressWarnings("rawtypes") + protected void prepareSink(SinkTester tester) throws Exception { + tester.prepareSink(); + } + + protected void submitSinkConnector(@SuppressWarnings("rawtypes") SinkTester tester, + String tenant, + String namespace, + String sinkName, + String inputTopicName) throws Exception { + String[] commands; + if (tester.getSinkType() != SinkTester.SinkType.UNDEFINED) { + commands = new String[] { + PulsarCluster.ADMIN_SCRIPT, + "sink", "create", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName, + "--sink-type", tester.sinkType().getValue().toLowerCase(), + "--sinkConfig", new Gson().toJson(tester.sinkConfig()), + "--inputs", inputTopicName + }; + } else { + commands = new String[] { + PulsarCluster.ADMIN_SCRIPT, + "sink", "create", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName, + "--archive", tester.getSinkArchive(), + "--classname", tester.getSinkClassName(), + "--sinkConfig", new Gson().toJson(tester.sinkConfig()), + "--inputs", inputTopicName + }; + } + log.info("Run command : {}", StringUtils.join(commands, ' ')); + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + assertTrue( + result.getStdout().contains("\"Created successfully\""), + result.getStdout()); + } + + protected void updateSinkConnector(@SuppressWarnings("rawtypes") SinkTester tester, + String tenant, + String namespace, + String sinkName, + String inputTopicName) throws Exception { + String[] commands; + if (tester.getSinkType() != SinkTester.SinkType.UNDEFINED) { + commands = new String[] { + PulsarCluster.ADMIN_SCRIPT, + "sink", "update", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName, + "--sink-type", tester.sinkType().getValue().toLowerCase(), + "--sinkConfig", new Gson().toJson(tester.sinkConfig()), + "--inputs", inputTopicName, + "--parallelism", "2" + }; + } else { + commands = new String[] { + PulsarCluster.ADMIN_SCRIPT, + "sink", "create", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName, + "--archive", tester.getSinkArchive(), + "--classname", tester.getSinkClassName(), + "--sinkConfig", new Gson().toJson(tester.sinkConfig()), + "--inputs", inputTopicName, + "--parallelism", "2" + }; + } + log.info("Run command : {}", StringUtils.join(commands, ' ')); + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + assertTrue( + result.getStdout().contains("\"Updated successfully\""), + result.getStdout()); + } + + protected void getSinkInfoSuccess(@SuppressWarnings("rawtypes") SinkTester tester, + String tenant, + String namespace, + String sinkName, + boolean builtin) throws Exception { + String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "sink", + "get", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName + }; + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + log.info("Get sink info : {}", result.getStdout()); + if (builtin) { + assertTrue( + result.getStdout().contains("\"archive\": \"builtin://" + tester.getSinkType().getValue().toLowerCase() + "\""), + result.getStdout() + ); + } else { + assertTrue( + result.getStdout().contains("\"className\": \"" + tester.getSinkClassName() + "\""), + result.getStdout() + ); + } + } + + protected void getSinkStatus(String tenant, String namespace, String sinkName) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "sink", + "status", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName + }; + + final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + log.info("Get sink status : {}", result.getStdout()); + + assertEquals(result.getExitCode(), 0); + + SinkStatus sinkStatus = SinkStatus.decode(result.getStdout()); + + assertEquals(sinkStatus.getNumInstances(), 1); + assertEquals(sinkStatus.getNumRunning(), 1); + assertEquals(sinkStatus.getInstances().size(), 1); + assertEquals(sinkStatus.getInstances().get(0).getInstanceId(), 0); + assertEquals(sinkStatus.getInstances().get(0).getStatus().isRunning(), true); + assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); + assertEquals(sinkStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); + } + + protected void deleteSink(String tenant, String namespace, String sinkName) throws Exception { + + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "sink", + "delete", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName + }; + + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + assertTrue( + result.getStdout().contains("Deleted successfully"), + result.getStdout() + ); + result.assertNoStderr(); + } + + protected void getSinkInfoNotFound(String tenant, String namespace, String sinkName) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "sink", + "get", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName + }; + try { + pulsarCluster.getAnyWorker().execCmd(commands); + fail("Command should have exited with non-zero"); + } catch (ContainerExecException e) { + assertTrue(e.getResult().getStderr().contains("Reason: Sink " + sinkName + " doesn't exist")); + } + } + + protected void waitForProcessingSinkMessages(String tenant, + String namespace, + String sinkName, + int numMessages) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "sink", + "status", + "--tenant", tenant, + "--namespace", namespace, + "--name", sinkName + }; + + final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + log.info("Get sink status : {}", result.getStdout()); + + assertEquals(result.getExitCode(), 0); + + final SinkStatus sinkStatus = SinkStatus.decode(result.getStdout()); + + assertEquals(sinkStatus.getNumInstances(), 1); + assertEquals(sinkStatus.getNumRunning(), 1); + assertEquals(sinkStatus.getInstances().size(), 1); + assertEquals(sinkStatus.getInstances().get(0).getInstanceId(), 0); + assertEquals(sinkStatus.getInstances().get(0).getStatus().isRunning(), true); + assertTrue(sinkStatus.getInstances().get(0).getStatus().getLastReceivedTime() > 0); + assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumReadFromPulsar(), numMessages); + assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumWrittenToSink(), numMessages); + assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); + assertEquals(sinkStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); + } + + // This for JdbcPostgresSinkTester + protected Map produceSchemaInsertMessagesToInputTopic(String inputTopicName, + int numMessages, + Schema schema) throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + Producer producer = client.newProducer(schema) + .topic(inputTopicName) + .create(); + + LinkedHashMap kvs = new LinkedHashMap<>(); + for (int i = 0; i < numMessages; i++) { + String key = "key-" + i; + + JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo(); + obj.setField1("field1_insert_" + i); + obj.setField2("field2_insert_" + i); + obj.setField3(i); + String value = new String(schema.encode(obj)); + Map properties = Maps.newHashMap(); + properties.put("ACTION", "INSERT"); + + kvs.put(key, value); + kvs.put("ACTION", "INSERT"); + producer.newMessage() + .properties(properties) + .key(key) + .value(obj) + .send(); + } + return kvs; + } + + // This for JdbcPostgresSinkTester + protected Map produceSchemaUpdateMessagesToInputTopic(String inputTopicName, + int numMessages, + Schema schema) throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + Producer producer = client.newProducer(schema) + .topic(inputTopicName) + .create(); + + LinkedHashMap kvs = new LinkedHashMap<>(); + log.info("update start"); + for (int i = 0; i < numMessages; i++) { + String key = "key-" + i; + + JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo(); + obj.setField1("field1_insert_" + i); + obj.setField2("field2_update_" + i); + obj.setField3(i); + String value = new String(schema.encode(obj)); + Map properties = Maps.newHashMap(); + properties.put("ACTION", "UPDATE"); + + kvs.put(key, value); + kvs.put("ACTION", "UPDATE"); + producer.newMessage() + .properties(properties) + .key(key) + .value(obj) + .send(); + } + log.info("update end"); + return kvs; + } + + // This for JdbcPostgresSinkTester + protected Map produceSchemaDeleteMessagesToInputTopic(String inputTopicName, + int numMessages, + Schema schema) throws Exception { + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + Producer producer = client.newProducer(schema) + .topic(inputTopicName) + .create(); + + LinkedHashMap kvs = new LinkedHashMap<>(); + for (int i = 0; i < numMessages; i++) { + String key = "key-" + i; + + JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo(); + obj.setField1("field1_insert_" + i); + obj.setField2("field2_update_" + i); + obj.setField3(i); + String value = new String(schema.encode(obj)); + Map properties = Maps.newHashMap(); + properties.put("ACTION", "DELETE"); + + kvs.put(key, value); + kvs.put("ACTION", "DELETE"); + producer.newMessage() + .properties(properties) + .key(key) + .value(obj) + .send(); + } + return kvs; + } +} \ No newline at end of file diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java index c133a0fe1a877..5c71bd4c14cc3 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java @@ -18,39 +18,14 @@ */ package org.apache.pulsar.tests.integration.io.sinks; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; -import java.util.LinkedHashMap; -import java.util.Map; - -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.impl.schema.AvroSchema; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.SinkStatus; -import org.apache.pulsar.tests.integration.docker.ContainerExecException; -import org.apache.pulsar.tests.integration.docker.ContainerExecResult; import org.apache.pulsar.tests.integration.io.PulsarIOTestBase; -import org.apache.pulsar.tests.integration.io.sinks.JdbcPostgresSinkTester.Foo; +import org.apache.pulsar.tests.integration.io.RabbitMQSinkTester; +import org.apache.pulsar.tests.integration.io.RabbitMQSourceTester; import org.apache.pulsar.tests.integration.io.sources.KafkaSourceTester; -import org.apache.pulsar.tests.integration.io.sources.RabbitMQSourceTester; -import org.apache.pulsar.tests.integration.io.sources.SourceTester; -import org.apache.pulsar.tests.integration.topologies.PulsarCluster; -import org.testcontainers.containers.GenericContainer; import org.testng.annotations.Test; -import org.testng.collections.Maps; - -import com.google.gson.Gson; -import lombok.Cleanup; -import lombok.extern.slf4j.Slf4j; -import net.jodah.failsafe.Failsafe; -@Slf4j public class PulsarSinksTest extends PulsarIOTestBase { @Test(groups = "sink") @@ -90,428 +65,4 @@ public void testRabbitMQSink() throws Exception { testSink(new RabbitMQSinkTester(containerName), true, new RabbitMQSourceTester(containerName)); } - private void testSink(SinkTester tester, boolean builtin) throws Exception { - if (pulsarCluster == null) { - super.setupCluster(); - super.setupFunctionWorkers(); - } - tester.startServiceContainer(pulsarCluster); - try { - runSinkTester(tester, builtin); - } finally { - tester.stopServiceContainer(pulsarCluster); - } - } - - private void testSink(SinkTester sinkTester, - boolean builtinSink, - SourceTester sourceTester) - throws Exception { - if (pulsarCluster == null) { - super.setupCluster(); - super.setupFunctionWorkers(); - } - ServiceContainerT serviceContainer = sinkTester.startServiceContainer(pulsarCluster); - try { - runSinkTester(sinkTester, builtinSink); - if (null != sourceTester) { - sourceTester.setServiceContainer(serviceContainer); -// testSource(sourceTester); - } - } finally { - sinkTester.stopServiceContainer(pulsarCluster); - } - } - - private void runSinkTester(SinkTester tester, boolean builtin) throws Exception { - final String tenant = TopicName.PUBLIC_TENANT; - final String namespace = TopicName.DEFAULT_NAMESPACE; - final String inputTopicName = "test-sink-connector-" - + tester.getSinkType() + "-" + functionRuntimeType + "-input-topic-" + randomName(8); - final String sinkName = "test-sink-connector-" - + tester.getSinkType().name().toLowerCase() + "-" + functionRuntimeType + "-name-" + randomName(8); - final int numMessages = 20; - - // prepare the testing environment for sink - prepareSink(tester); - - ensureSubscriptionCreated( - inputTopicName, - String.format("public/default/%s", sinkName), - tester.getInputTopicSchema()); - - // submit the sink connector - submitSinkConnector(tester, tenant, namespace, sinkName, inputTopicName); - - // get sink info - getSinkInfoSuccess(tester, tenant, namespace, sinkName, builtin); - - // get sink status - Failsafe.with(statusRetryPolicy).run(() -> getSinkStatus(tenant, namespace, sinkName)); - - // produce messages - Map kvs; - if (tester instanceof JdbcPostgresSinkTester) { - kvs = produceSchemaInsertMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcPostgresSinkTester.Foo.class)); - // wait for sink to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages)); - - // validate the sink result - tester.validateSinkResult(kvs); - - kvs = produceSchemaUpdateMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcPostgresSinkTester.Foo.class)); - - // wait for sink to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages + 20)); - - // validate the sink result - tester.validateSinkResult(kvs); - - kvs = produceSchemaDeleteMessagesToInputTopic(inputTopicName, numMessages, AvroSchema.of(JdbcPostgresSinkTester.Foo.class)); - - // wait for sink to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages + 20 + 20)); - - // validate the sink result - tester.validateSinkResult(kvs); - - } else { - kvs = produceMessagesToInputTopic(inputTopicName, numMessages); - // wait for sink to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSinkMessages(tenant, namespace, sinkName, numMessages)); - // validate the sink result - tester.validateSinkResult(kvs); - } - - // update the sink - updateSinkConnector(tester, tenant, namespace, sinkName, inputTopicName); - - // delete the sink - deleteSink(tenant, namespace, sinkName); - - // get sink info (sink should be deleted) - getSinkInfoNotFound(tenant, namespace, sinkName); - } - - protected void prepareSink(SinkTester tester) throws Exception { - tester.prepareSink(); - } - - protected void submitSinkConnector(SinkTester tester, - String tenant, - String namespace, - String sinkName, - String inputTopicName) throws Exception { - String[] commands; - if (tester.getSinkType() != SinkTester.SinkType.UNDEFINED) { - commands = new String[] { - PulsarCluster.ADMIN_SCRIPT, - "sink", "create", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName, - "--sink-type", tester.sinkType().getValue().toLowerCase(), - "--sinkConfig", new Gson().toJson(tester.sinkConfig()), - "--inputs", inputTopicName - }; - } else { - commands = new String[] { - PulsarCluster.ADMIN_SCRIPT, - "sink", "create", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName, - "--archive", tester.getSinkArchive(), - "--classname", tester.getSinkClassName(), - "--sinkConfig", new Gson().toJson(tester.sinkConfig()), - "--inputs", inputTopicName - }; - } - log.info("Run command : {}", StringUtils.join(commands, ' ')); - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - assertTrue( - result.getStdout().contains("\"Created successfully\""), - result.getStdout()); - } - - protected void updateSinkConnector(SinkTester tester, - String tenant, - String namespace, - String sinkName, - String inputTopicName) throws Exception { - String[] commands; - if (tester.getSinkType() != SinkTester.SinkType.UNDEFINED) { - commands = new String[] { - PulsarCluster.ADMIN_SCRIPT, - "sink", "update", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName, - "--sink-type", tester.sinkType().getValue().toLowerCase(), - "--sinkConfig", new Gson().toJson(tester.sinkConfig()), - "--inputs", inputTopicName, - "--parallelism", "2" - }; - } else { - commands = new String[] { - PulsarCluster.ADMIN_SCRIPT, - "sink", "create", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName, - "--archive", tester.getSinkArchive(), - "--classname", tester.getSinkClassName(), - "--sinkConfig", new Gson().toJson(tester.sinkConfig()), - "--inputs", inputTopicName, - "--parallelism", "2" - }; - } - log.info("Run command : {}", StringUtils.join(commands, ' ')); - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - assertTrue( - result.getStdout().contains("\"Updated successfully\""), - result.getStdout()); - } - - protected void getSinkInfoSuccess(SinkTester tester, - String tenant, - String namespace, - String sinkName, - boolean builtin) throws Exception { - String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "sink", - "get", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName - }; - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - log.info("Get sink info : {}", result.getStdout()); - if (builtin) { - assertTrue( - result.getStdout().contains("\"archive\": \"builtin://" + tester.getSinkType().getValue().toLowerCase() + "\""), - result.getStdout() - ); - } else { - assertTrue( - result.getStdout().contains("\"className\": \"" + tester.getSinkClassName() + "\""), - result.getStdout() - ); - } - } - - protected void getSinkStatus(String tenant, String namespace, String sinkName) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "sink", - "status", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName - }; - - final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - log.info("Get sink status : {}", result.getStdout()); - - assertEquals(result.getExitCode(), 0); - - SinkStatus sinkStatus = SinkStatus.decode(result.getStdout()); - - assertEquals(sinkStatus.getNumInstances(), 1); - assertEquals(sinkStatus.getNumRunning(), 1); - assertEquals(sinkStatus.getInstances().size(), 1); - assertEquals(sinkStatus.getInstances().get(0).getInstanceId(), 0); - assertEquals(sinkStatus.getInstances().get(0).getStatus().isRunning(), true); - assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); - assertEquals(sinkStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); - } - - protected void deleteSink(String tenant, String namespace, String sinkName) throws Exception { - - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "sink", - "delete", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName - }; - - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - assertTrue( - result.getStdout().contains("Deleted successfully"), - result.getStdout() - ); - result.assertNoStderr(); - } - - protected void getSinkInfoNotFound(String tenant, String namespace, String sinkName) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "sink", - "get", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName - }; - try { - pulsarCluster.getAnyWorker().execCmd(commands); - fail("Command should have exited with non-zero"); - } catch (ContainerExecException e) { - assertTrue(e.getResult().getStderr().contains("Reason: Sink " + sinkName + " doesn't exist")); - } - } - - - protected void waitForProcessingSinkMessages(String tenant, - String namespace, - String sinkName, - int numMessages) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "sink", - "status", - "--tenant", tenant, - "--namespace", namespace, - "--name", sinkName - }; - - final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - log.info("Get sink status : {}", result.getStdout()); - - assertEquals(result.getExitCode(), 0); - - final SinkStatus sinkStatus = SinkStatus.decode(result.getStdout()); - - assertEquals(sinkStatus.getNumInstances(), 1); - assertEquals(sinkStatus.getNumRunning(), 1); - assertEquals(sinkStatus.getInstances().size(), 1); - assertEquals(sinkStatus.getInstances().get(0).getInstanceId(), 0); - assertEquals(sinkStatus.getInstances().get(0).getStatus().isRunning(), true); - assertTrue(sinkStatus.getInstances().get(0).getStatus().getLastReceivedTime() > 0); - assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumReadFromPulsar(), numMessages); - assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumWrittenToSink(), numMessages); - assertEquals(sinkStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); - assertEquals(sinkStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); - } - - // This for JdbcPostgresSinkTester - protected Map produceSchemaInsertMessagesToInputTopic(String inputTopicName, - int numMessages, - Schema schema) throws Exception { - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - Producer producer = client.newProducer(schema) - .topic(inputTopicName) - .create(); - - LinkedHashMap kvs = new LinkedHashMap<>(); - for (int i = 0; i < numMessages; i++) { - String key = "key-" + i; - - JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo(); - obj.setField1("field1_insert_" + i); - obj.setField2("field2_insert_" + i); - obj.setField3(i); - String value = new String(schema.encode(obj)); - Map properties = Maps.newHashMap(); - properties.put("ACTION", "INSERT"); - - kvs.put(key, value); - kvs.put("ACTION", "INSERT"); - producer.newMessage() - .properties(properties) - .key(key) - .value(obj) - .send(); - } - return kvs; - } - - // This for JdbcPostgresSinkTester - protected Map produceSchemaUpdateMessagesToInputTopic(String inputTopicName, - int numMessages, - Schema schema) throws Exception { - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - Producer producer = client.newProducer(schema) - .topic(inputTopicName) - .create(); - - LinkedHashMap kvs = new LinkedHashMap<>(); - log.info("update start"); - for (int i = 0; i < numMessages; i++) { - String key = "key-" + i; - - JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo(); - obj.setField1("field1_insert_" + i); - obj.setField2("field2_update_" + i); - obj.setField3(i); - String value = new String(schema.encode(obj)); - Map properties = Maps.newHashMap(); - properties.put("ACTION", "UPDATE"); - - kvs.put(key, value); - kvs.put("ACTION", "UPDATE"); - producer.newMessage() - .properties(properties) - .key(key) - .value(obj) - .send(); - } - log.info("update end"); - return kvs; - } - - // This for JdbcPostgresSinkTester - protected Map produceSchemaDeleteMessagesToInputTopic(String inputTopicName, - int numMessages, - Schema schema) throws Exception { - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - Producer producer = client.newProducer(schema) - .topic(inputTopicName) - .create(); - - LinkedHashMap kvs = new LinkedHashMap<>(); - for (int i = 0; i < numMessages; i++) { - String key = "key-" + i; - - JdbcPostgresSinkTester.Foo obj = new JdbcPostgresSinkTester.Foo(); - obj.setField1("field1_insert_" + i); - obj.setField2("field2_update_" + i); - obj.setField3(i); - String value = new String(schema.encode(obj)); - Map properties = Maps.newHashMap(); - properties.put("ACTION", "DELETE"); - - kvs.put(key, value); - kvs.put("ACTION", "DELETE"); - producer.newMessage() - .properties(properties) - .key(key) - .value(obj) - .send(); - } - return kvs; - } - - } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarIOSourceRunner.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarIOSourceRunner.java new file mode 100644 index 0000000000000..e99be1157b765 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarIOSourceRunner.java @@ -0,0 +1,296 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.io.sources; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.SourceStatus; +import org.apache.pulsar.tests.integration.docker.ContainerExecException; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.io.PulsarIOTestRunner; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.apache.pulsar.tests.integration.topologies.PulsarTestBase; +import org.testcontainers.containers.GenericContainer; + +import com.google.gson.Gson; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import net.jodah.failsafe.Failsafe; + +@Slf4j +public class PulsarIOSourceRunner extends PulsarIOTestRunner { + + public PulsarIOSourceRunner(PulsarCluster cluster, String functionRuntimeType) { + super(cluster, functionRuntimeType); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public void testSource(SourceTester tester) throws Exception { + final String tenant = TopicName.PUBLIC_TENANT; + final String namespace = TopicName.DEFAULT_NAMESPACE; + final String outputTopicName = "test-source-connector-" + + functionRuntimeType + "-output-topic-" + PulsarTestBase.randomName(8); + final String sourceName = "test-source-connector-" + + functionRuntimeType + "-name-" + PulsarTestBase.randomName(8); + final int numMessages = 20; + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); + admin.topics().createNonPartitionedTopic(outputTopicName); + + @Cleanup + Consumer consumer = client.newConsumer(Schema.STRING) + .topic(outputTopicName) + .subscriptionName("source-tester") + .subscriptionType(SubscriptionType.Exclusive) + .subscribe(); + + // prepare the testing environment for source + prepareSource(tester); + + // submit the source connector + submitSourceConnector(tester, tenant, namespace, sourceName, outputTopicName); + + // get source info + getSourceInfoSuccess(tester, tenant, namespace, sourceName); + + // get source status + Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); + + // produce messages + Map kvs = tester.produceSourceMessages(numMessages); + + // wait for source to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); + + // validate the source result + validateSourceResult(consumer, kvs); + + // update the source connector + updateSourceConnector(tester, tenant, namespace, sourceName, outputTopicName); + + // delete the source + deleteSource(tenant, namespace, sourceName); + + // get source info (source should be deleted) + getSourceInfoNotFound(tenant, namespace, sourceName); + } + + @SuppressWarnings("rawtypes") + protected void prepareSource(SourceTester tester) throws Exception { + tester.prepareSource(); + } + + @SuppressWarnings("rawtypes") + protected void submitSourceConnector(SourceTester tester, + String tenant, + String namespace, + String sourceName, + String outputTopicName) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", "create", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName, + "--source-type", tester.sourceType(), + "--sourceConfig", new Gson().toJson(tester.sourceConfig()), + "--destinationTopicName", outputTopicName + }; + + log.info("Run command : {}", StringUtils.join(commands, ' ')); + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + assertTrue( + result.getStdout().contains("\"Created successfully\""), + result.getStdout()); + } + + @SuppressWarnings("rawtypes") + protected void updateSourceConnector(SourceTester tester, + String tenant, + String namespace, + String sourceName, + String outputTopicName) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", "update", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName, + "--source-type", tester.sourceType(), + "--sourceConfig", new Gson().toJson(tester.sourceConfig()), + "--destinationTopicName", outputTopicName, + "--parallelism", "2" + }; + + log.info("Run command : {}", StringUtils.join(commands, ' ')); + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + assertTrue( + result.getStdout().contains("\"Updated successfully\""), + result.getStdout()); + } + + @SuppressWarnings("rawtypes") + protected void getSourceInfoSuccess(SourceTester tester, + String tenant, + String namespace, + String sourceName) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", + "get", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName + }; + + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + log.info("Get source info : {}", result.getStdout()); + assertTrue( + result.getStdout().contains("\"archive\": \"builtin://" + tester.getSourceType() + "\""), + result.getStdout() + ); + } + + protected void getSourceStatus(String tenant, String namespace, String sourceName) throws Exception { + + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", + "status", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName + }; + + final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + log.info("Get source status : {}", result.getStdout()); + + assertEquals(result.getExitCode(), 0); + + final SourceStatus sourceStatus = SourceStatus.decode(result.getStdout()); + + assertEquals(sourceStatus.getNumInstances(), 1); + assertEquals(sourceStatus.getNumRunning(), 1); + assertEquals(sourceStatus.getInstances().size(), 1); + assertEquals(sourceStatus.getInstances().get(0).getStatus().isRunning(), true); + assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); + assertEquals(sourceStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); + + assertTrue(result.getStdout().contains("\"running\" : true")); + + } + + protected void validateSourceResult(Consumer consumer, + Map kvs) throws Exception { + for (Map.Entry kv : kvs.entrySet()) { + Message msg = consumer.receive(); + assertEquals(kv.getKey(), msg.getKey()); + assertEquals(kv.getValue(), msg.getValue()); + } + } + + protected void waitForProcessingSourceMessages(String tenant, + String namespace, + String sourceName, + int numMessages) throws Exception { + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", + "status", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName + }; + + final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + log.info("Get source status : {}", result.getStdout()); + + assertEquals(result.getExitCode(), 0); + + SourceStatus sourceStatus = SourceStatus.decode(result.getStdout()); + assertEquals(sourceStatus.getNumInstances(), 1); + assertEquals(sourceStatus.getNumRunning(), 1); + assertEquals(sourceStatus.getInstances().size(), 1); + assertEquals(sourceStatus.getInstances().get(0).getInstanceId(), 0); + assertEquals(sourceStatus.getInstances().get(0).getStatus().isRunning(), true); + assertTrue(sourceStatus.getInstances().get(0).getStatus().getLastReceivedTime() > 0); + assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumReceivedFromSource(), numMessages); + assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumWritten(), numMessages); + assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); + assertEquals(sourceStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); + } + + protected void deleteSource(String tenant, String namespace, String sourceName) throws Exception { + + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", + "delete", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName + }; + + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); + assertTrue( + result.getStdout().contains("Delete source successfully"), + result.getStdout() + ); + result.assertNoStderr(); + } + + protected void getSourceInfoNotFound(String tenant, String namespace, String sourceName) throws Exception { + + final String[] commands = { + PulsarCluster.ADMIN_SCRIPT, + "source", + "get", + "--tenant", tenant, + "--namespace", namespace, + "--name", sourceName + }; + + try { + pulsarCluster.getAnyWorker().execCmd(commands); + fail("Command should have exited with non-zero"); + } catch (ContainerExecException e) { + assertTrue(e.getResult().getStderr().contains("Reason: Source " + sourceName + " doesn't exist")); + } + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarSourcesTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarSourcesTest.java deleted file mode 100644 index 6d8cf83e0e447..0000000000000 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarSourcesTest.java +++ /dev/null @@ -1,588 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.tests.integration.io.sources; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; - -import java.util.Map; - -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.api.Consumer; -import org.apache.pulsar.client.api.Message; -import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.client.api.Schema; -import org.apache.pulsar.client.api.SubscriptionInitialPosition; -import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.SourceStatus; -import org.apache.pulsar.common.schema.SchemaInfo; -import org.apache.pulsar.tests.integration.containers.DebeziumMongoDbContainer; -import org.apache.pulsar.tests.integration.containers.DebeziumMySQLContainer; -import org.apache.pulsar.tests.integration.containers.DebeziumPostgreSqlContainer; -import org.apache.pulsar.tests.integration.docker.ContainerExecException; -import org.apache.pulsar.tests.integration.docker.ContainerExecResult; -import org.apache.pulsar.tests.integration.io.PulsarIOTestBase; -import org.apache.pulsar.tests.integration.topologies.PulsarCluster; -import org.testcontainers.containers.GenericContainer; -import org.testng.annotations.Test; - -import com.google.gson.Gson; - -import lombok.Cleanup; -import lombok.extern.slf4j.Slf4j; -import net.jodah.failsafe.Failsafe; - -@Slf4j -public class PulsarSourcesTest extends PulsarIOTestBase { - - @Test(groups = "source") - public void testDebeziumMySqlSourceJson() throws Exception { - testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter", true); - } - - @Test(groups = "source") - public void testDebeziumMySqlSourceAvro() throws Exception { - testDebeziumMySqlConnect( - "org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter", false); - } - - @Test(groups = "source") - public void testDebeziumPostgreSqlSource() throws Exception { - testDebeziumPostgreSqlConnect("org.apache.kafka.connect.json.JsonConverter", true); - } - - @Test(groups = "source") - public void testDebeziumMongoDbSource() throws Exception{ - testDebeziumMongoDbConnect("org.apache.kafka.connect.json.JsonConverter", true); - } - - private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { - - final String tenant = TopicName.PUBLIC_TENANT; - final String namespace = TopicName.DEFAULT_NAMESPACE; - final String outputTopicName = "debe-output-topic-name-" + testId.getAndIncrement(); - boolean isJsonConverter = converterClassName.endsWith("JsonConverter"); - final String consumeTopicName = "debezium/mysql-" - + (isJsonConverter ? "json" : "avro") - + "/dbserver1.inventory.products"; - final String sourceName = "test-source-debezium-mysql" + (isJsonConverter ? "json" : "avro") - + "-" + functionRuntimeType + "-" + randomName(8); - - // This is the binlog count that contained in mysql container. - final int numMessages = 47; - - if (pulsarCluster == null) { - super.setupCluster(); - super.setupFunctionWorkers(); - } - - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); - initNamespace(admin); - - try { - SchemaInfo lastSchemaInfo = admin.schemas().getSchemaInfo(consumeTopicName); - log.info("lastSchemaInfo: {}", lastSchemaInfo == null ? "null" : lastSchemaInfo.toString()); - } catch (Exception e) { - log.warn("failed to get schemaInfo for topic: {}, exceptions message: {}", - consumeTopicName, e.getMessage()); - } - - admin.topics().createNonPartitionedTopic(outputTopicName); - - @Cleanup - DebeziumMySqlSourceTester sourceTester = new DebeziumMySqlSourceTester(pulsarCluster, converterClassName); - sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); - - // setup debezium mysql server - DebeziumMySQLContainer mySQLContainer = new DebeziumMySQLContainer(pulsarCluster.getClusterName()); - sourceTester.setServiceContainer(mySQLContainer); - - // prepare the testing environment for source - prepareSource(sourceTester); - - // submit the source connector - submitSourceConnector(sourceTester, tenant, namespace, sourceName, outputTopicName); - - // get source info - getSourceInfoSuccess(sourceTester, tenant, namespace, sourceName); - - // get source status - Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); - - // wait for source to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); - - @Cleanup - Consumer consumer = client.newConsumer(getSchema(jsonWithEnvelope)) - .topic(consumeTopicName) - .subscriptionName("debezium-source-tester") - .subscriptionType(SubscriptionType.Exclusive) - .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) - .subscribe(); - log.info("[debezium mysql test] create consumer finish. converterName: {}", converterClassName); - - // validate the source result - sourceTester.validateSourceResult(consumer, 9, null, converterClassName); - - // prepare insert event - sourceTester.prepareInsertEvent(); - - // validate the source insert event - sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT, converterClassName); - - // prepare update event - sourceTester.prepareUpdateEvent(); - - // validate the source update event - sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE, converterClassName); - - // prepare delete event - sourceTester.prepareDeleteEvent(); - - // validate the source delete event - sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE, converterClassName); - - // delete the source - deleteSource(tenant, namespace, sourceName); - - // get source info (source should be deleted) - getSourceInfoNotFound(tenant, namespace, sourceName); - } - - private void testDebeziumPostgreSqlConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { - - final String tenant = TopicName.PUBLIC_TENANT; - final String namespace = TopicName.DEFAULT_NAMESPACE; - final String outputTopicName = "debe-output-topic-name-" + testId.getAndIncrement(); - final String consumeTopicName = "debezium/postgresql/dbserver1.inventory.products"; - final String sourceName = "test-source-debezium-postgersql-" + functionRuntimeType + "-" + randomName(8); - - - // This is the binlog count that contained in postgresql container. - final int numMessages = 26; - - if (pulsarCluster == null) { - super.setupCluster(); - super.setupFunctionWorkers(); - } - - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); - initNamespace(admin); - - admin.topics().createNonPartitionedTopic(consumeTopicName); - admin.topics().createNonPartitionedTopic(outputTopicName); - - @Cleanup - Consumer consumer = client.newConsumer(getSchema(jsonWithEnvelope)) - .topic(consumeTopicName) - .subscriptionName("debezium-source-tester") - .subscriptionType(SubscriptionType.Exclusive) - .subscribe(); - - @Cleanup - DebeziumPostgreSqlSourceTester sourceTester = new DebeziumPostgreSqlSourceTester(pulsarCluster); - sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); - - // setup debezium postgresql server - DebeziumPostgreSqlContainer postgreSqlContainer = new DebeziumPostgreSqlContainer(pulsarCluster.getClusterName()); - sourceTester.setServiceContainer(postgreSqlContainer); - - // prepare the testing environment for source - prepareSource(sourceTester); - - // submit the source connector - submitSourceConnector(sourceTester, tenant, namespace, sourceName, outputTopicName); - - // get source info - getSourceInfoSuccess(sourceTester, tenant, namespace, sourceName); - - // get source status - Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); - - // wait for source to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); - - // validate the source result - sourceTester.validateSourceResult(consumer, 9, null, converterClassName); - - // prepare insert event - sourceTester.prepareInsertEvent(); - - // validate the source insert event - sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT, converterClassName); - - // prepare update event - sourceTester.prepareUpdateEvent(); - - // validate the source update event - sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE, converterClassName); - - // prepare delete event - sourceTester.prepareDeleteEvent(); - - // validate the source delete event - sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE, converterClassName); - - // delete the source - deleteSource(tenant, namespace, sourceName); - - // get source info (source should be deleted) - getSourceInfoNotFound(tenant, namespace, sourceName); - } - - private void testDebeziumMongoDbConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { - - final String tenant = TopicName.PUBLIC_TENANT; - final String namespace = TopicName.DEFAULT_NAMESPACE; - final String outputTopicName = "debe-output-topic-name"; - final String consumeTopicName = "debezium/mongodb/dbserver1.inventory.products"; - final String sourceName = "test-source-connector-" - + functionRuntimeType + "-name-" + randomName(8); - - // This is the binlog count that contained in mongodb container. - final int numMessages = 17; - - if (pulsarCluster == null) { - super.setupCluster(); - super.setupFunctionWorkers(); - } - - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); - initNamespace(admin); - - admin.topics().createNonPartitionedTopic(consumeTopicName); - admin.topics().createNonPartitionedTopic(outputTopicName); - - @Cleanup - Consumer consumer = client.newConsumer(getSchema(jsonWithEnvelope)) - .topic(consumeTopicName) - .subscriptionName("debezium-source-tester") - .subscriptionType(SubscriptionType.Exclusive) - .subscribe(); - - @Cleanup - DebeziumMongoDbSourceTester sourceTester = new DebeziumMongoDbSourceTester(pulsarCluster); - sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); - - // setup debezium mongodb server - DebeziumMongoDbContainer mongoDbContainer = new DebeziumMongoDbContainer(pulsarCluster.getClusterName()); - sourceTester.setServiceContainer(mongoDbContainer); - // prepare the testing environment for source - prepareSource(sourceTester); - - // submit the source connector - submitSourceConnector(sourceTester, tenant, namespace, sourceName, outputTopicName); - - // get source info - getSourceInfoSuccess(sourceTester, tenant, namespace, sourceName); - - // get source status - Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); - - // wait for source to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); - - // validate the source result - sourceTester.validateSourceResult(consumer, 9, null, converterClassName); - - // prepare insert event - sourceTester.prepareInsertEvent(); - - // validate the source insert event - sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT, converterClassName); - - // prepare update event - sourceTester.prepareUpdateEvent(); - - // validate the source update event - sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE, converterClassName); - - // prepare delete event - sourceTester.prepareDeleteEvent(); - - // validate the source delete event - sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE, converterClassName); - - // delete the source - deleteSource(tenant, namespace, sourceName); - - // get source info (source should be deleted) - getSourceInfoNotFound(tenant, namespace, sourceName); - } - - private void testSource(SourceTester tester) throws Exception { - final String tenant = TopicName.PUBLIC_TENANT; - final String namespace = TopicName.DEFAULT_NAMESPACE; - final String outputTopicName = "test-source-connector-" - + functionRuntimeType + "-output-topic-" + randomName(8); - final String sourceName = "test-source-connector-" - + functionRuntimeType + "-name-" + randomName(8); - final int numMessages = 20; - - @Cleanup - PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup - PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); - admin.topics().createNonPartitionedTopic(outputTopicName); - - @Cleanup - Consumer consumer = client.newConsumer(Schema.STRING) - .topic(outputTopicName) - .subscriptionName("source-tester") - .subscriptionType(SubscriptionType.Exclusive) - .subscribe(); - - // prepare the testing environment for source - prepareSource(tester); - - // submit the source connector - submitSourceConnector(tester, tenant, namespace, sourceName, outputTopicName); - - // get source info - getSourceInfoSuccess(tester, tenant, namespace, sourceName); - - // get source status - Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); - - // produce messages - Map kvs = tester.produceSourceMessages(numMessages); - - // wait for source to process messages - Failsafe.with(statusRetryPolicy).run(() -> - waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); - - // validate the source result - validateSourceResult(consumer, kvs); - - // update the source connector - updateSourceConnector(tester, tenant, namespace, sourceName, outputTopicName); - - // delete the source - deleteSource(tenant, namespace, sourceName); - - // get source info (source should be deleted) - getSourceInfoNotFound(tenant, namespace, sourceName); - } - - protected void prepareSource(SourceTester tester) throws Exception { - tester.prepareSource(); - } - - protected void submitSourceConnector(SourceTester tester, - String tenant, - String namespace, - String sourceName, - String outputTopicName) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", "create", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName, - "--source-type", tester.sourceType(), - "--sourceConfig", new Gson().toJson(tester.sourceConfig()), - "--destinationTopicName", outputTopicName - }; - - log.info("Run command : {}", StringUtils.join(commands, ' ')); - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - assertTrue( - result.getStdout().contains("\"Created successfully\""), - result.getStdout()); - } - - protected void updateSourceConnector(SourceTester tester, - String tenant, - String namespace, - String sourceName, - String outputTopicName) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", "update", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName, - "--source-type", tester.sourceType(), - "--sourceConfig", new Gson().toJson(tester.sourceConfig()), - "--destinationTopicName", outputTopicName, - "--parallelism", "2" - }; - - log.info("Run command : {}", StringUtils.join(commands, ' ')); - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - assertTrue( - result.getStdout().contains("\"Updated successfully\""), - result.getStdout()); - } - - protected void getSourceInfoSuccess(SourceTester tester, - String tenant, - String namespace, - String sourceName) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", - "get", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName - }; - - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - log.info("Get source info : {}", result.getStdout()); - assertTrue( - result.getStdout().contains("\"archive\": \"builtin://" + tester.getSourceType() + "\""), - result.getStdout() - ); - } - - protected void getSourceStatus(String tenant, String namespace, String sourceName) throws Exception { - - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", - "status", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName - }; - - final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - log.info("Get source status : {}", result.getStdout()); - - assertEquals(result.getExitCode(), 0); - - final SourceStatus sourceStatus = SourceStatus.decode(result.getStdout()); - - assertEquals(sourceStatus.getNumInstances(), 1); - assertEquals(sourceStatus.getNumRunning(), 1); - assertEquals(sourceStatus.getInstances().size(), 1); - assertEquals(sourceStatus.getInstances().get(0).getStatus().isRunning(), true); - assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); - assertEquals(sourceStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); - - assertTrue(result.getStdout().contains("\"running\" : true")); - - } - - protected void validateSourceResult(Consumer consumer, - Map kvs) throws Exception { - for (Map.Entry kv : kvs.entrySet()) { - Message msg = consumer.receive(); - assertEquals(kv.getKey(), msg.getKey()); - assertEquals(kv.getValue(), msg.getValue()); - } - } - - protected void waitForProcessingSourceMessages(String tenant, - String namespace, - String sourceName, - int numMessages) throws Exception { - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", - "status", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName - }; - - final ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - log.info("Get source status : {}", result.getStdout()); - - assertEquals(result.getExitCode(), 0); - - SourceStatus sourceStatus = SourceStatus.decode(result.getStdout()); - assertEquals(sourceStatus.getNumInstances(), 1); - assertEquals(sourceStatus.getNumRunning(), 1); - assertEquals(sourceStatus.getInstances().size(), 1); - assertEquals(sourceStatus.getInstances().get(0).getInstanceId(), 0); - assertEquals(sourceStatus.getInstances().get(0).getStatus().isRunning(), true); - assertTrue(sourceStatus.getInstances().get(0).getStatus().getLastReceivedTime() > 0); - assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumReceivedFromSource(), numMessages); - assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumWritten(), numMessages); - assertEquals(sourceStatus.getInstances().get(0).getStatus().getNumRestarts(), 0); - assertEquals(sourceStatus.getInstances().get(0).getStatus().getLatestSystemExceptions().size(), 0); - } - - protected void deleteSource(String tenant, String namespace, String sourceName) throws Exception { - - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", - "delete", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName - }; - - ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd(commands); - assertTrue( - result.getStdout().contains("Delete source successfully"), - result.getStdout() - ); - result.assertNoStderr(); - } - - protected void getSourceInfoNotFound(String tenant, String namespace, String sourceName) throws Exception { - - final String[] commands = { - PulsarCluster.ADMIN_SCRIPT, - "source", - "get", - "--tenant", tenant, - "--namespace", namespace, - "--name", sourceName - }; - - try { - pulsarCluster.getAnyWorker().execCmd(commands); - fail("Command should have exited with non-zero"); - } catch (ContainerExecException e) { - assertTrue(e.getResult().getStderr().contains("Reason: Source " + sourceName + " doesn't exist")); - } - } -} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumMongoDbSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMongoDbSourceTester.java similarity index 97% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumMongoDbSourceTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMongoDbSourceTester.java index 41f2b26eeab22..6eb4f06777ea9 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumMongoDbSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMongoDbSourceTester.java @@ -16,12 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io.sources; +package org.apache.pulsar.tests.integration.io.sources.debezium; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.tests.integration.containers.DebeziumMongoDbContainer; import org.apache.pulsar.tests.integration.containers.PulsarContainer; +import org.apache.pulsar.tests.integration.io.sources.SourceTester; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import java.io.Closeable; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumMySqlSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMySqlSourceTester.java similarity index 97% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumMySqlSourceTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMySqlSourceTester.java index dc982862860eb..ec7e07ccf0be2 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumMySqlSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumMySqlSourceTester.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io.sources; +package org.apache.pulsar.tests.integration.io.sources.debezium; import java.io.Closeable; import java.util.Map; @@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.tests.integration.containers.DebeziumMySQLContainer; import org.apache.pulsar.tests.integration.containers.PulsarContainer; +import org.apache.pulsar.tests.integration.io.sources.SourceTester; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; /** diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumPostgreSqlSourceTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumPostgreSqlSourceTester.java similarity index 97% rename from tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumPostgreSqlSourceTester.java rename to tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumPostgreSqlSourceTester.java index 7a876bdba7aa4..192597e782718 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/DebeziumPostgreSqlSourceTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/DebeziumPostgreSqlSourceTester.java @@ -16,12 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.tests.integration.io.sources; +package org.apache.pulsar.tests.integration.io.sources.debezium; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.tests.integration.containers.DebeziumPostgreSqlContainer; import org.apache.pulsar.tests.integration.containers.PulsarContainer; +import org.apache.pulsar.tests.integration.io.sources.SourceTester; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import java.io.Closeable; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java new file mode 100644 index 0000000000000..202eca93f7b14 --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarDebeziumSourcesTest.java @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.io.sources.debezium; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.tests.integration.containers.DebeziumMongoDbContainer; +import org.apache.pulsar.tests.integration.containers.DebeziumMySQLContainer; +import org.apache.pulsar.tests.integration.containers.DebeziumPostgreSqlContainer; +import org.apache.pulsar.tests.integration.io.PulsarIOTestBase; +import org.testcontainers.shaded.com.google.common.collect.Sets; +import org.testng.annotations.Test; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class PulsarDebeziumSourcesTest extends PulsarIOTestBase { + + protected final AtomicInteger testId = new AtomicInteger(0); + + @Test(groups = "source") + public void testDebeziumMySqlSourceJson() throws Exception { + testDebeziumMySqlConnect("org.apache.kafka.connect.json.JsonConverter", true); + } + + @Test(groups = "source") + public void testDebeziumMySqlSourceAvro() throws Exception { + testDebeziumMySqlConnect( + "org.apache.pulsar.kafka.shade.io.confluent.connect.avro.AvroConverter", false); + } + + @Test(groups = "source") + public void testDebeziumPostgreSqlSource() throws Exception { + testDebeziumPostgreSqlConnect("org.apache.kafka.connect.json.JsonConverter", true); + } + + @Test(groups = "source") + public void testDebeziumMongoDbSource() throws Exception{ + testDebeziumMongoDbConnect("org.apache.kafka.connect.json.JsonConverter", true); + } + + private void testDebeziumMySqlConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { + + final String tenant = TopicName.PUBLIC_TENANT; + final String namespace = TopicName.DEFAULT_NAMESPACE; + final String outputTopicName = "debe-output-topic-name-" + testId.getAndIncrement(); + boolean isJsonConverter = converterClassName.endsWith("JsonConverter"); + final String consumeTopicName = "debezium/mysql-" + + (isJsonConverter ? "json" : "avro") + + "/dbserver1.inventory.products"; + final String sourceName = "test-source-debezium-mysql" + (isJsonConverter ? "json" : "avro") + + "-" + functionRuntimeType + "-" + randomName(8); + + // This is the binlog count that contained in mysql container. + final int numMessages = 47; + + if (pulsarCluster == null) { + super.setupCluster(); + super.setupFunctionWorkers(); + } + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); + initNamespace(admin); + + try { + SchemaInfo lastSchemaInfo = admin.schemas().getSchemaInfo(consumeTopicName); + log.info("lastSchemaInfo: {}", lastSchemaInfo == null ? "null" : lastSchemaInfo.toString()); + } catch (Exception e) { + log.warn("failed to get schemaInfo for topic: {}, exceptions message: {}", + consumeTopicName, e.getMessage()); + } + + admin.topics().createNonPartitionedTopic(outputTopicName); + + @Cleanup + DebeziumMySqlSourceTester sourceTester = new DebeziumMySqlSourceTester(pulsarCluster, converterClassName); + sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); + + // setup debezium mysql server + DebeziumMySQLContainer mySQLContainer = new DebeziumMySQLContainer(pulsarCluster.getClusterName()); + sourceTester.setServiceContainer(mySQLContainer); + + PulsarIODebeziumSourceRunner runner = new PulsarIODebeziumSourceRunner(pulsarCluster, functionRuntimeType.toString(), + converterClassName, tenant, namespace, sourceName, outputTopicName, numMessages, jsonWithEnvelope, + consumeTopicName, client); + + runner.testSource(sourceTester); + } + + private void testDebeziumPostgreSqlConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { + + final String tenant = TopicName.PUBLIC_TENANT; + final String namespace = TopicName.DEFAULT_NAMESPACE; + final String outputTopicName = "debe-output-topic-name-" + testId.getAndIncrement(); + final String consumeTopicName = "debezium/postgresql/dbserver1.inventory.products"; + final String sourceName = "test-source-debezium-postgersql-" + functionRuntimeType + "-" + randomName(8); + + + // This is the binlog count that contained in postgresql container. + final int numMessages = 26; + + if (pulsarCluster == null) { + super.setupCluster(); + super.setupFunctionWorkers(); + } + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); + initNamespace(admin); + + admin.topics().createNonPartitionedTopic(consumeTopicName); + admin.topics().createNonPartitionedTopic(outputTopicName); + + @Cleanup + DebeziumPostgreSqlSourceTester sourceTester = new DebeziumPostgreSqlSourceTester(pulsarCluster); + sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); + + // setup debezium postgresql server + DebeziumPostgreSqlContainer postgreSqlContainer = new DebeziumPostgreSqlContainer(pulsarCluster.getClusterName()); + sourceTester.setServiceContainer(postgreSqlContainer); + + PulsarIODebeziumSourceRunner runner = new PulsarIODebeziumSourceRunner(pulsarCluster, functionRuntimeType.toString(), + converterClassName, tenant, namespace, sourceName, outputTopicName, numMessages, jsonWithEnvelope, + consumeTopicName, client); + + runner.testSource(sourceTester); + } + + private void testDebeziumMongoDbConnect(String converterClassName, boolean jsonWithEnvelope) throws Exception { + + final String tenant = TopicName.PUBLIC_TENANT; + final String namespace = TopicName.DEFAULT_NAMESPACE; + final String outputTopicName = "debe-output-topic-name"; + final String consumeTopicName = "debezium/mongodb/dbserver1.inventory.products"; + final String sourceName = "test-source-connector-" + + functionRuntimeType + "-name-" + randomName(8); + + // This is the binlog count that contained in mongodb container. + final int numMessages = 17; + + if (pulsarCluster == null) { + super.setupCluster(); + super.setupFunctionWorkers(); + } + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) + .build(); + + @Cleanup + PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarCluster.getHttpServiceUrl()).build(); + initNamespace(admin); + + admin.topics().createNonPartitionedTopic(consumeTopicName); + admin.topics().createNonPartitionedTopic(outputTopicName); + + @Cleanup + DebeziumMongoDbSourceTester sourceTester = new DebeziumMongoDbSourceTester(pulsarCluster); + sourceTester.getSourceConfig().put("json-with-envelope", jsonWithEnvelope); + + // setup debezium mongodb server + DebeziumMongoDbContainer mongoDbContainer = new DebeziumMongoDbContainer(pulsarCluster.getClusterName()); + sourceTester.setServiceContainer(mongoDbContainer); + + PulsarIODebeziumSourceRunner runner = new PulsarIODebeziumSourceRunner(pulsarCluster, functionRuntimeType.toString(), + converterClassName, tenant, namespace, sourceName, outputTopicName, numMessages, jsonWithEnvelope, + consumeTopicName, client); + + runner.testSource(sourceTester); + } + + protected void initNamespace(PulsarAdmin admin) { + log.info("[initNamespace] start."); + try { + admin.tenants().createTenant("debezium", new TenantInfo(Sets.newHashSet(), + Sets.newHashSet(pulsarCluster.getClusterName()))); + admin.namespaces().createNamespace("debezium/mysql-json"); + admin.namespaces().createNamespace("debezium/mysql-avro"); + admin.namespaces().createNamespace("debezium/mongodb"); + admin.namespaces().createNamespace("debezium/postgresql"); + } catch (Exception e) { + log.info("[initNamespace] msg: {}", e.getMessage()); + } + log.info("[initNamespace] finish."); + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java new file mode 100644 index 0000000000000..2766319d0b1de --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.io.sources.debezium; + +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.tests.integration.io.sources.PulsarIOSourceRunner; +import org.apache.pulsar.tests.integration.io.sources.SourceTester; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.testcontainers.containers.GenericContainer; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import net.jodah.failsafe.Failsafe; + +@Slf4j +public class PulsarIODebeziumSourceRunner extends PulsarIOSourceRunner { + + private String converterClassName; + private String tenant; + private String namespace; + private String sourceName; + private String outputTopicName; + private String consumeTopicName; + private int numMessages; + private boolean jsonWithEnvelope; + private PulsarClient client; + + public PulsarIODebeziumSourceRunner(PulsarCluster cluster, String functionRuntimeType, String converterClassName, + String tenant, String ns, String sourceName, String outputTopic, int numMessages, boolean jsonWithEnvelope, + String consumeTopicName, PulsarClient client) { + super(cluster, functionRuntimeType); + this.converterClassName = converterClassName; + this.tenant = tenant; + this.namespace = ns; + this.sourceName = sourceName; + this.outputTopicName = outputTopic; + this.numMessages = numMessages; + this.jsonWithEnvelope = jsonWithEnvelope; + this.consumeTopicName = consumeTopicName; + this.client = client; + } + + @Override + @SuppressWarnings({ "rawtypes", "unchecked" }) + public void testSource(SourceTester sourceTester) throws Exception { + // prepare the testing environment for source + prepareSource(sourceTester); + + // submit the source connector + submitSourceConnector(sourceTester, tenant, namespace, sourceName, outputTopicName); + + // get source info + getSourceInfoSuccess(sourceTester, tenant, namespace, sourceName); + + // get source status + Failsafe.with(statusRetryPolicy).run(() -> getSourceStatus(tenant, namespace, sourceName)); + + // wait for source to process messages + Failsafe.with(statusRetryPolicy).run(() -> + waitForProcessingSourceMessages(tenant, namespace, sourceName, numMessages)); + + @Cleanup + Consumer consumer = client.newConsumer(getSchema(jsonWithEnvelope)) + .topic(consumeTopicName) + .subscriptionName("debezium-source-tester") + .subscriptionType(SubscriptionType.Exclusive) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + log.info("[debezium mysql test] create consumer finish. converterName: {}", converterClassName); + + // validate the source result + sourceTester.validateSourceResult(consumer, 9, null, converterClassName); + + // prepare insert event + sourceTester.prepareInsertEvent(); + + // validate the source insert event + sourceTester.validateSourceResult(consumer, 1, SourceTester.INSERT, converterClassName); + + // prepare update event + sourceTester.prepareUpdateEvent(); + + // validate the source update event + sourceTester.validateSourceResult(consumer, 1, SourceTester.UPDATE, converterClassName); + + // prepare delete event + sourceTester.prepareDeleteEvent(); + + // validate the source delete event + sourceTester.validateSourceResult(consumer, 1, SourceTester.DELETE, converterClassName); + + // delete the source + deleteSource(tenant, namespace, sourceName); + + // get source info (source should be deleted) + getSourceInfoNotFound(tenant, namespace, sourceName); + } +} diff --git a/tests/integration/src/test/resources/pulsar-io-sources.xml b/tests/integration/src/test/resources/pulsar-io-sources.xml index 4b034febc72b3..a5afc5d8d0322 100644 --- a/tests/integration/src/test/resources/pulsar-io-sources.xml +++ b/tests/integration/src/test/resources/pulsar-io-sources.xml @@ -22,7 +22,7 @@ - + \ No newline at end of file From 8e449a9babf75ed81c37ae9d3a9f3fbeaa816827 Mon Sep 17 00:00:00 2001 From: David Kjerrumgaard Date: Thu, 6 May 2021 14:51:37 -0700 Subject: [PATCH 6/6] Fixed bronled integration tests --- .../tests/integration/functions/PulsarFunctionsTestBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java index 75a3ae07190e2..dbe3994ceec8c 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java @@ -100,7 +100,7 @@ protected PulsarFunctionsTestBase(FunctionRuntimeType functionRuntimeType) { this.functionRuntimeType = functionRuntimeType; } - @BeforeClass + @BeforeClass(alwaysRun = true) public void setupFunctionWorkers() { final int numFunctionWorkers = 2; log.info("Setting up {} function workers : function runtime type = {}",