From b95ac858ece5495ef41a22778cdf3d7fc7f9dbce Mon Sep 17 00:00:00 2001 From: Pierre Zemb Date: Thu, 19 Dec 2019 17:30:21 +0100 Subject: [PATCH 01/13] Initial commit for integration tests using Sarama --- integrations/golang/sarama/Dockerfile | 9 ++ integrations/golang/sarama/go.mod | 5 + integrations/golang/sarama/go.sum | 52 ++++++++ integrations/golang/sarama/main.go | 125 ++++++++++++++++++ pom.xml | 6 + .../kop/MockKafkaServiceBaseTest.java | 1 - .../kop/integrations/GolangSaramaTest.java | 105 +++++++++++++++ 7 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 integrations/golang/sarama/Dockerfile create mode 100644 integrations/golang/sarama/go.mod create mode 100644 integrations/golang/sarama/go.sum create mode 100644 integrations/golang/sarama/main.go create mode 100644 src/test/java/io/streamnative/kop/integrations/GolangSaramaTest.java diff --git a/integrations/golang/sarama/Dockerfile b/integrations/golang/sarama/Dockerfile new file mode 100644 index 0000000000..fb4b3b85c9 --- /dev/null +++ b/integrations/golang/sarama/Dockerfile @@ -0,0 +1,9 @@ +FROM golang:1.13.4-stretch + +WORKDIR /go/src/app +COPY . . + +RUN go get -d -v ./... +RUN go install -v ./... + +CMD [ "/go/bin/sarama-golang" ] \ No newline at end of file diff --git a/integrations/golang/sarama/go.mod b/integrations/golang/sarama/go.mod new file mode 100644 index 0000000000..c13ce190b7 --- /dev/null +++ b/integrations/golang/sarama/go.mod @@ -0,0 +1,5 @@ +module github.com/apache/pulsar/kop/integration/sarama-golang + +go 1.13 + +require github.com/Shopify/sarama v1.24.1 diff --git a/integrations/golang/sarama/go.sum b/integrations/golang/sarama/go.sum new file mode 100644 index 0000000000..f327335702 --- /dev/null +++ b/integrations/golang/sarama/go.sum @@ -0,0 +1,52 @@ +github.com/Shopify/sarama v1.24.1 h1:svn9vfN3R1Hz21WR2Gj0VW9ehaDGkiOS+VqlIcZOkMI= +github.com/Shopify/sarama v1.24.1/go.mod h1:fGP8eQ6PugKEI0iUETYYtnP6d1pH/bdDMTel1X5ajsU= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjLBdCp5PRlCFijNjvcYANOZXzCfXwCM= +github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/klauspost/compress v1.8.2 h1:Bx0qjetmNjdFXASH02NSAREKpiaDwkO1DRZ3dV2KCcs= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pierrec/lz4 v2.2.6+incompatible h1:6aCX4/YZ9v8q69hTyiR7dNLnTA3fgtKHVVW5BCd5Znw= +github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.2.3 h1:hHMV/yKPwMnJhPuPx7pH2Uw/3Qyf+thJYlisUc44010= +gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= diff --git a/integrations/golang/sarama/main.go b/integrations/golang/sarama/main.go new file mode 100644 index 0000000000..5e6da7052e --- /dev/null +++ b/integrations/golang/sarama/main.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "strconv" + "sync" + + "github.com/Shopify/sarama" +) + +type exampleConsumerGroupHandler struct { + counter int + limit int + wg *sync.WaitGroup +} + +func (exampleConsumerGroupHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } +func (exampleConsumerGroupHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil } +func (h exampleConsumerGroupHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + for msg := range claim.Messages() { + fmt.Printf("Message topic:%q partition:%d offset:%d\n", msg.Topic, msg.Partition, msg.Offset) + sess.MarkMessage(msg, "") + h.counter++ + fmt.Printf("received msg %d/%d\n", h.counter, h.limit) + if h.counter == h.limit { + fmt.Println("limit reached, exiting") + h.wg.Done() + } + } + return nil +} + +func main() { + + nbrMessages, err := strconv.Atoi(getEnv("KOP_NBR_MESSAGES", "10")) + if err != nil { + panic(err) + } + limit, err := strconv.Atoi(getEnv("KOP_EXPECT_MESSAGES", "10")) + if err != nil { + panic(err) + } + + // Init config, specify appropriate version + config := sarama.NewConfig() + config.Version = sarama.V2_0_0_0 + config.Metadata.Retry.Max = 0 + config.Consumer.Return.Errors = true + config.Producer.Return.Successes = true + brokers := []string{getEnv("KOP_BROKER", "localhost:9092")} + topic := getEnv("KOP_TOPIC", "my-sarama-topic") + topics := []string{topic} + sarama.Logger = log.New(os.Stdout, "", log.Ltime) + + fmt.Println("connecting to", brokers) + + // Start with a client + client, err := sarama.NewClient(brokers, config) + if err != nil { + panic(err) + } + defer func() { _ = client.Close() }() + + var waitgroup sync.WaitGroup + waitgroup.Add(1) + + // Start a new consumer group + group, err := sarama.NewConsumerGroupFromClient("sarama-consumer", client) + if err != nil { + panic(err) + } + defer func() { _ = group.Close() }() + + fmt.Println("ready to consume") + + // Iterate over consumer sessions. + ctx := context.Background() + go func() { + handler := exampleConsumerGroupHandler{counter: 0, limit: limit, wg: &waitgroup} + + err := group.Consume(ctx, topics, handler) + if err != nil { + panic(err) + } + + }() + + syncProducer, err := sarama.NewSyncProducerFromClient(client) + if err != nil { + panic(err) + } + defer func() { _ = syncProducer.Close() }() + + fmt.Println("starting to produce") + + for i := 0; i < nbrMessages; i++ { + msg := &sarama.ProducerMessage{ + Topic: topic, + Value: sarama.StringEncoder("hello from sarama"), + Metadata: "test", + } + + fmt.Println("send a message") + + _, _, err := syncProducer.SendMessage(msg) + if err != nil { + panic(err) + } + } + fmt.Printf("produced %d messages, waiting for consumption...\n", nbrMessages) + + waitgroup.Wait() + fmt.Println("exiting normally") +} + +func getEnv(key, fallback string) string { + value, exists := os.LookupEnv(key) + if !exists { + value = fallback + } + return value +} diff --git a/pom.xml b/pom.xml index 8f73984412..95979da6d9 100644 --- a/pom.xml +++ b/pom.xml @@ -212,6 +212,12 @@ test + + org.testcontainers + testcontainers + 1.12.4 + test + diff --git a/src/test/java/io/streamnative/kop/MockKafkaServiceBaseTest.java b/src/test/java/io/streamnative/kop/MockKafkaServiceBaseTest.java index 652577db88..e07982f827 100644 --- a/src/test/java/io/streamnative/kop/MockKafkaServiceBaseTest.java +++ b/src/test/java/io/streamnative/kop/MockKafkaServiceBaseTest.java @@ -105,7 +105,6 @@ protected void resetConfig() { this.conf.setAdvertisedAddress("localhost"); this.conf.setWebServicePort(Optional.ofNullable(brokerWebservicePort)); this.conf.setClusterName(configClusterName); - this.conf.setAdvertisedAddress("localhost"); this.conf.setListeners( PLAINTEXT_PREFIX + "localhost:" + kafkaBrokerPort + "," + SSL_PREFIX + "localhost:" + kafkaBrokerPortTls); diff --git a/src/test/java/io/streamnative/kop/integrations/GolangSaramaTest.java b/src/test/java/io/streamnative/kop/integrations/GolangSaramaTest.java new file mode 100644 index 0000000000..f401c883c6 --- /dev/null +++ b/src/test/java/io/streamnative/kop/integrations/GolangSaramaTest.java @@ -0,0 +1,105 @@ +package io.streamnative.kop.integrations; + +import com.google.common.collect.Sets; +import io.streamnative.kop.MockKafkaServiceBaseTest; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.junit.AfterClass; +import org.junit.ClassRule; +import org.testcontainers.Testcontainers; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.output.WaitingConsumer; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; + +import static org.testcontainers.containers.output.OutputFrame.OutputType.STDOUT; +import static org.testng.AssertJUnit.assertFalse; + +@Slf4j +public class GolangSaramaTest extends MockKafkaServiceBaseTest { + + @ClassRule + public GenericContainer container = new GenericContainer<>( + new ImageFromDockerfile().withFileFromPath(".", Paths.get("integrations/golang/sarama"))); + + @BeforeClass + @Override + protected void setup() throws Exception { + + super.resetConfig(); + super.internalSetup(); + + if (!admin.clusters().getClusters().contains(configClusterName)) { + // so that clients can test short names + admin.clusters().createCluster(configClusterName, + new ClusterData("http://127.0.0.1:" + brokerWebservicePort)); + } else { + admin.clusters().updateCluster(configClusterName, + new ClusterData("http://127.0.0.1:" + brokerWebservicePort)); + } + + if (!admin.tenants().getTenants().contains("public")) { + admin.tenants().createTenant("public", + new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); + } else { + admin.tenants().updateTenant("public", + new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); + } + if (!admin.namespaces().getNamespaces("public").contains("public/default")) { + admin.namespaces().createNamespace("public/default"); + admin.namespaces().setNamespaceReplicationClusters("public/default", Sets.newHashSet("test")); + admin.namespaces().setRetention("public/default", + new RetentionPolicies(60, 1000)); + } + if (!admin.namespaces().getNamespaces("public").contains("public/__kafka")) { + admin.namespaces().createNamespace("public/__kafka"); + admin.namespaces().setNamespaceReplicationClusters("public/__kafka", Sets.newHashSet("test")); + admin.namespaces().setRetention("public/__kafka", + new RetentionPolicies(-1, -1)); + } + getAdmin().topics().createPartitionedTopic("persistent://public/default/my-sarama-topic", 1); + + container + .withEnv("KOP_BROKER", "localhost:" + super.kafkaBrokerPort) + .withNetworkMode("host"); + Testcontainers.exposeHostPorts(ImmutableMap.of(super.kafkaBrokerPort, super.kafkaBrokerPort)); + } + + @Test(timeOut = 120_000) + void simpleProduceAndConsume() throws Exception { + System.out.println("building container"); + container.start(); + System.out.println("container started"); + + Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(log); + container.followOutput(logConsumer); + WaitingConsumer consumer = new WaitingConsumer(); + container.followOutput(consumer, STDOUT); + + consumer.waitUntil(frame -> + frame.getUtf8String().contains("reached"), 30, TimeUnit.SECONDS); + System.out.println("after reached"); + + checkForSaramaErrors(container.getLogs()); + } + + @Override + @AfterClass + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + private void checkForSaramaErrors(String logs) { + assertFalse(logs.contains("no available broker to send metadata request to")); + assertFalse(logs.contains("panic")); + assertFalse(logs.contains("correlation ID didn't match")); + } +} From f61b388625601994f9a326e21cf4f1f18bf2011b Mon Sep 17 00:00:00 2001 From: Pierre Zemb Date: Mon, 30 Dec 2019 10:16:13 +0100 Subject: [PATCH 02/13] Split sarama integration tests into two applications --- integrations/golang/sarama/Dockerfile | 2 +- integrations/golang/sarama/main.go | 79 +++++++++------ .../streamnative/kop/KafkaRequestHandler.java | 5 +- .../kop/integrations/GolangSaramaTest.java | 95 +++++++++++++++---- 4 files changed, 127 insertions(+), 54 deletions(-) diff --git a/integrations/golang/sarama/Dockerfile b/integrations/golang/sarama/Dockerfile index fb4b3b85c9..3d97f9d004 100644 --- a/integrations/golang/sarama/Dockerfile +++ b/integrations/golang/sarama/Dockerfile @@ -6,4 +6,4 @@ COPY . . RUN go get -d -v ./... RUN go install -v ./... -CMD [ "/go/bin/sarama-golang" ] \ No newline at end of file +CMD sh -c '/go/bin/sarama-golang; echo "ExitCode=$?"' \ No newline at end of file diff --git a/integrations/golang/sarama/main.go b/integrations/golang/sarama/main.go index 5e6da7052e..03f8dbc44c 100644 --- a/integrations/golang/sarama/main.go +++ b/integrations/golang/sarama/main.go @@ -28,6 +28,7 @@ func (h exampleConsumerGroupHandler) ConsumeClaim(sess sarama.ConsumerGroupSessi if h.counter == h.limit { fmt.Println("limit reached, exiting") h.wg.Done() + return nil } } return nil @@ -35,6 +36,8 @@ func (h exampleConsumerGroupHandler) ConsumeClaim(sess sarama.ConsumerGroupSessi func main() { + fmt.Println("starting...") + nbrMessages, err := strconv.Atoi(getEnv("KOP_NBR_MESSAGES", "10")) if err != nil { panic(err) @@ -44,11 +47,22 @@ func main() { panic(err) } + shouldProduce, err := strconv.ParseBool(getEnv("KOP_PRODUCE", "false")) + if err != nil { + panic(err) + } + + shouldConsume, err := strconv.ParseBool(getEnv("KOP_CONSUME", "false")) + if err != nil { + panic(err) + } + // Init config, specify appropriate version config := sarama.NewConfig() config.Version = sarama.V2_0_0_0 config.Metadata.Retry.Max = 0 config.Consumer.Return.Errors = true + config.Consumer.Offsets.Initial = sarama.OffsetOldest config.Producer.Return.Successes = true brokers := []string{getEnv("KOP_BROKER", "localhost:9092")} topic := getEnv("KOP_TOPIC", "my-sarama-topic") @@ -65,55 +79,58 @@ func main() { defer func() { _ = client.Close() }() var waitgroup sync.WaitGroup - waitgroup.Add(1) + if shouldConsume { + waitgroup.Add(1) - // Start a new consumer group - group, err := sarama.NewConsumerGroupFromClient("sarama-consumer", client) - if err != nil { - panic(err) - } - defer func() { _ = group.Close() }() + // Start a new consumer group + group, err := sarama.NewConsumerGroupFromClient("sarama-consumer", client) + if err != nil { + panic(err) + } + defer func() { _ = group.Close() }() - fmt.Println("ready to consume") + fmt.Println("ready to consume") - // Iterate over consumer sessions. - ctx := context.Background() - go func() { + // Iterate over consumer sessions. + ctx := context.Background() handler := exampleConsumerGroupHandler{counter: 0, limit: limit, wg: &waitgroup} - err := group.Consume(ctx, topics, handler) + err = group.Consume(ctx, topics, handler) if err != nil { panic(err) } - }() - - syncProducer, err := sarama.NewSyncProducerFromClient(client) - if err != nil { - panic(err) } - defer func() { _ = syncProducer.Close() }() - fmt.Println("starting to produce") - - for i := 0; i < nbrMessages; i++ { - msg := &sarama.ProducerMessage{ - Topic: topic, - Value: sarama.StringEncoder("hello from sarama"), - Metadata: "test", + if shouldProduce { + syncProducer, err := sarama.NewSyncProducerFromClient(client) + if err != nil { + panic(err) } + defer func() { _ = syncProducer.Close() }() - fmt.Println("send a message") + fmt.Println("starting to produce") - _, _, err := syncProducer.SendMessage(msg) - if err != nil { - panic(err) + for i := 0; i < nbrMessages; i++ { + msg := &sarama.ProducerMessage{ + Topic: topic, + Value: sarama.StringEncoder("hello from sarama"), + Metadata: "test", + } + + fmt.Println("send a message") + + _, _, err := syncProducer.SendMessage(msg) + if err != nil { + panic(err) + } } - } - fmt.Printf("produced %d messages, waiting for consumption...\n", nbrMessages) + fmt.Printf("produced all messages successfully (%d) \n", nbrMessages) + } waitgroup.Wait() fmt.Println("exiting normally") + } func getEnv(key, fallback string) string { diff --git a/src/main/java/io/streamnative/kop/KafkaRequestHandler.java b/src/main/java/io/streamnative/kop/KafkaRequestHandler.java index ef31ab44b3..5997e2062d 100644 --- a/src/main/java/io/streamnative/kop/KafkaRequestHandler.java +++ b/src/main/java/io/streamnative/kop/KafkaRequestHandler.java @@ -388,7 +388,10 @@ protected CompletableFuture handleTopicMetadataRequest(Kafka allTopicMetadata.add( new TopicMetadata( Errors.NONE, - TopicName.get(topic).getLocalName(), + // we should answer with the right name, either local of full-name, + // depending on what was asked + topic.startsWith("persistent://") ? + TopicName.get(topic).toString(): TopicName.get(topic).getLocalName(), false, partitionMetadatas)); diff --git a/src/test/java/io/streamnative/kop/integrations/GolangSaramaTest.java b/src/test/java/io/streamnative/kop/integrations/GolangSaramaTest.java index f401c883c6..58c655a381 100644 --- a/src/test/java/io/streamnative/kop/integrations/GolangSaramaTest.java +++ b/src/test/java/io/streamnative/kop/integrations/GolangSaramaTest.java @@ -7,14 +7,15 @@ import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TenantInfo; import org.junit.AfterClass; -import org.junit.ClassRule; import org.testcontainers.Testcontainers; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.output.WaitingConsumer; +import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.images.builder.ImageFromDockerfile; import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.nio.file.Paths; @@ -22,13 +23,20 @@ import static org.testcontainers.containers.output.OutputFrame.OutputType.STDOUT; import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertTrue; @Slf4j public class GolangSaramaTest extends MockKafkaServiceBaseTest { - @ClassRule - public GenericContainer container = new GenericContainer<>( - new ImageFromDockerfile().withFileFromPath(".", Paths.get("integrations/golang/sarama"))); + private static final String SHORT_TOPIC_NAME = "my-sarama-topic"; + private static final String LONG_TOPIC_NAME = "persistent://public/default/my-sarama-topic-full-name"; + + @DataProvider + public static Object[][] topics() { + return new Object[][]{ + {SHORT_TOPIC_NAME}, {LONG_TOPIC_NAME} + }; + } @BeforeClass @Override @@ -65,30 +73,55 @@ protected void setup() throws Exception { admin.namespaces().setRetention("public/__kafka", new RetentionPolicies(-1, -1)); } - getAdmin().topics().createPartitionedTopic("persistent://public/default/my-sarama-topic", 1); + getAdmin().topics().createPartitionedTopic("persistent://public/default/" + SHORT_TOPIC_NAME, 1); + getAdmin().topics().createPartitionedTopic(LONG_TOPIC_NAME, 1); + - container - .withEnv("KOP_BROKER", "localhost:" + super.kafkaBrokerPort) - .withNetworkMode("host"); Testcontainers.exposeHostPorts(ImmutableMap.of(super.kafkaBrokerPort, super.kafkaBrokerPort)); } - @Test(timeOut = 120_000) - void simpleProduceAndConsume() throws Exception { - System.out.println("building container"); - container.start(); - System.out.println("container started"); + @Test(timeOut = 60_000, dataProvider = "topics") + void simpleProduceAndConsume(String topic) throws Exception { - Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(log); - container.followOutput(logConsumer); - WaitingConsumer consumer = new WaitingConsumer(); - container.followOutput(consumer, STDOUT); + GenericContainer producer = new GenericContainer<>( + new ImageFromDockerfile().withFileFromPath(".", Paths.get("integrations/golang/sarama"))); + + GenericContainer consumer = new GenericContainer<>( + new ImageFromDockerfile().withFileFromPath(".", Paths.get("integrations/golang/sarama"))); + + producer + .withEnv("KOP_BROKER", "localhost:" + super.kafkaBrokerPort) + .withEnv("KOP_PRODUCE", "true") + .withEnv("KOP_TOPIC", topic) + .waitingFor( + Wait.forLogMessage("starting to produce\\n", 1) + ) + .withNetworkMode("host"); + + consumer + .withEnv("KOP_BROKER", "localhost:" + super.kafkaBrokerPort) + .withEnv("KOP_TOPIC", topic) + .withEnv("KOP_CONSUME", "true") + .waitingFor( + Wait.forLogMessage("ready to consume\\n", 1) + ) + .withNetworkMode("host"); + + producer.start(); + WaitingConsumer consumerWaitingConsumer = createLogFollower(producer); + System.out.println("producer started"); + consumer.start(); + WaitingConsumer producerWaitingConsumer = createLogFollower(consumer); + System.out.println("consumer started"); - consumer.waitUntil(frame -> - frame.getUtf8String().contains("reached"), 30, TimeUnit.SECONDS); - System.out.println("after reached"); - checkForSaramaErrors(container.getLogs()); + producerWaitingConsumer.waitUntil(frame -> + frame.getUtf8String().contains("ExitCode"), 30, TimeUnit.SECONDS); + consumerWaitingConsumer.waitUntil(frame -> + frame.getUtf8String().contains("ExitCode"), 30, TimeUnit.SECONDS); + + checkForSaramaErrors(producer.getLogs()); + checkForSaramaErrors(consumer.getLogs()); } @Override @@ -97,9 +130,29 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + + private WaitingConsumer createLogFollower(GenericContainer container) { + Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(log); + container.followOutput(logConsumer); + WaitingConsumer waitingConsumer = new WaitingConsumer(); + container.followOutput(waitingConsumer, STDOUT); + + return waitingConsumer; + } + private void checkForSaramaErrors(String logs) { assertFalse(logs.contains("no available broker to send metadata request to")); assertFalse(logs.contains("panic")); assertFalse(logs.contains("correlation ID didn't match")); + + if (logs.contains("starting to produce")) { + assertTrue(logs.contains("produced all messages successfully")); + } + + if (logs.contains("ready to consume")) { + assertTrue(logs.contains("received msg")); + assertTrue(logs.contains("limit reached, exiting")); + } + assertTrue(logs.contains("ExitCode=0")); } } From bd50ea67782a4093c41f62e40e11027d6704f2c9 Mon Sep 17 00:00:00 2001 From: Pierre Zemb Date: Mon, 30 Dec 2019 14:33:23 +0100 Subject: [PATCH 03/13] refactor integration tests --- .../sarama => golang-sarama}/Dockerfile | 0 .../{golang/sarama => golang-sarama}/go.mod | 0 .../{golang/sarama => golang-sarama}/go.sum | 0 .../{golang/sarama => golang-sarama}/main.go | 4 +- ...amaTest.java => KafkaIntegrationTest.java} | 48 ++++++++----------- 5 files changed, 21 insertions(+), 31 deletions(-) rename integrations/{golang/sarama => golang-sarama}/Dockerfile (100%) rename integrations/{golang/sarama => golang-sarama}/go.mod (100%) rename integrations/{golang/sarama => golang-sarama}/go.sum (100%) rename integrations/{golang/sarama => golang-sarama}/main.go (98%) rename src/test/java/io/streamnative/kop/{integrations/GolangSaramaTest.java => KafkaIntegrationTest.java} (82%) diff --git a/integrations/golang/sarama/Dockerfile b/integrations/golang-sarama/Dockerfile similarity index 100% rename from integrations/golang/sarama/Dockerfile rename to integrations/golang-sarama/Dockerfile diff --git a/integrations/golang/sarama/go.mod b/integrations/golang-sarama/go.mod similarity index 100% rename from integrations/golang/sarama/go.mod rename to integrations/golang-sarama/go.mod diff --git a/integrations/golang/sarama/go.sum b/integrations/golang-sarama/go.sum similarity index 100% rename from integrations/golang/sarama/go.sum rename to integrations/golang-sarama/go.sum diff --git a/integrations/golang/sarama/main.go b/integrations/golang-sarama/main.go similarity index 98% rename from integrations/golang/sarama/main.go rename to integrations/golang-sarama/main.go index 03f8dbc44c..aa9f4c6f64 100644 --- a/integrations/golang/sarama/main.go +++ b/integrations/golang-sarama/main.go @@ -36,8 +36,6 @@ func (h exampleConsumerGroupHandler) ConsumeClaim(sess sarama.ConsumerGroupSessi func main() { - fmt.Println("starting...") - nbrMessages, err := strconv.Atoi(getEnv("KOP_NBR_MESSAGES", "10")) if err != nil { panic(err) @@ -89,7 +87,7 @@ func main() { } defer func() { _ = group.Close() }() - fmt.Println("ready to consume") + fmt.Println("starting to consume") // Iterate over consumer sessions. ctx := context.Background() diff --git a/src/test/java/io/streamnative/kop/integrations/GolangSaramaTest.java b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java similarity index 82% rename from src/test/java/io/streamnative/kop/integrations/GolangSaramaTest.java rename to src/test/java/io/streamnative/kop/KafkaIntegrationTest.java index 58c655a381..ca3dae120e 100644 --- a/src/test/java/io/streamnative/kop/integrations/GolangSaramaTest.java +++ b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java @@ -1,4 +1,4 @@ -package io.streamnative.kop.integrations; +package io.streamnative.kop; import com.google.common.collect.Sets; import io.streamnative.kop.MockKafkaServiceBaseTest; @@ -26,15 +26,13 @@ import static org.testng.AssertJUnit.assertTrue; @Slf4j -public class GolangSaramaTest extends MockKafkaServiceBaseTest { - - private static final String SHORT_TOPIC_NAME = "my-sarama-topic"; - private static final String LONG_TOPIC_NAME = "persistent://public/default/my-sarama-topic-full-name"; +public class KafkaIntegrationTest extends MockKafkaServiceBaseTest { @DataProvider - public static Object[][] topics() { - return new Object[][]{ - {SHORT_TOPIC_NAME}, {LONG_TOPIC_NAME} + public static Object[][] integrations() { + return new Object[][] { + {"golang-sarama", "my-sarama-topic"}, + {"golang-sarama", "persistent://public/default/my-sarama-topic-full-name"}, }; } @@ -73,38 +71,32 @@ protected void setup() throws Exception { admin.namespaces().setRetention("public/__kafka", new RetentionPolicies(-1, -1)); } - getAdmin().topics().createPartitionedTopic("persistent://public/default/" + SHORT_TOPIC_NAME, 1); - getAdmin().topics().createPartitionedTopic(LONG_TOPIC_NAME, 1); - - Testcontainers.exposeHostPorts(ImmutableMap.of(super.kafkaBrokerPort, super.kafkaBrokerPort)); } - @Test(timeOut = 60_000, dataProvider = "topics") - void simpleProduceAndConsume(String topic) throws Exception { - - GenericContainer producer = new GenericContainer<>( - new ImageFromDockerfile().withFileFromPath(".", Paths.get("integrations/golang/sarama"))); + @Test(timeOut = 60_000, dataProvider = "integrations") + void simpleProduceAndConsume(String integration, String topic) throws Exception { - GenericContainer consumer = new GenericContainer<>( - new ImageFromDockerfile().withFileFromPath(".", Paths.get("integrations/golang/sarama"))); + getAdmin().topics().createPartitionedTopic(topic, 1); - producer + GenericContainer producer = new GenericContainer<>( + new ImageFromDockerfile().withFileFromPath(".", Paths.get("integrations/" + integration))) .withEnv("KOP_BROKER", "localhost:" + super.kafkaBrokerPort) .withEnv("KOP_PRODUCE", "true") .withEnv("KOP_TOPIC", topic) - .waitingFor( - Wait.forLogMessage("starting to produce\\n", 1) - ) + .withEnv("KOP_NBR_MESSAGES", "10") + .withEnv("KOP_EXPECT_MESSAGES", "10") + .waitingFor(Wait.forLogMessage("starting to produce\\n", 1)) .withNetworkMode("host"); - consumer + GenericContainer consumer = new GenericContainer<>( + new ImageFromDockerfile().withFileFromPath(".", Paths.get("integrations/" + integration))) .withEnv("KOP_BROKER", "localhost:" + super.kafkaBrokerPort) .withEnv("KOP_TOPIC", topic) .withEnv("KOP_CONSUME", "true") - .waitingFor( - Wait.forLogMessage("ready to consume\\n", 1) - ) + .withEnv("KOP_NBR_MESSAGES", "10") + .withEnv("KOP_EXPECT_MESSAGES", "10") + .waitingFor(Wait.forLogMessage("starting to consume\\n", 1)) .withNetworkMode("host"); producer.start(); @@ -149,7 +141,7 @@ private void checkForSaramaErrors(String logs) { assertTrue(logs.contains("produced all messages successfully")); } - if (logs.contains("ready to consume")) { + if (logs.contains("starting to consume")) { assertTrue(logs.contains("received msg")); assertTrue(logs.contains("limit reached, exiting")); } From 34d6855016aa8301f51a15857fb4889f7ee35cb2 Mon Sep 17 00:00:00 2001 From: Pierre Zemb Date: Mon, 30 Dec 2019 16:59:52 +0100 Subject: [PATCH 04/13] initial commit for integration test for confluent-golang client --- .../golang-confluent-kafka/Dockerfile | 12 ++ integrations/golang-confluent-kafka/go.mod | 5 + integrations/golang-confluent-kafka/go.sum | 2 + integrations/golang-confluent-kafka/main.go | 115 ++++++++++++++++++ .../streamnative/kop/KafkaRequestHandler.java | 27 ++-- .../kop/KafkaIntegrationTest.java | 12 +- 6 files changed, 159 insertions(+), 14 deletions(-) create mode 100644 integrations/golang-confluent-kafka/Dockerfile create mode 100644 integrations/golang-confluent-kafka/go.mod create mode 100644 integrations/golang-confluent-kafka/go.sum create mode 100644 integrations/golang-confluent-kafka/main.go diff --git a/integrations/golang-confluent-kafka/Dockerfile b/integrations/golang-confluent-kafka/Dockerfile new file mode 100644 index 0000000000..4698ac37a7 --- /dev/null +++ b/integrations/golang-confluent-kafka/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.13.4-alpine + +RUN apk -U add ca-certificates git bash build-base sudo pkgconf build-base +RUN git clone https://github.com/edenhill/librdkafka.git && cd librdkafka && ./configure --prefix /usr && make && make install + +WORKDIR /go/src/app +COPY . . + +RUN go get -d -v ./... +RUN go install -v ./... + +CMD sh -c '/go/bin/golang-confluent-kafka; echo "ExitCode=$?"' \ No newline at end of file diff --git a/integrations/golang-confluent-kafka/go.mod b/integrations/golang-confluent-kafka/go.mod new file mode 100644 index 0000000000..21f587579c --- /dev/null +++ b/integrations/golang-confluent-kafka/go.mod @@ -0,0 +1,5 @@ +module github.com/apache/pulsar/kop/integration/golang-confluent-kafka + +go 1.13 + +require gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0 diff --git a/integrations/golang-confluent-kafka/go.sum b/integrations/golang-confluent-kafka/go.sum new file mode 100644 index 0000000000..21841b7e75 --- /dev/null +++ b/integrations/golang-confluent-kafka/go.sum @@ -0,0 +1,2 @@ +gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0 h1:roy97m/3wj9/o8OuU3sZ5wildk30ep38k2x8nhNbKrI= +gopkg.in/confluentinc/confluent-kafka-go.v1 v1.1.0/go.mod h1:ZdI3yfYmdNSLQPNCpO1y00EHyWaHG5EnQEyL/ntAegY= diff --git a/integrations/golang-confluent-kafka/main.go b/integrations/golang-confluent-kafka/main.go new file mode 100644 index 0000000000..fa10b5f2b9 --- /dev/null +++ b/integrations/golang-confluent-kafka/main.go @@ -0,0 +1,115 @@ +package main + +import ( + "fmt" + "os" + "strconv" + "strings" + + "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" +) + +func main() { + nbrMessages, err := strconv.Atoi(getEnv("KOP_NBR_MESSAGES", "10")) + if err != nil { + panic(err) + } + limit, err := strconv.Atoi(getEnv("KOP_EXPECT_MESSAGES", "10")) + if err != nil { + panic(err) + } + + shouldProduce, err := strconv.ParseBool(getEnv("KOP_PRODUCE", "false")) + if err != nil { + panic(err) + } + + shouldConsume, err := strconv.ParseBool(getEnv("KOP_CONSUME", "false")) + if err != nil { + panic(err) + } + + brokers := []string{getEnv("KOP_BROKER", "localhost:9092")} + topic := getEnv("KOP_TOPIC", "my-confluent-go-topic") + topics := []string{topic} + + if shouldProduce { + + fmt.Println("starting to produce") + + p, err := kafka.NewProducer(&kafka.ConfigMap{"bootstrap.servers": getEnv("KOP_BROKER", "localhost:9092")}) + if err != nil { + panic(err) + } + defer p.Close() + + // Delivery report handler for produced messages + go func() { + for e := range p.Events() { + switch ev := e.(type) { + case *kafka.Message: + if ev.TopicPartition.Error != nil { + fmt.Printf("Delivery failed: %v\n", ev.TopicPartition) + } else { + fmt.Printf("Delivered message to %v\n", ev.TopicPartition) + } + } + } + }() + + for i := 0; i < nbrMessages; i++ { + p.Produce(&kafka.Message{ + TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny}, + Value: []byte("hello from confluent go"), + }, nil) + fmt.Println("send a message") + + } + fmt.Printf("produced all messages successfully (%d) \n", nbrMessages) + // Wait for message deliveries before shutting down + p.Flush(15 * 1000) + + } + + if shouldConsume { + fmt.Println("starting to consume") + + c, err := kafka.NewConsumer(&kafka.ConfigMap{ + "bootstrap.servers": strings.Join(brokers, ","), + "group.id": "myGroup", + "auto.offset.reset": "earliest", + "broker.version.fallback": "2.0.0", + "debug": "all", + }) + if err != nil { + panic(err) + } + + c.SubscribeTopics(topics, nil) + + counter := 0 + + for counter < limit { + msg, err := c.ReadMessage(-1) + if err == nil { + fmt.Printf("Message on %s: %s\n", msg.TopicPartition, string(msg.Value)) + counter++ + } else { + // The client will automatically try to recover from all errors. + fmt.Printf("Consumer error: %v (%v)\n", err, msg) + panic(err) + } + } + fmt.Println("limit reached, exiting") + } + + fmt.Println("exiting normally") +} + +func getEnv(key, fallback string) string { + value, exists := os.LookupEnv(key) + if !exists { + value = fallback + } + return value +} diff --git a/src/main/java/io/streamnative/kop/KafkaRequestHandler.java b/src/main/java/io/streamnative/kop/KafkaRequestHandler.java index 5997e2062d..2218f11f1a 100644 --- a/src/main/java/io/streamnative/kop/KafkaRequestHandler.java +++ b/src/main/java/io/streamnative/kop/KafkaRequestHandler.java @@ -40,13 +40,7 @@ import java.net.InetSocketAddress; import java.net.URI; import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -65,6 +59,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.RecordBatch; @@ -174,13 +169,29 @@ public KafkaRequestHandler(PulsarService pulsarService, } protected CompletableFuture handleApiVersionsRequest(KafkaHeaderAndRequest apiVersionRequest) { - AbstractResponse apiResponse = ApiVersionsResponse.defaultApiVersionsResponse(); + AbstractResponse apiResponse = overloadDefaultApiVersionsResponse(); CompletableFuture resultFuture = new CompletableFuture<>(); resultFuture.complete(ResponseAndRequest.of(apiResponse, apiVersionRequest)); return resultFuture; } + protected ApiVersionsResponse overloadDefaultApiVersionsResponse() { + List versionList = new ArrayList<>(); + for (ApiKeys apiKey : ApiKeys.values()) { + if (apiKey.minRequiredInterBrokerMagic <= RecordBatch.CURRENT_MAGIC_VALUE) { + switch (apiKey) { + case LIST_OFFSETS: + versionList.add(new ApiVersionsResponse.ApiVersion((short) 2, (short) 1, apiKey.latestVersion())); + break; + default: + versionList.add(new ApiVersionsResponse.ApiVersion(apiKey)); + } + } + } + return new ApiVersionsResponse(0, Errors.NONE, versionList); + } + protected CompletableFuture handleError(KafkaHeaderAndRequest kafkaHeaderAndRequest) { CompletableFuture resultFuture = new CompletableFuture<>(); String err = String.format("Kafka API (%s) Not supported by kop server.", diff --git a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java index ca3dae120e..2b1aa14af2 100644 --- a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java +++ b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java @@ -1,7 +1,6 @@ package io.streamnative.kop; import com.google.common.collect.Sets; -import io.streamnative.kop.MockKafkaServiceBaseTest; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.RetentionPolicies; @@ -31,8 +30,9 @@ public class KafkaIntegrationTest extends MockKafkaServiceBaseTest { @DataProvider public static Object[][] integrations() { return new Object[][] { - {"golang-sarama", "my-sarama-topic"}, - {"golang-sarama", "persistent://public/default/my-sarama-topic-full-name"}, + // {"golang-sarama", "my-sarama-topic"}, + // {"golang-sarama", "persistent://public/default/my-sarama-topic-full-name"}, + {"golang-confluent-kafka", "confluent-go"} }; } @@ -112,8 +112,8 @@ void simpleProduceAndConsume(String integration, String topic) throws Exception consumerWaitingConsumer.waitUntil(frame -> frame.getUtf8String().contains("ExitCode"), 30, TimeUnit.SECONDS); - checkForSaramaErrors(producer.getLogs()); - checkForSaramaErrors(consumer.getLogs()); + checkForErrorsInLogs(producer.getLogs()); + checkForErrorsInLogs(consumer.getLogs()); } @Override @@ -132,7 +132,7 @@ private WaitingConsumer createLogFollower(GenericContainer container) { return waitingConsumer; } - private void checkForSaramaErrors(String logs) { + private void checkForErrorsInLogs(String logs) { assertFalse(logs.contains("no available broker to send metadata request to")); assertFalse(logs.contains("panic")); assertFalse(logs.contains("correlation ID didn't match")); From 0c1c308ae481d2d17c643833c5c8cf5868cf49a0 Mon Sep 17 00:00:00 2001 From: Pierre Zemb Date: Tue, 31 Dec 2019 12:06:26 +0100 Subject: [PATCH 05/13] Initial commit for integration tests for rustlang --- integrations/rustlang-rdkafka/Cargo.lock | 185 ++++++++++++++++++ integrations/rustlang-rdkafka/Cargo.toml | 17 ++ integrations/rustlang-rdkafka/Dockerfile | 11 ++ integrations/rustlang-rdkafka/src/main.rs | 182 +++++++++++++++++ .../kop/KafkaIntegrationTest.java | 20 +- 5 files changed, 405 insertions(+), 10 deletions(-) create mode 100644 integrations/rustlang-rdkafka/Cargo.lock create mode 100644 integrations/rustlang-rdkafka/Cargo.toml create mode 100644 integrations/rustlang-rdkafka/Dockerfile create mode 100644 integrations/rustlang-rdkafka/src/main.rs diff --git a/integrations/rustlang-rdkafka/Cargo.lock b/integrations/rustlang-rdkafka/Cargo.lock new file mode 100644 index 0000000000..cbc41472ab --- /dev/null +++ b/integrations/rustlang-rdkafka/Cargo.lock @@ -0,0 +1,185 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "cc" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cmake" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "itoa" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.66" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libz-sys" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", + "vcpkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pkg-config" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quote" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rdkafka" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rdkafka-sys 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rdkafka-sys" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cmake 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustlang-rdkafka" +version = "0.1.0" +dependencies = [ + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rdkafka 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rdkafka-sys 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ryu" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde_derive" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_json" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "syn" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "vcpkg" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)" = "f52a465a666ca3d838ebbf08b241383421412fe7ebb463527bba275526d89f76" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum cmake 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "81fb25b677f8bf1eb325017cb6bb8452f87969db0fedb4f757b297bee78a7c62" +"checksum futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" +"checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f" +"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +"checksum libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe" +"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +"checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" +"checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27" +"checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +"checksum rdkafka 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9c39cf211c5e933622b600e892ae6fa7bc9e08cb56c4129424e07d91cc3c9088" +"checksum rdkafka-sys 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "78d4b56497b669efc00f1741cc39763e94f712f07817dfc3d96181fe41b535ea" +"checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" +"checksum serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)" = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" +"checksum serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)" = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" +"checksum serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)" = "48c575e0cc52bdd09b47f330f646cf59afc586e9c4e3ccd6fc1f625b8ea1dad7" +"checksum syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238" +"checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +"checksum vcpkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168" diff --git a/integrations/rustlang-rdkafka/Cargo.toml b/integrations/rustlang-rdkafka/Cargo.toml new file mode 100644 index 0000000000..3496eb2381 --- /dev/null +++ b/integrations/rustlang-rdkafka/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "rustlang-rdkafka" +version = "0.1.0" +authors = ["Pierre Zemb "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rdkafka = { version = "0.22", features = ["cmake-build"] } +rdkafka-sys = { version = "1.2.1", default-features = false } +futures = "0.1.21" +libc = "0.2.0" +log = "0.4.8" +serde = "1.0.0" +serde_derive = "1.0.0" +serde_json = "1.0.0" diff --git a/integrations/rustlang-rdkafka/Dockerfile b/integrations/rustlang-rdkafka/Dockerfile new file mode 100644 index 0000000000..d3384e6731 --- /dev/null +++ b/integrations/rustlang-rdkafka/Dockerfile @@ -0,0 +1,11 @@ +FROM rust:1.40-stretch + +RUN apt update +RUN apt install -y cmake libc6-dev + +WORKDIR /usr/src/rustlang-rdkafka +COPY . . + +RUN cargo install --path . + +CMD sh -c 'rustlang-rdkafka; echo "ExitCode=$?"' \ No newline at end of file diff --git a/integrations/rustlang-rdkafka/src/main.rs b/integrations/rustlang-rdkafka/src/main.rs new file mode 100644 index 0000000000..e364cd657c --- /dev/null +++ b/integrations/rustlang-rdkafka/src/main.rs @@ -0,0 +1,182 @@ +#[macro_use] +extern crate log; +extern crate futures; +extern crate rdkafka; + +use crate::futures::Future; +use futures::Stream; +use rdkafka::config::{ClientConfig, RDKafkaLogLevel}; +use rdkafka::consumer::stream_consumer::StreamConsumer; +use rdkafka::consumer::{CommitMode, Consumer, ConsumerContext, Rebalance}; +use rdkafka::error::KafkaResult; +use rdkafka::message::OwnedHeaders; +use rdkafka::message::{Headers, Message}; +use rdkafka::producer::{FutureProducer, FutureRecord}; +use rdkafka::ClientContext; +use rdkafka_sys; +use std::env; + +fn main() { + let brokers = env::var("KOP_BROKER").unwrap_or_else(|_| String::from("localhost:9092")); + let topic = env::var("KOP_TOPIC").unwrap_or_else(|_| String::from("rustlang")); + let limit: i8 = env::var("KOP_EXPECT_MESSAGES") + .unwrap_or_else(|_| String::from("10")) + .parse() + .unwrap_or(10); + let nbr_messages_produced: i8 = env::var("KOP_NBR_MESSAGES") + .unwrap_or_else(|_| String::from("10")) + .parse() + .unwrap_or(10); + let should_produce: bool = env::var("KOP_PRODUCE") + .unwrap_or_else(|_| String::from("false")) + .parse() + .unwrap_or(false); + let should_consume: bool = env::var("KOP_CONSUME") + .unwrap_or_else(|_| String::from("false")) + .parse() + .unwrap_or(false); + + println!( + "produce={}, consume={}, limit={}", + should_produce, should_consume, limit + ); + + if should_produce { + produce(brokers.as_ref(), topic.as_str(), nbr_messages_produced); + } + if should_consume { + println!("starting to consume"); + consume_and_print( + brokers.as_ref(), + "rustlang-librdkafka", + vec![topic.as_str()], + limit, + ); + } + println!("exiting normally"); +} +fn produce(brokers: &str, topic_name: &str, nbr_messages_produced: i8) { + println!("starting to produce"); + let producer: FutureProducer = ClientConfig::new() + .set("bootstrap.servers", brokers) + .set("message.timeout.ms", "5000") + .create() + .expect("Producer creation error"); + + // This loop is non blocking: all messages will be sent one after the other, without waiting + // for the results. + let futures = (0..nbr_messages_produced) + .map(|i| { + // The send operation on the topic returns a future, that will be completed once the + // result or failure from Kafka will be received. + producer + .send( + FutureRecord::to(topic_name) + .payload(&format!("Message {}", i)) + .key(&format!("Key {}", i)) + .headers(OwnedHeaders::new().add("header_key", "header_value")), + 0, + ) + .map(move |delivery_status| { + // This will be executed onw the result is received + println!("Delivery status for message {} received", i); + delivery_status + }) + }) + .collect::>(); + + // This loop will wait until all delivery statuses have been received received. + for future in futures { + println!("Future completed. Result: {:?}", future.wait()); + } + println!( + "produced all messages successfully ({})", + nbr_messages_produced + ) +} + +// A context can be used to change the behavior of producers and consumers by adding callbacks +// that will be executed by librdkafka. +// This particular context sets up custom callbacks to log rebalancing events. +struct CustomContext; + +impl ClientContext for CustomContext {} + +impl ConsumerContext for CustomContext { + fn pre_rebalance(&self, rebalance: &Rebalance) { + println!("Pre rebalance {:?}", rebalance); + } + + fn post_rebalance(&self, rebalance: &Rebalance) { + println!("Post rebalance {:?}", rebalance); + } + + fn commit_callback( + &self, + result: KafkaResult<()>, + _offsets: *mut rdkafka_sys::RDKafkaTopicPartitionList, + ) { + info!("Committing offsets: {:?}", result); + } +} + +// A type alias with your custom consumer can be created for convenience. +type LoggingConsumer = StreamConsumer; + +fn consume_and_print(brokers: &str, group_id: &str, topics: Vec<&str>, limit: i8) { + let context = CustomContext; + + let consumer: LoggingConsumer = ClientConfig::new() + .set("group.id", group_id) + .set("bootstrap.servers", brokers) + .set("enable.partition.eof", "false") + .set("session.timeout.ms", "6000") + .set("enable.auto.commit", "true") + //.set("statistics.interval.ms", "30000") + .set("auto.offset.reset", "earliest") + .set_log_level(RDKafkaLogLevel::Debug) + .create_with_context(context) + .expect("Consumer creation failed"); + + consumer + .subscribe(topics.as_slice()) + .expect("Can't subscribe to specified topics"); + + // consumer.start() returns a stream. The stream can be used ot chain together expensive steps, + // such as complex computations on a thread pool or asynchronous IO. + let message_stream = consumer.start(); + + let mut i = 0; + + for message in message_stream.wait() { + match message { + Err(_) => println!("Error while reading from stream."), + Ok(Err(e)) => println!("Kafka error: {}", e), + Ok(Ok(m)) => { + let payload = match m.payload_view::() { + None => "", + Some(Ok(s)) => s, + Some(Err(e)) => { + println!("Error while deserializing message payload: {:?}", e); + "" + } + }; + println!("key: '{:?}', payload: '{}', topic: {}, partition: {}, offset: {}, timestamp: {:?}", + m.key(), payload, m.topic(), m.partition(), m.offset(), m.timestamp()); + if let Some(headers) = m.headers() { + for i in 0..headers.count() { + let header = headers.get(i).unwrap(); + info!(" Header {:#?}: {:?}", header.0, header.1); + } + } + consumer.commit_message(&m, CommitMode::Sync).unwrap(); + i += 1; + println!("i={}, limit={}", i, limit); + if i == limit { + println!("limit reached, exiting"); + return; + } + } + }; + } +} diff --git a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java index 2b1aa14af2..353630f0f1 100644 --- a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java +++ b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java @@ -8,7 +8,6 @@ import org.junit.AfterClass; import org.testcontainers.Testcontainers; import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.output.WaitingConsumer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.images.builder.ImageFromDockerfile; @@ -20,7 +19,6 @@ import java.nio.file.Paths; import java.util.concurrent.TimeUnit; -import static org.testcontainers.containers.output.OutputFrame.OutputType.STDOUT; import static org.testng.AssertJUnit.assertFalse; import static org.testng.AssertJUnit.assertTrue; @@ -30,9 +28,10 @@ public class KafkaIntegrationTest extends MockKafkaServiceBaseTest { @DataProvider public static Object[][] integrations() { return new Object[][] { - // {"golang-sarama", "my-sarama-topic"}, - // {"golang-sarama", "persistent://public/default/my-sarama-topic-full-name"}, - {"golang-confluent-kafka", "confluent-go"} + {"golang-sarama", "my-sarama-topic"}, + {"golang-sarama", "persistent://public/default/my-sarama-topic-full-name"}, + {"golang-confluent-kafka", "confluent-go"}, + {"rustlang-rdkafka", "rustlang-topic"}, }; } @@ -74,7 +73,7 @@ protected void setup() throws Exception { Testcontainers.exposeHostPorts(ImmutableMap.of(super.kafkaBrokerPort, super.kafkaBrokerPort)); } - @Test(timeOut = 60_000, dataProvider = "integrations") + @Test(timeOut = 120_000, dataProvider = "integrations") void simpleProduceAndConsume(String integration, String topic) throws Exception { getAdmin().topics().createPartitionedTopic(topic, 1); @@ -86,6 +85,7 @@ void simpleProduceAndConsume(String integration, String topic) throws Exception .withEnv("KOP_TOPIC", topic) .withEnv("KOP_NBR_MESSAGES", "10") .withEnv("KOP_EXPECT_MESSAGES", "10") + .withLogConsumer(new org.testcontainers.containers.output.Slf4jLogConsumer(log)) .waitingFor(Wait.forLogMessage("starting to produce\\n", 1)) .withNetworkMode("host"); @@ -96,6 +96,7 @@ void simpleProduceAndConsume(String integration, String topic) throws Exception .withEnv("KOP_CONSUME", "true") .withEnv("KOP_NBR_MESSAGES", "10") .withEnv("KOP_EXPECT_MESSAGES", "10") + .withLogConsumer(new org.testcontainers.containers.output.Slf4jLogConsumer(log)) .waitingFor(Wait.forLogMessage("starting to consume\\n", 1)) .withNetworkMode("host"); @@ -124,11 +125,8 @@ protected void cleanup() throws Exception { private WaitingConsumer createLogFollower(GenericContainer container) { - Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(log); - container.followOutput(logConsumer); WaitingConsumer waitingConsumer = new WaitingConsumer(); - container.followOutput(waitingConsumer, STDOUT); - + container.followOutput(waitingConsumer); return waitingConsumer; } @@ -136,6 +134,8 @@ private void checkForErrorsInLogs(String logs) { assertFalse(logs.contains("no available broker to send metadata request to")); assertFalse(logs.contains("panic")); assertFalse(logs.contains("correlation ID didn't match")); + assertFalse(logs.contains("Required feature not supported by broker")); + if (logs.contains("starting to produce")) { assertTrue(logs.contains("produced all messages successfully")); From ad5f2c32c8e765f997917e1b5fc72ce2107caad0 Mon Sep 17 00:00:00 2001 From: Pierre Zemb Date: Thu, 2 Jan 2020 15:54:59 +0100 Subject: [PATCH 06/13] Adding support of ListOffset in v0 --- integrations/golang-confluent-kafka/main.go | 1 + integrations/rustlang-rdkafka/src/main.rs | 9 +- .../streamnative/kop/KafkaRequestHandler.java | 180 +++++++++++++----- 3 files changed, 138 insertions(+), 52 deletions(-) diff --git a/integrations/golang-confluent-kafka/main.go b/integrations/golang-confluent-kafka/main.go index fa10b5f2b9..2733236ff0 100644 --- a/integrations/golang-confluent-kafka/main.go +++ b/integrations/golang-confluent-kafka/main.go @@ -92,6 +92,7 @@ func main() { for counter < limit { msg, err := c.ReadMessage(-1) if err == nil { + fmt.Println("received msg") fmt.Printf("Message on %s: %s\n", msg.TopicPartition, string(msg.Value)) counter++ } else { diff --git a/integrations/rustlang-rdkafka/src/main.rs b/integrations/rustlang-rdkafka/src/main.rs index e364cd657c..ab92e585d7 100644 --- a/integrations/rustlang-rdkafka/src/main.rs +++ b/integrations/rustlang-rdkafka/src/main.rs @@ -42,6 +42,7 @@ fn main() { ); if should_produce { + println!("starting to produce"); produce(brokers.as_ref(), topic.as_str(), nbr_messages_produced); } if should_consume { @@ -150,14 +151,14 @@ fn consume_and_print(brokers: &str, group_id: &str, topics: Vec<&str>, limit: i8 for message in message_stream.wait() { match message { - Err(_) => println!("Error while reading from stream."), - Ok(Err(e)) => println!("Kafka error: {}", e), + Err(_) => panic!("Error while reading from stream."), + Ok(Err(e)) => panic!("Kafka error: {}", e), Ok(Ok(m)) => { let payload = match m.payload_view::() { None => "", Some(Ok(s)) => s, Some(Err(e)) => { - println!("Error while deserializing message payload: {:?}", e); + panic!("Error while deserializing message payload: {:?}", e); "" } }; @@ -171,7 +172,7 @@ fn consume_and_print(brokers: &str, group_id: &str, topics: Vec<&str>, limit: i8 } consumer.commit_message(&m, CommitMode::Sync).unwrap(); i += 1; - println!("i={}, limit={}", i, limit); + println!("received msg"); if i == limit { println!("limit reached, exiting"); return; diff --git a/src/main/java/io/streamnative/kop/KafkaRequestHandler.java b/src/main/java/io/streamnative/kop/KafkaRequestHandler.java index 2218f11f1a..c139264bd1 100644 --- a/src/main/java/io/streamnative/kop/KafkaRequestHandler.java +++ b/src/main/java/io/streamnative/kop/KafkaRequestHandler.java @@ -182,7 +182,7 @@ protected ApiVersionsResponse overloadDefaultApiVersionsResponse() { if (apiKey.minRequiredInterBrokerMagic <= RecordBatch.CURRENT_MAGIC_VALUE) { switch (apiKey) { case LIST_OFFSETS: - versionList.add(new ApiVersionsResponse.ApiVersion((short) 2, (short) 1, apiKey.latestVersion())); + versionList.add(new ApiVersionsResponse.ApiVersion((short) 2, (short) 0, apiKey.latestVersion())); break; default: versionList.add(new ApiVersionsResponse.ApiVersion(apiKey)); @@ -571,7 +571,7 @@ protected CompletableFuture handleOffsetFetchRequest(KafkaHe } private CompletableFuture - fetchOffsetForTimestamp(PersistentTopic persistentTopic, Long timestamp) { + fetchOffsetForTimestamp(PersistentTopic persistentTopic, Long timestamp, boolean legacyMode) { ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); @@ -588,11 +588,20 @@ protected CompletableFuture handleOffsetFetchRequest(KafkaHe // no entry in ledger, then entry id could be -1 long entryId = position.getEntryId(); - partitionData.complete(new ListOffsetResponse.PartitionData( - Errors.NONE, - RecordBatch.NO_TIMESTAMP, - MessageIdUtils - .getOffset(position.getLedgerId(), entryId == -1 ? 0 : entryId))); + if (legacyMode) { + partitionData.complete(new ListOffsetResponse.PartitionData( + Errors.NONE, + Collections.singletonList(MessageIdUtils + .getOffset(position.getLedgerId(), entryId == -1 ? 0 : entryId)))); + + } else { + partitionData.complete(new ListOffsetResponse.PartitionData( + Errors.NONE, + RecordBatch.NO_TIMESTAMP, + MessageIdUtils + .getOffset(position.getLedgerId(), entryId == -1 ? 0 : entryId))); + } + } else if (timestamp == ListOffsetRequest.EARLIEST_TIMESTAMP) { PositionImpl position = OffsetFinder.getFirstValidPosition(managedLedger); @@ -601,10 +610,17 @@ protected CompletableFuture handleOffsetFetchRequest(KafkaHe persistentTopic.getName(), timestamp, position); } - partitionData.complete(new ListOffsetResponse.PartitionData( - Errors.NONE, - RecordBatch.NO_TIMESTAMP, - MessageIdUtils.getOffset(position.getLedgerId(), position.getEntryId()))); + if (legacyMode) { + partitionData.complete(new ListOffsetResponse.PartitionData( + Errors.NONE, + Collections.singletonList(MessageIdUtils.getOffset(position.getLedgerId(), position.getEntryId())))); + } else { + partitionData.complete(new ListOffsetResponse.PartitionData( + Errors.NONE, + RecordBatch.NO_TIMESTAMP, + MessageIdUtils.getOffset(position.getLedgerId(), position.getEntryId()))); + } + } else { // find with real wanted timestamp OffsetFinder offsetFinder = new OffsetFinder(managedLedger); @@ -619,11 +635,18 @@ public void findEntryComplete(Position position, Object ctx) { log.warn("Unable to find position for topic {} time {}. get NULL position", persistentTopic.getName(), timestamp); - partitionData.complete(new ListOffsetResponse - .PartitionData( - Errors.UNKNOWN_SERVER_ERROR, - ListOffsetResponse.UNKNOWN_TIMESTAMP, - ListOffsetResponse.UNKNOWN_OFFSET)); + if (legacyMode) { + partitionData.complete(new ListOffsetResponse + .PartitionData( + Errors.UNKNOWN_SERVER_ERROR, + Collections.emptyList())); + } else { + partitionData.complete(new ListOffsetResponse + .PartitionData( + Errors.UNKNOWN_SERVER_ERROR, + ListOffsetResponse.UNKNOWN_TIMESTAMP, + ListOffsetResponse.UNKNOWN_OFFSET)); + } return; } } else { @@ -634,10 +657,16 @@ public void findEntryComplete(Position position, Object ctx) { log.debug("Find position for topic {} time {}. position: {}", persistentTopic.getName(), timestamp, finalPosition); } - partitionData.complete(new ListOffsetResponse.PartitionData( - Errors.NONE, - RecordBatch.NO_TIMESTAMP, - MessageIdUtils.getOffset(finalPosition.getLedgerId(), finalPosition.getEntryId()))); + if (legacyMode) { + partitionData.complete(new ListOffsetResponse.PartitionData( + Errors.NONE, + Collections.singletonList(MessageIdUtils.getOffset(finalPosition.getLedgerId(), finalPosition.getEntryId())))); + } else { + partitionData.complete(new ListOffsetResponse.PartitionData( + Errors.NONE, + RecordBatch.NO_TIMESTAMP, + MessageIdUtils.getOffset(finalPosition.getLedgerId(), finalPosition.getEntryId()))); + } } @Override @@ -645,11 +674,18 @@ public void findEntryFailed(ManagedLedgerException exception, Optional position, Object ctx) { log.warn("Unable to find position for topic {} time {}. Exception:", persistentTopic.getName(), timestamp, exception); - partitionData.complete(new ListOffsetResponse - .PartitionData( - Errors.UNKNOWN_SERVER_ERROR, - ListOffsetResponse.UNKNOWN_TIMESTAMP, - ListOffsetResponse.UNKNOWN_OFFSET)); + if (legacyMode) { + partitionData.complete(new ListOffsetResponse + .PartitionData( + Errors.UNKNOWN_SERVER_ERROR, + Collections.emptyList())); + } else { + partitionData.complete(new ListOffsetResponse + .PartitionData( + Errors.UNKNOWN_SERVER_ERROR, + ListOffsetResponse.UNKNOWN_TIMESTAMP, + ListOffsetResponse.UNKNOWN_OFFSET)); + } return; } }); @@ -657,16 +693,77 @@ public void findEntryFailed(ManagedLedgerException exception, } catch (Exception e) { log.error("Failed while get position for topic: {} ts: {}.", persistentTopic.getName(), timestamp, e); - - partitionData.complete(new ListOffsetResponse.PartitionData( - Errors.UNKNOWN_SERVER_ERROR, - ListOffsetResponse.UNKNOWN_TIMESTAMP, - ListOffsetResponse.UNKNOWN_OFFSET)); + if (legacyMode) { + partitionData.complete(new ListOffsetResponse + .PartitionData( + Errors.UNKNOWN_SERVER_ERROR, + Collections.emptyList())); + } else { + partitionData.complete(new ListOffsetResponse + .PartitionData( + Errors.UNKNOWN_SERVER_ERROR, + ListOffsetResponse.UNKNOWN_TIMESTAMP, + ListOffsetResponse.UNKNOWN_OFFSET)); + } } - return partitionData; } + + // Some info can be found here https://web.archive.org/web/20170309152525/https://cfchou.github.io/blog/2015/04/23/a-closer-look-at-kafka-offsetrequest/ + private CompletableFuture handleListOffsetRequestV0(KafkaHeaderAndRequest listOffset) { + ListOffsetRequest request = (ListOffsetRequest) listOffset.getRequest(); + + CompletableFuture resultFuture = new CompletableFuture<>(); + Map> responseData = Maps.newHashMap(); + + // in v0, the iterator is offsetData, + // in v1, the iterator is partitionTimestamps, + log.warn("received a v0 listOffset: {}", request.toString(true)); + request.offsetData().entrySet().stream().forEach(tms -> { + TopicPartition topic = tms.getKey(); + TopicName pulsarTopic = pulsarTopicName(topic, namespace); + Long times = tms.getValue().timestamp; + CompletableFuture partitionData; + + // num_num_offsets > 1 is not handled for now, returning an error + if (tms.getValue().maxNumOffsets > 1) { + log.warn("request is asking for multiples offsets for {}, not supported for now", pulsarTopic.toString()); + partitionData = new CompletableFuture<>(); + partitionData.complete(new ListOffsetResponse + .PartitionData( + Errors.UNKNOWN_SERVER_ERROR, + Collections.singletonList(ListOffsetResponse.UNKNOWN_OFFSET))); + } + + // topic not exist, return UNKNOWN_TOPIC_OR_PARTITION + if (!topicManager.topicExists(pulsarTopic.toString())) { + log.warn("Topic {} not exist in topic manager while list offset.", pulsarTopic.toString()); + partitionData = new CompletableFuture<>(); + partitionData.complete(new ListOffsetResponse + .PartitionData( + Errors.UNKNOWN_TOPIC_OR_PARTITION, + Collections.singletonList(ListOffsetResponse.UNKNOWN_OFFSET))); + } else { + PersistentTopic persistentTopic = topicManager.getTopic(pulsarTopic.toString()); + partitionData = fetchOffsetForTimestamp(persistentTopic, times, true); + } + responseData.put(topic, partitionData); + }); + + CompletableFuture + .allOf(responseData.values().stream().toArray(CompletableFuture[]::new)) + .whenComplete((ignore, ex) -> { + ListOffsetResponse response = + new ListOffsetResponse(CoreUtils.mapValue(responseData, future -> future.join())); + + resultFuture.complete(ResponseAndRequest + .of(response, listOffset)); + }); + + return resultFuture; + } + private CompletableFuture handleListOffsetRequestV1AndAbove(KafkaHeaderAndRequest listOffset) { ListOffsetRequest request = (ListOffsetRequest) listOffset.getRequest(); @@ -690,7 +787,7 @@ private CompletableFuture handleListOffsetRequestV1AndAbove( ListOffsetResponse.UNKNOWN_OFFSET)); } else { PersistentTopic persistentTopic = topicManager.getTopic(pulsarTopic.toString()); - partitionData = fetchOffsetForTimestamp(persistentTopic, times); + partitionData = fetchOffsetForTimestamp(persistentTopic, times, false); } responseData.put(topic, partitionData); @@ -712,24 +809,11 @@ private CompletableFuture handleListOffsetRequestV1AndAbove( // get offset from underline managedLedger protected CompletableFuture handleListOffsetRequest(KafkaHeaderAndRequest listOffset) { checkArgument(listOffset.getRequest() instanceof ListOffsetRequest); - - // not support version 0 + // the only difference between v0 and v1 is the `max_num_offsets => INT32` + // v0 is required because it is used by librdkafka if (listOffset.getHeader().apiVersion() == 0) { - CompletableFuture resultFuture = new CompletableFuture<>(); - ListOffsetRequest request = (ListOffsetRequest) listOffset.getRequest(); - - log.error("ListOffset not support V0 format request"); - - ListOffsetResponse response = new ListOffsetResponse(CoreUtils.mapValue(request.partitionTimestamps(), - ignored -> new ListOffsetResponse - .PartitionData(Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT, Lists.newArrayList()))); - - resultFuture.complete(ResponseAndRequest - .of(response, listOffset)); - - return resultFuture; + return handleListOffsetRequestV0(listOffset); } - return handleListOffsetRequestV1AndAbove(listOffset); } From 5a521107d241992b8180261288450b7b01798770 Mon Sep 17 00:00:00 2001 From: Pierre Zemb Date: Mon, 6 Jan 2020 13:40:36 +0100 Subject: [PATCH 07/13] Initial commit for node-kafka integration test --- integrations/README.md | 12 ++++ integrations/node-kafka-node/Dockerfile | 12 ++++ integrations/node-kafka-node/main.js | 68 +++++++++++++++++++ integrations/node-kafka-node/package.json | 9 +++ .../streamnative/kop/KafkaRequestHandler.java | 5 ++ .../kop/KafkaIntegrationTest.java | 53 +++++++++------ 6 files changed, 139 insertions(+), 20 deletions(-) create mode 100644 integrations/README.md create mode 100644 integrations/node-kafka-node/Dockerfile create mode 100644 integrations/node-kafka-node/main.js create mode 100644 integrations/node-kafka-node/package.json diff --git a/integrations/README.md b/integrations/README.md new file mode 100644 index 0000000000..22168a6dfd --- /dev/null +++ b/integrations/README.md @@ -0,0 +1,12 @@ +# Integration tests for KoP + +# not working librairies + +### Kafka-node + +Producing is working, but consuming is failing as the library does not care about API_VERSIONS response and is sending FETCH v0. + +``` +DEBUG io.streamnative.kop.KafkaCommandDecoder - Write kafka cmd response back to client. request: RequestHeader(apiKey=FETCH, apiVersion=0, clientId=kafka-node-client, correlationId=4) +INFO io.streamnative.kop.KafkaIntegrationTest - STDOUT: Error: Not a message set. Magic byte is 2 +``` \ No newline at end of file diff --git a/integrations/node-kafka-node/Dockerfile b/integrations/node-kafka-node/Dockerfile new file mode 100644 index 0000000000..8392e48964 --- /dev/null +++ b/integrations/node-kafka-node/Dockerfile @@ -0,0 +1,12 @@ +FROM node:13-alpine + +RUN apk add yarn snappy + +# Create app directory +WORKDIR /usr/src/app + +COPY . . + +RUN yarn install + +CMD sh -c 'node main.js; echo "ExitCode=$?"' \ No newline at end of file diff --git a/integrations/node-kafka-node/main.js b/integrations/node-kafka-node/main.js new file mode 100644 index 0000000000..4770ff543d --- /dev/null +++ b/integrations/node-kafka-node/main.js @@ -0,0 +1,68 @@ + +'use strict'; + +var kafka = require('kafka-node'); +var HighLevelProducer = kafka.HighLevelProducer; +var Consumer = kafka.Consumer; +var broker = process.env.KOP_BROKER || 'localhost:9092'; +var topic = process.env.KOP_TOPIC || 'topic1'; +var limit = process.env.KOP_EXPECT_MESSAGES || 10; +var to_produced = process.env.KOP_NBR_MESSAGES || 10; +var should_produce = process.env.KOP_PRODUCE || false; +var should_consume = process.env.KOP_CONSUME || false; + +var counter = 0; + +var Client = kafka.KafkaClient; +var client = new Client({ kafkaHost: broker }); + +if (should_consume) { + // if fetchMaxBytes is set too low the broker could start sending fetch responses in RecordBatch format instead of MessageSet :( + // https://www.npmjs.com/package/kafka-node#error-not-a-message-set-magic-byte-is-2 + var consumer = new Consumer(client, [{ topic: topic, partition: 0, fetchMaxBytes: 1024 * 1024 * 1024 * 1024 }], { + autoCommit: true + }); + console.log('starting to consume'); + + consumer.on("message", function (message) { + console.log('received msg ' + message); + counter++; + if (counter == limit) { + console.log("limit reached, exiting"); + process.exit(); + }; + }); + + consumer.on("error", function (err) { + console.log(err); + process.exit(1); + }); +} + +if (should_produce) { + var producer = new HighLevelProducer(client); + producer.on('ready', function () { + console.log('starting to produce'); + setInterval(send, 500); + }); + + producer.on('error', function (err) { + console.log('error', err); + process.exit(1); + }); +} + +function send() { + var message = new Date().toString(); + producer.send([{ topic: topic, messages: [message] }], function (err, data) { + if (err) { + console.log('error', err); + process.exit(1); + } + counter++; + if (counter == limit) { + console.log("limit reached, exiting"); + process.exit(); + }; + }); +} \ No newline at end of file diff --git a/integrations/node-kafka-node/package.json b/integrations/node-kafka-node/package.json new file mode 100644 index 0000000000..59c0358b30 --- /dev/null +++ b/integrations/node-kafka-node/package.json @@ -0,0 +1,9 @@ +{ + "name": "node-kafka-node", + "version": "1.0.0", + "main": "index.js", + "license": "Apache-2.0", + "dependencies": { + "kafka-node": "^5.0.0" + } +} diff --git a/src/main/java/io/streamnative/kop/KafkaRequestHandler.java b/src/main/java/io/streamnative/kop/KafkaRequestHandler.java index c139264bd1..bbda0ced69 100644 --- a/src/main/java/io/streamnative/kop/KafkaRequestHandler.java +++ b/src/main/java/io/streamnative/kop/KafkaRequestHandler.java @@ -181,7 +181,12 @@ protected ApiVersionsResponse overloadDefaultApiVersionsResponse() { for (ApiKeys apiKey : ApiKeys.values()) { if (apiKey.minRequiredInterBrokerMagic <= RecordBatch.CURRENT_MAGIC_VALUE) { switch (apiKey) { + case FETCH: + // V4 added MessageSets responses. We need to make sure RecordBatch format is not used + versionList.add(new ApiVersionsResponse.ApiVersion((short) 1, (short) 4, apiKey.latestVersion())); + break; case LIST_OFFSETS: + // V0 is needed for librdkafka versionList.add(new ApiVersionsResponse.ApiVersion((short) 2, (short) 0, apiKey.latestVersion())); break; default: diff --git a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java index 353630f0f1..768f18be47 100644 --- a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java +++ b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java @@ -17,6 +17,7 @@ import org.testng.annotations.Test; import java.nio.file.Paths; +import java.util.Optional; import java.util.concurrent.TimeUnit; import static org.testng.AssertJUnit.assertFalse; @@ -28,10 +29,12 @@ public class KafkaIntegrationTest extends MockKafkaServiceBaseTest { @DataProvider public static Object[][] integrations() { return new Object[][] { - {"golang-sarama", "my-sarama-topic"}, - {"golang-sarama", "persistent://public/default/my-sarama-topic-full-name"}, - {"golang-confluent-kafka", "confluent-go"}, - {"rustlang-rdkafka", "rustlang-topic"}, + {"golang-sarama", Optional.empty(), true, true}, + {"golang-sarama", Optional.of("persistent://public/default/my-sarama-topic-full-name"), true, true}, + {"golang-confluent-kafka", Optional.empty(), true, true}, + {"rustlang-rdkafka", Optional.empty(), true, true}, + // consumer is broken, see integrations/README.md + {"node-kafka-node", Optional.empty(), true, false}, }; } @@ -74,15 +77,15 @@ protected void setup() throws Exception { } @Test(timeOut = 120_000, dataProvider = "integrations") - void simpleProduceAndConsume(String integration, String topic) throws Exception { + void simpleProduceAndConsume(String integration, Optional topic, boolean shouldProduce, boolean shouldConsume) throws Exception { - getAdmin().topics().createPartitionedTopic(topic, 1); + getAdmin().topics().createPartitionedTopic(topic.orElse(integration), 1); GenericContainer producer = new GenericContainer<>( new ImageFromDockerfile().withFileFromPath(".", Paths.get("integrations/" + integration))) .withEnv("KOP_BROKER", "localhost:" + super.kafkaBrokerPort) .withEnv("KOP_PRODUCE", "true") - .withEnv("KOP_TOPIC", topic) + .withEnv("KOP_TOPIC", topic.orElse(integration)) .withEnv("KOP_NBR_MESSAGES", "10") .withEnv("KOP_EXPECT_MESSAGES", "10") .withLogConsumer(new org.testcontainers.containers.output.Slf4jLogConsumer(log)) @@ -92,7 +95,7 @@ void simpleProduceAndConsume(String integration, String topic) throws Exception GenericContainer consumer = new GenericContainer<>( new ImageFromDockerfile().withFileFromPath(".", Paths.get("integrations/" + integration))) .withEnv("KOP_BROKER", "localhost:" + super.kafkaBrokerPort) - .withEnv("KOP_TOPIC", topic) + .withEnv("KOP_TOPIC", topic.orElse(integration)) .withEnv("KOP_CONSUME", "true") .withEnv("KOP_NBR_MESSAGES", "10") .withEnv("KOP_EXPECT_MESSAGES", "10") @@ -100,21 +103,31 @@ void simpleProduceAndConsume(String integration, String topic) throws Exception .waitingFor(Wait.forLogMessage("starting to consume\\n", 1)) .withNetworkMode("host"); - producer.start(); - WaitingConsumer consumerWaitingConsumer = createLogFollower(producer); - System.out.println("producer started"); - consumer.start(); - WaitingConsumer producerWaitingConsumer = createLogFollower(consumer); - System.out.println("consumer started"); + WaitingConsumer producerWaitingConsumer = null; + WaitingConsumer consumerWaitingConsumer = null; + if (shouldProduce) { + producer.start(); + producerWaitingConsumer = createLogFollower(producer); + System.out.println("producer started"); + } + if (shouldConsume) { + consumer.start(); + consumerWaitingConsumer = createLogFollower(consumer); + System.out.println("consumer started"); + } - producerWaitingConsumer.waitUntil(frame -> - frame.getUtf8String().contains("ExitCode"), 30, TimeUnit.SECONDS); - consumerWaitingConsumer.waitUntil(frame -> - frame.getUtf8String().contains("ExitCode"), 30, TimeUnit.SECONDS); + if (shouldProduce) { + producerWaitingConsumer.waitUntil(frame -> + frame.getUtf8String().contains("ExitCode"), 30, TimeUnit.SECONDS); + checkForErrorsInLogs(producer.getLogs()); + } - checkForErrorsInLogs(producer.getLogs()); - checkForErrorsInLogs(consumer.getLogs()); + if (shouldConsume) { + consumerWaitingConsumer.waitUntil(frame -> + frame.getUtf8String().contains("ExitCode"), 30, TimeUnit.SECONDS); + checkForErrorsInLogs(consumer.getLogs()); + } } @Override From 1e6256ced37c0ca9851ddeea2f4e87b43a95d09a Mon Sep 17 00:00:00 2001 From: Pierre Zemb Date: Mon, 6 Jan 2020 17:21:38 +0100 Subject: [PATCH 08/13] Initial commit for node-rdkafka --- integrations/node-kafka-node/main.js | 5 +- integrations/node-rdkafka/Dockerfile | 26 ++++ integrations/node-rdkafka/main.js | 134 ++++++++++++++++++ integrations/node-rdkafka/package.json | 9 ++ .../kop/KafkaIntegrationTest.java | 3 +- 5 files changed, 173 insertions(+), 4 deletions(-) create mode 100644 integrations/node-rdkafka/Dockerfile create mode 100644 integrations/node-rdkafka/main.js create mode 100644 integrations/node-rdkafka/package.json diff --git a/integrations/node-kafka-node/main.js b/integrations/node-kafka-node/main.js index 4770ff543d..950c3cda7d 100644 --- a/integrations/node-kafka-node/main.js +++ b/integrations/node-kafka-node/main.js @@ -6,8 +6,7 @@ var HighLevelProducer = kafka.HighLevelProducer; var Consumer = kafka.Consumer; var broker = process.env.KOP_BROKER || 'localhost:9092'; var topic = process.env.KOP_TOPIC || 'topic1'; -var limit = process.env.KOP_EXPECT_MESSAGES || 10; -var to_produced = process.env.KOP_NBR_MESSAGES || 10; +var limit = parseInt(process.env.KOP_LIMIT || 10, 10); var should_produce = process.env.KOP_PRODUCE || false; var should_consume = process.env.KOP_CONSUME || false; @@ -61,7 +60,7 @@ function send() { } counter++; if (counter == limit) { - console.log("limit reached, exiting"); + console.log("produced all messages successfully"); process.exit(); }; }); diff --git a/integrations/node-rdkafka/Dockerfile b/integrations/node-rdkafka/Dockerfile new file mode 100644 index 0000000000..7c2bc0cf25 --- /dev/null +++ b/integrations/node-rdkafka/Dockerfile @@ -0,0 +1,26 @@ +FROM node:8-alpine + +RUN apk add yarn snappy + +RUN apk --no-cache add \ + bash \ + g++ \ + ca-certificates \ + lz4-dev \ + musl-dev \ + cyrus-sasl-dev \ + openssl-dev \ + make \ + python + +RUN apk add --no-cache --virtual .build-deps gcc zlib-dev libc-dev bsd-compat-headers py-setuptools bash + + +# Create app directory +WORKDIR /usr/src/app + +COPY . . + +RUN yarn install + +CMD sh -c 'node main.js; echo "ExitCode=$?"' \ No newline at end of file diff --git a/integrations/node-rdkafka/main.js b/integrations/node-rdkafka/main.js new file mode 100644 index 0000000000..4ebc788aed --- /dev/null +++ b/integrations/node-rdkafka/main.js @@ -0,0 +1,134 @@ +'use strict'; +const Kafka = require('node-rdkafka'); + +var broker = process.env.KOP_BROKER || 'localhost:9092'; +var topic = process.env.KOP_TOPIC || 'topic1'; +var limit = parseInt(process.env.KOP_LIMIT || 10, 10); +var should_produce = process.env.KOP_PRODUCE || false; +var should_consume = process.env.KOP_CONSUME || false; + +var counter = 0; + +if (should_consume) { + var consumer = new Kafka.KafkaConsumer({ + //'debug': 'all', + 'metadata.broker.list': broker, + 'group.id': 'node-rdkafka-consumer-flow-example', + 'enable.auto.commit': false + }, { + "auto.offset.reset": "earliest", + }); + + var topicName = topic; + + //logging debug messages, if debug is enabled + consumer.on('event.log', function (log) { + console.log(log); + }); + + //logging all errors + consumer.on('event.error', function (err) { + console.error('Error from consumer'); + console.error(err); + process.exit(1); + }); + + + consumer.on('ready', function (arg) { + console.log('consumer ready.' + JSON.stringify(arg)); + console.log("starting to consume"); + + consumer.subscribe([topicName]); + //start consuming messages + consumer.consume(); + }); + + consumer.on('data', function (m) { + + console.log("received msg", m) + + console.log('calling commit'); + consumer.commit(m); + + // Output the actual message contents + console.log(JSON.stringify(m)); + console.log(m.value.toString()); + + counter++; + + if (counter === limit) { + console.log("limit reached, exiting"); + consumer.disconnect(); + } + }); + + consumer.on('disconnected', function (arg) { + console.log('consumer disconnected. ' + JSON.stringify(arg)); + process.exit(); + + }); + + //starting the consumer + consumer.connect(); +} + +if (should_produce) { + var producer = new Kafka.Producer({ + // 'debug' : 'all', + 'metadata.broker.list': broker, + 'dr_cb': true //delivery report callback + }); + + var topicName = topic; + + //logging debug messages, if debug is enabled + producer.on('event.log', function (log) { + console.log(log); + }); + + //logging all errors + producer.on('event.error', function (err) { + console.error('Error from producer'); + console.error(err); + process.exit(1); + }); + + producer.on('delivery-report', function (err, report) { + console.log('delivery-report: ' + JSON.stringify(report)); + counter++; + console.log(counter, limit); + if (counter === limit) { + console.log("produced all messages successfully"); + producer.disconnect(); + process.exit(); + } + }); + + //Wait for the ready event before producing + producer.on('ready', function (arg) { + console.log('producer ready.' + JSON.stringify(arg)); + console.log('starting to produce'); + + for (var i = 0; i < limit; i++) { + var value = Buffer.from('value-' + i); + var key = "key-" + i; + // if partition is set to -1, librdkafka will use the default partitioner + var partition = -1; + producer.produce(topicName, partition, value, key); + } + + //need to keep polling for a while to ensure the delivery reports are received + var pollLoop = setInterval(function () { + producer.poll(); + }, 1000); + + }); + + producer.on('disconnected', function (arg) { + console.log('producer disconnected. ' + JSON.stringify(arg)); + }); + + //starting the producer + producer.connect(); +} + diff --git a/integrations/node-rdkafka/package.json b/integrations/node-rdkafka/package.json new file mode 100644 index 0000000000..3090730a9a --- /dev/null +++ b/integrations/node-rdkafka/package.json @@ -0,0 +1,9 @@ +{ + "name": "node-rdkafka", + "version": "1.0.0", + "main": "main.js", + "license": "Apache-2.0", + "dependencies": { + "node-rdkafka": "^2.7.4" + } +} diff --git a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java index 768f18be47..be8141c08d 100644 --- a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java +++ b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java @@ -35,6 +35,7 @@ public static Object[][] integrations() { {"rustlang-rdkafka", Optional.empty(), true, true}, // consumer is broken, see integrations/README.md {"node-kafka-node", Optional.empty(), true, false}, + {"node-rdkafka", Optional.empty(), true, true}, }; } @@ -76,7 +77,7 @@ protected void setup() throws Exception { Testcontainers.exposeHostPorts(ImmutableMap.of(super.kafkaBrokerPort, super.kafkaBrokerPort)); } - @Test(timeOut = 120_000, dataProvider = "integrations") + @Test(timeOut = 5 * 60_000, dataProvider = "integrations") void simpleProduceAndConsume(String integration, Optional topic, boolean shouldProduce, boolean shouldConsume) throws Exception { getAdmin().topics().createPartitionedTopic(topic.orElse(integration), 1); From 0890ff73727081c51627b208a8cb8a2c4a580f01 Mon Sep 17 00:00:00 2001 From: Pierre Zemb Date: Wed, 8 Jan 2020 10:42:18 +0100 Subject: [PATCH 09/13] checkstyle --- integrations/README.md | 21 ++- .../golang-confluent-kafka/Dockerfile | 13 ++ integrations/golang-confluent-kafka/main.go | 17 +- integrations/golang-sarama/Dockerfile | 13 ++ integrations/golang-sarama/main.go | 16 +- integrations/node-kafka-node/Dockerfile | 13 ++ integrations/node-kafka-node/main.js | 2 +- integrations/node-rdkafka/Dockerfile | 13 ++ integrations/node-rdkafka/main.js | 16 +- integrations/rustlang-rdkafka/Dockerfile | 13 ++ integrations/rustlang-rdkafka/src/main.rs | 16 +- .../streamnative/kop/KafkaRequestHandler.java | 136 +++++++++------- .../kop/KafkaIntegrationTest.java | 152 +++++++++++------- 13 files changed, 311 insertions(+), 130 deletions(-) diff --git a/integrations/README.md b/integrations/README.md index 22168a6dfd..217700aae9 100644 --- a/integrations/README.md +++ b/integrations/README.md @@ -1,10 +1,25 @@ # Integration tests for KoP -# not working librairies +# Produce and Consume support -### Kafka-node +## Golang -Producing is working, but consuming is failing as the library does not care about API_VERSIONS response and is sending FETCH v0. +* [https://github.com/Shopify/sarama](https://github.com/Shopify/sarama) +* [https://github.com/confluentinc/confluent-kafka-go](https://github.com/confluentinc/confluent-kafka-go) + +## Rust + +* [https://github.com/fede1024/rust-rdkafka](https://github.com/fede1024/rust-rdkafka) + +## NodeJS + +* [https://github.com/Blizzard/node-rdkafka](https://github.com/Blizzard/node-rdkafka) + +# Partial support + +### [kafka-node](https://www.npmjs.com/package/kafka-node) + +Producing is working, but consuming is failing as the library is sending FETCH v0 regardless of API_VERSIONS responses: ``` DEBUG io.streamnative.kop.KafkaCommandDecoder - Write kafka cmd response back to client. request: RequestHeader(apiKey=FETCH, apiVersion=0, clientId=kafka-node-client, correlationId=4) diff --git a/integrations/golang-confluent-kafka/Dockerfile b/integrations/golang-confluent-kafka/Dockerfile index 4698ac37a7..be8e224df8 100644 --- a/integrations/golang-confluent-kafka/Dockerfile +++ b/integrations/golang-confluent-kafka/Dockerfile @@ -1,3 +1,16 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# FROM golang:1.13.4-alpine RUN apk -U add ca-certificates git bash build-base sudo pkgconf build-base diff --git a/integrations/golang-confluent-kafka/main.go b/integrations/golang-confluent-kafka/main.go index 2733236ff0..d71828603f 100644 --- a/integrations/golang-confluent-kafka/main.go +++ b/integrations/golang-confluent-kafka/main.go @@ -1,3 +1,18 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + package main import ( @@ -101,7 +116,7 @@ func main() { panic(err) } } - fmt.Println("limit reached, exiting") + fmt.Println("consumed all messages successfully") } fmt.Println("exiting normally") diff --git a/integrations/golang-sarama/Dockerfile b/integrations/golang-sarama/Dockerfile index 3d97f9d004..d15800a409 100644 --- a/integrations/golang-sarama/Dockerfile +++ b/integrations/golang-sarama/Dockerfile @@ -1,3 +1,16 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# FROM golang:1.13.4-stretch WORKDIR /go/src/app diff --git a/integrations/golang-sarama/main.go b/integrations/golang-sarama/main.go index aa9f4c6f64..7b7db591bc 100644 --- a/integrations/golang-sarama/main.go +++ b/integrations/golang-sarama/main.go @@ -1,3 +1,17 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// package main import ( @@ -26,7 +40,7 @@ func (h exampleConsumerGroupHandler) ConsumeClaim(sess sarama.ConsumerGroupSessi h.counter++ fmt.Printf("received msg %d/%d\n", h.counter, h.limit) if h.counter == h.limit { - fmt.Println("limit reached, exiting") + fmt.Println("consumed all messages successfully") h.wg.Done() return nil } diff --git a/integrations/node-kafka-node/Dockerfile b/integrations/node-kafka-node/Dockerfile index 8392e48964..b0b5863f3b 100644 --- a/integrations/node-kafka-node/Dockerfile +++ b/integrations/node-kafka-node/Dockerfile @@ -1,3 +1,16 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# FROM node:13-alpine RUN apk add yarn snappy diff --git a/integrations/node-kafka-node/main.js b/integrations/node-kafka-node/main.js index 950c3cda7d..15793ca121 100644 --- a/integrations/node-kafka-node/main.js +++ b/integrations/node-kafka-node/main.js @@ -27,7 +27,7 @@ if (should_consume) { console.log('received msg ' + message); counter++; if (counter == limit) { - console.log("limit reached, exiting"); + console.log("consumed all messages successfully"); process.exit(); }; }); diff --git a/integrations/node-rdkafka/Dockerfile b/integrations/node-rdkafka/Dockerfile index 7c2bc0cf25..b357c1d231 100644 --- a/integrations/node-rdkafka/Dockerfile +++ b/integrations/node-rdkafka/Dockerfile @@ -1,3 +1,16 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# FROM node:8-alpine RUN apk add yarn snappy diff --git a/integrations/node-rdkafka/main.js b/integrations/node-rdkafka/main.js index 4ebc788aed..216c973973 100644 --- a/integrations/node-rdkafka/main.js +++ b/integrations/node-rdkafka/main.js @@ -1,3 +1,17 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// 'use strict'; const Kafka = require('node-rdkafka'); @@ -57,7 +71,7 @@ if (should_consume) { counter++; if (counter === limit) { - console.log("limit reached, exiting"); + console.log("consumed all messages successfully"); consumer.disconnect(); } }); diff --git a/integrations/rustlang-rdkafka/Dockerfile b/integrations/rustlang-rdkafka/Dockerfile index d3384e6731..994953741c 100644 --- a/integrations/rustlang-rdkafka/Dockerfile +++ b/integrations/rustlang-rdkafka/Dockerfile @@ -1,3 +1,16 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# FROM rust:1.40-stretch RUN apt update diff --git a/integrations/rustlang-rdkafka/src/main.rs b/integrations/rustlang-rdkafka/src/main.rs index ab92e585d7..91d7f08394 100644 --- a/integrations/rustlang-rdkafka/src/main.rs +++ b/integrations/rustlang-rdkafka/src/main.rs @@ -1,3 +1,17 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// #[macro_use] extern crate log; extern crate futures; @@ -174,7 +188,7 @@ fn consume_and_print(brokers: &str, group_id: &str, topics: Vec<&str>, limit: i8 i += 1; println!("received msg"); if i == limit { - println!("limit reached, exiting"); + println!("consumed all messages successfully"); return; } } diff --git a/src/main/java/io/streamnative/kop/KafkaRequestHandler.java b/src/main/java/io/streamnative/kop/KafkaRequestHandler.java index bbda0ced69..f9b13d8f6f 100644 --- a/src/main/java/io/streamnative/kop/KafkaRequestHandler.java +++ b/src/main/java/io/streamnative/kop/KafkaRequestHandler.java @@ -40,7 +40,14 @@ import java.net.InetSocketAddress; import java.net.URI; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -168,6 +175,57 @@ public KafkaRequestHandler(PulsarService pulsarService, kafkaConfig.getKafkaNamespace()); } + static Node newNode(InetSocketAddress address) { + if (log.isDebugEnabled()) { + log.debug("Return Broker Node of {}. {}:{}", address, address.getHostString(), address.getPort()); + } + return new Node( + Murmur3_32Hash.getInstance().makeHash((address.getHostString() + address.getPort()).getBytes(UTF_8)), + address.getHostString(), + address.getPort()); + } + + static PartitionMetadata newPartitionMetadata(TopicName topicName, Node node) { + int pulsarPartitionIndex = topicName.getPartitionIndex(); + int kafkaPartitionIndex = pulsarPartitionIndex == -1 ? 0 : pulsarPartitionIndex; + + if (log.isDebugEnabled()) { + log.debug("Return PartitionMetadata node: {}, topicName: {}", node, topicName); + } + + return new PartitionMetadata( + Errors.NONE, + kafkaPartitionIndex, + node, // leader + Lists.newArrayList(node), // replicas + Lists.newArrayList(node), // isr + Collections.emptyList() // offline replicas + ); + } + + static PartitionMetadata newFailedPartitionMetadata(TopicName topicName) { + int pulsarPartitionIndex = topicName.getPartitionIndex(); + int kafkaPartitionIndex = pulsarPartitionIndex == -1 ? 0 : pulsarPartitionIndex; + + log.warn("Failed find Broker metadata, create PartitionMetadata with INVALID_PARTITIONS"); + + return new PartitionMetadata( + Errors.UNKNOWN_SERVER_ERROR, + kafkaPartitionIndex, + Node.noNode(), // leader + Lists.newArrayList(Node.noNode()), // replicas + Lists.newArrayList(Node.noNode()), // isr + Collections.emptyList() // offline replicas + ); + } + + static AbstractResponse failedResponse(KafkaHeaderAndRequest requestHar, Throwable e) { + if (log.isDebugEnabled()) { + log.debug("Request {} get failed response ", requestHar.getHeader().apiKey(), e); + } + return requestHar.getRequest().getErrorResponse(((Integer) THROTTLE_TIME_MS.defaultValue), e); + } + protected CompletableFuture handleApiVersionsRequest(KafkaHeaderAndRequest apiVersionRequest) { AbstractResponse apiResponse = overloadDefaultApiVersionsResponse(); CompletableFuture resultFuture = new CompletableFuture<>(); @@ -183,11 +241,13 @@ protected ApiVersionsResponse overloadDefaultApiVersionsResponse() { switch (apiKey) { case FETCH: // V4 added MessageSets responses. We need to make sure RecordBatch format is not used - versionList.add(new ApiVersionsResponse.ApiVersion((short) 1, (short) 4, apiKey.latestVersion())); + versionList.add(new ApiVersionsResponse.ApiVersion( + (short) 1, (short) 4, apiKey.latestVersion())); break; case LIST_OFFSETS: // V0 is needed for librdkafka - versionList.add(new ApiVersionsResponse.ApiVersion((short) 2, (short) 0, apiKey.latestVersion())); + versionList.add(new ApiVersionsResponse.ApiVersion( + (short) 2, (short) 0, apiKey.latestVersion())); break; default: versionList.add(new ApiVersionsResponse.ApiVersion(apiKey)); @@ -406,8 +466,8 @@ protected CompletableFuture handleTopicMetadataRequest(Kafka Errors.NONE, // we should answer with the right name, either local of full-name, // depending on what was asked - topic.startsWith("persistent://") ? - TopicName.get(topic).toString(): TopicName.get(topic).getLocalName(), + topic.startsWith("persistent://") + ? TopicName.get(topic).toString() : TopicName.get(topic).getLocalName(), false, partitionMetadatas)); @@ -618,7 +678,8 @@ protected CompletableFuture handleOffsetFetchRequest(KafkaHe if (legacyMode) { partitionData.complete(new ListOffsetResponse.PartitionData( Errors.NONE, - Collections.singletonList(MessageIdUtils.getOffset(position.getLedgerId(), position.getEntryId())))); + Collections.singletonList(MessageIdUtils.getOffset( + position.getLedgerId(), position.getEntryId())))); } else { partitionData.complete(new ListOffsetResponse.PartitionData( Errors.NONE, @@ -665,7 +726,8 @@ public void findEntryComplete(Position position, Object ctx) { if (legacyMode) { partitionData.complete(new ListOffsetResponse.PartitionData( Errors.NONE, - Collections.singletonList(MessageIdUtils.getOffset(finalPosition.getLedgerId(), finalPosition.getEntryId())))); + Collections.singletonList(MessageIdUtils.getOffset( + finalPosition.getLedgerId(), finalPosition.getEntryId())))); } else { partitionData.complete(new ListOffsetResponse.PartitionData( Errors.NONE, @@ -714,8 +776,9 @@ public void findEntryFailed(ManagedLedgerException exception, return partitionData; } - - // Some info can be found here https://web.archive.org/web/20170309152525/https://cfchou.github.io/blog/2015/04/23/a-closer-look-at-kafka-offsetrequest/ + // Some info can be found here: + // https://cfchou.github.io/blog/2015/04/23/a-closer-look-at-kafka-offsetrequest/ + // Site is available through https://web.archive.org/ private CompletableFuture handleListOffsetRequestV0(KafkaHeaderAndRequest listOffset) { ListOffsetRequest request = (ListOffsetRequest) listOffset.getRequest(); @@ -733,7 +796,8 @@ private CompletableFuture handleListOffsetRequestV0(KafkaHea // num_num_offsets > 1 is not handled for now, returning an error if (tms.getValue().maxNumOffsets > 1) { - log.warn("request is asking for multiples offsets for {}, not supported for now", pulsarTopic.toString()); + log.warn("request is asking for multiples offsets for {}, not supported for now", + pulsarTopic.toString()); partitionData = new CompletableFuture<>(); partitionData.complete(new ListOffsetResponse .PartitionData( @@ -1288,16 +1352,6 @@ private CompletableFuture findBroker(PulsarService pulsarServ } } - static Node newNode(InetSocketAddress address) { - if (log.isDebugEnabled()) { - log.debug("Return Broker Node of {}. {}:{}", address, address.getHostString(), address.getPort()); - } - return new Node( - Murmur3_32Hash.getInstance().makeHash((address.getHostString() + address.getPort()).getBytes(UTF_8)), - address.getHostString(), - address.getPort()); - } - Node newSelfNode() { String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress( kafkaConfig.getAdvertisedAddress()); @@ -1313,46 +1367,4 @@ Node newSelfNode() { hostname, port); } - - - static PartitionMetadata newPartitionMetadata(TopicName topicName, Node node) { - int pulsarPartitionIndex = topicName.getPartitionIndex(); - int kafkaPartitionIndex = pulsarPartitionIndex == -1 ? 0 : pulsarPartitionIndex; - - if (log.isDebugEnabled()) { - log.debug("Return PartitionMetadata node: {}, topicName: {}", node, topicName); - } - - return new PartitionMetadata( - Errors.NONE, - kafkaPartitionIndex, - node, // leader - Lists.newArrayList(node), // replicas - Lists.newArrayList(node), // isr - Collections.emptyList() // offline replicas - ); - } - - static PartitionMetadata newFailedPartitionMetadata(TopicName topicName) { - int pulsarPartitionIndex = topicName.getPartitionIndex(); - int kafkaPartitionIndex = pulsarPartitionIndex == -1 ? 0 : pulsarPartitionIndex; - - log.warn("Failed find Broker metadata, create PartitionMetadata with INVALID_PARTITIONS"); - - return new PartitionMetadata( - Errors.UNKNOWN_SERVER_ERROR, - kafkaPartitionIndex, - Node.noNode(), // leader - Lists.newArrayList(Node.noNode()), // replicas - Lists.newArrayList(Node.noNode()), // isr - Collections.emptyList() // offline replicas - ); - } - - static AbstractResponse failedResponse(KafkaHeaderAndRequest requestHar, Throwable e) { - if (log.isDebugEnabled()) { - log.debug("Request {} get failed response ", requestHar.getHeader().apiKey(), e); - } - return requestHar.getRequest().getErrorResponse(((Integer) THROTTLE_TIME_MS.defaultValue), e); - } } diff --git a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java index be8141c08d..459572def4 100644 --- a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java +++ b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java @@ -1,6 +1,25 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.streamnative.kop; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertTrue; + import com.google.common.collect.Sets; +import java.nio.file.Paths; +import java.util.Optional; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.RetentionPolicies; @@ -16,19 +35,33 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import java.nio.file.Paths; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -import static org.testng.AssertJUnit.assertFalse; -import static org.testng.AssertJUnit.assertTrue; - +/** + * This class is running Integration tests for Kafka clients. + * It uses testcontainers to spawn containers that will either: + * * produce a number of messages + * * consume a number of messages + * + *

As testcontainers is not capable of checking exitCode of the app running in the container, + * Every container should print the exitCode in stdout. + * + *

This class is waiting for some precise logs to come-up: + * * "ready to produce" + * * "ready to consume" + * * "produced all messages successfully" + * * "consumed all messages successfully" + * + *

This class is using environment variables to control the containers, such as: + * * broker address, + * * topic name, + * * produce or consume mode, + * * how many message to produce/consume, + */ @Slf4j public class KafkaIntegrationTest extends MockKafkaServiceBaseTest { @DataProvider public static Object[][] integrations() { - return new Object[][] { + return new Object[][]{ {"golang-sarama", Optional.empty(), true, true}, {"golang-sarama", Optional.of("persistent://public/default/my-sarama-topic-full-name"), true, true}, {"golang-confluent-kafka", Optional.empty(), true, true}, @@ -39,6 +72,29 @@ public static Object[][] integrations() { }; } + private static WaitingConsumer createLogFollower(final GenericContainer container) { + final WaitingConsumer waitingConsumer = new WaitingConsumer(); + container.followOutput(waitingConsumer); + return waitingConsumer; + } + + private static void checkForErrorsInLogs(final String logs) { + assertFalse(logs.contains("no available broker to send metadata request to")); + assertFalse(logs.contains("panic")); + assertFalse(logs.contains("correlation ID didn't match")); + assertFalse(logs.contains("Required feature not supported by broker")); + + + if (logs.contains("starting to produce")) { + assertTrue(logs.contains("produced all messages successfully")); + } + + if (logs.contains("starting to consume")) { + assertTrue(logs.contains("consumed all messages successfully")); + } + assertTrue(logs.contains("ExitCode=0")); + } + @BeforeClass @Override protected void setup() throws Exception { @@ -46,61 +102,62 @@ protected void setup() throws Exception { super.resetConfig(); super.internalSetup(); - if (!admin.clusters().getClusters().contains(configClusterName)) { + if (!this.admin.clusters().getClusters().contains(this.configClusterName)) { // so that clients can test short names - admin.clusters().createCluster(configClusterName, - new ClusterData("http://127.0.0.1:" + brokerWebservicePort)); + this.admin.clusters().createCluster(this.configClusterName, + new ClusterData("http://127.0.0.1:" + this.brokerWebservicePort)); } else { - admin.clusters().updateCluster(configClusterName, - new ClusterData("http://127.0.0.1:" + brokerWebservicePort)); + this.admin.clusters().updateCluster(this.configClusterName, + new ClusterData("http://127.0.0.1:" + this.brokerWebservicePort)); } - if (!admin.tenants().getTenants().contains("public")) { - admin.tenants().createTenant("public", + if (!this.admin.tenants().getTenants().contains("public")) { + this.admin.tenants().createTenant("public", new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); } else { - admin.tenants().updateTenant("public", + this.admin.tenants().updateTenant("public", new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); } - if (!admin.namespaces().getNamespaces("public").contains("public/default")) { - admin.namespaces().createNamespace("public/default"); - admin.namespaces().setNamespaceReplicationClusters("public/default", Sets.newHashSet("test")); - admin.namespaces().setRetention("public/default", + if (!this.admin.namespaces().getNamespaces("public").contains("public/default")) { + this.admin.namespaces().createNamespace("public/default"); + this.admin.namespaces().setNamespaceReplicationClusters("public/default", Sets.newHashSet("test")); + this.admin.namespaces().setRetention("public/default", new RetentionPolicies(60, 1000)); } - if (!admin.namespaces().getNamespaces("public").contains("public/__kafka")) { - admin.namespaces().createNamespace("public/__kafka"); - admin.namespaces().setNamespaceReplicationClusters("public/__kafka", Sets.newHashSet("test")); - admin.namespaces().setRetention("public/__kafka", + if (!this.admin.namespaces().getNamespaces("public").contains("public/__kafka")) { + this.admin.namespaces().createNamespace("public/__kafka"); + this.admin.namespaces().setNamespaceReplicationClusters("public/__kafka", Sets.newHashSet("test")); + this.admin.namespaces().setRetention("public/__kafka", new RetentionPolicies(-1, -1)); } Testcontainers.exposeHostPorts(ImmutableMap.of(super.kafkaBrokerPort, super.kafkaBrokerPort)); } - @Test(timeOut = 5 * 60_000, dataProvider = "integrations") - void simpleProduceAndConsume(String integration, Optional topic, boolean shouldProduce, boolean shouldConsume) throws Exception { + @Test(timeOut = 3 * 60_000, dataProvider = "integrations") + void simpleProduceAndConsume(final String integration, final Optional topic, + final boolean shouldProduce, final boolean shouldConsume) throws Exception { - getAdmin().topics().createPartitionedTopic(topic.orElse(integration), 1); + this.getAdmin().topics().createPartitionedTopic(topic.orElse(integration), 1); - GenericContainer producer = new GenericContainer<>( + final GenericContainer producer = new GenericContainer<>( new ImageFromDockerfile().withFileFromPath(".", Paths.get("integrations/" + integration))) .withEnv("KOP_BROKER", "localhost:" + super.kafkaBrokerPort) .withEnv("KOP_PRODUCE", "true") .withEnv("KOP_TOPIC", topic.orElse(integration)) .withEnv("KOP_NBR_MESSAGES", "10") .withEnv("KOP_EXPECT_MESSAGES", "10") - .withLogConsumer(new org.testcontainers.containers.output.Slf4jLogConsumer(log)) + .withLogConsumer(new org.testcontainers.containers.output.Slf4jLogConsumer(KafkaIntegrationTest.log)) .waitingFor(Wait.forLogMessage("starting to produce\\n", 1)) .withNetworkMode("host"); - GenericContainer consumer = new GenericContainer<>( + final GenericContainer consumer = new GenericContainer<>( new ImageFromDockerfile().withFileFromPath(".", Paths.get("integrations/" + integration))) .withEnv("KOP_BROKER", "localhost:" + super.kafkaBrokerPort) .withEnv("KOP_TOPIC", topic.orElse(integration)) .withEnv("KOP_CONSUME", "true") .withEnv("KOP_NBR_MESSAGES", "10") .withEnv("KOP_EXPECT_MESSAGES", "10") - .withLogConsumer(new org.testcontainers.containers.output.Slf4jLogConsumer(log)) + .withLogConsumer(new org.testcontainers.containers.output.Slf4jLogConsumer(KafkaIntegrationTest.log)) .waitingFor(Wait.forLogMessage("starting to consume\\n", 1)) .withNetworkMode("host"); @@ -108,26 +165,26 @@ void simpleProduceAndConsume(String integration, Optional topic, boolean WaitingConsumer consumerWaitingConsumer = null; if (shouldProduce) { producer.start(); - producerWaitingConsumer = createLogFollower(producer); + producerWaitingConsumer = KafkaIntegrationTest.createLogFollower(producer); System.out.println("producer started"); } if (shouldConsume) { consumer.start(); - consumerWaitingConsumer = createLogFollower(consumer); + consumerWaitingConsumer = KafkaIntegrationTest.createLogFollower(consumer); System.out.println("consumer started"); } if (shouldProduce) { producerWaitingConsumer.waitUntil(frame -> frame.getUtf8String().contains("ExitCode"), 30, TimeUnit.SECONDS); - checkForErrorsInLogs(producer.getLogs()); + KafkaIntegrationTest.checkForErrorsInLogs(producer.getLogs()); } if (shouldConsume) { consumerWaitingConsumer.waitUntil(frame -> frame.getUtf8String().contains("ExitCode"), 30, TimeUnit.SECONDS); - checkForErrorsInLogs(consumer.getLogs()); + KafkaIntegrationTest.checkForErrorsInLogs(consumer.getLogs()); } } @@ -136,29 +193,4 @@ void simpleProduceAndConsume(String integration, Optional topic, boolean protected void cleanup() throws Exception { super.internalCleanup(); } - - - private WaitingConsumer createLogFollower(GenericContainer container) { - WaitingConsumer waitingConsumer = new WaitingConsumer(); - container.followOutput(waitingConsumer); - return waitingConsumer; - } - - private void checkForErrorsInLogs(String logs) { - assertFalse(logs.contains("no available broker to send metadata request to")); - assertFalse(logs.contains("panic")); - assertFalse(logs.contains("correlation ID didn't match")); - assertFalse(logs.contains("Required feature not supported by broker")); - - - if (logs.contains("starting to produce")) { - assertTrue(logs.contains("produced all messages successfully")); - } - - if (logs.contains("starting to consume")) { - assertTrue(logs.contains("received msg")); - assertTrue(logs.contains("limit reached, exiting")); - } - assertTrue(logs.contains("ExitCode=0")); - } } From e59c31edffbeb040b0f89077a5c7916f175c3d55 Mon Sep 17 00:00:00 2001 From: Pierre Zemb Date: Wed, 8 Jan 2020 11:19:12 +0100 Subject: [PATCH 10/13] Refactor env vars to simplify describing a limit --- integrations/golang-confluent-kafka/main.go | 10 +++------- integrations/golang-sarama/main.go | 10 +++------- integrations/rustlang-rdkafka/src/main.rs | 14 +++++--------- .../io/streamnative/kop/KafkaIntegrationTest.java | 6 ++---- 4 files changed, 13 insertions(+), 27 deletions(-) diff --git a/integrations/golang-confluent-kafka/main.go b/integrations/golang-confluent-kafka/main.go index d71828603f..6763ea60ad 100644 --- a/integrations/golang-confluent-kafka/main.go +++ b/integrations/golang-confluent-kafka/main.go @@ -25,11 +25,7 @@ import ( ) func main() { - nbrMessages, err := strconv.Atoi(getEnv("KOP_NBR_MESSAGES", "10")) - if err != nil { - panic(err) - } - limit, err := strconv.Atoi(getEnv("KOP_EXPECT_MESSAGES", "10")) + limit, err := strconv.Atoi(getEnv("KOP_LIMIT", "10")) if err != nil { panic(err) } @@ -72,7 +68,7 @@ func main() { } }() - for i := 0; i < nbrMessages; i++ { + for i := 0; i < limit; i++ { p.Produce(&kafka.Message{ TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny}, Value: []byte("hello from confluent go"), @@ -80,7 +76,7 @@ func main() { fmt.Println("send a message") } - fmt.Printf("produced all messages successfully (%d) \n", nbrMessages) + fmt.Printf("produced all messages successfully (%d) \n", limit) // Wait for message deliveries before shutting down p.Flush(15 * 1000) diff --git a/integrations/golang-sarama/main.go b/integrations/golang-sarama/main.go index 7b7db591bc..c5290c47ef 100644 --- a/integrations/golang-sarama/main.go +++ b/integrations/golang-sarama/main.go @@ -50,11 +50,7 @@ func (h exampleConsumerGroupHandler) ConsumeClaim(sess sarama.ConsumerGroupSessi func main() { - nbrMessages, err := strconv.Atoi(getEnv("KOP_NBR_MESSAGES", "10")) - if err != nil { - panic(err) - } - limit, err := strconv.Atoi(getEnv("KOP_EXPECT_MESSAGES", "10")) + limit, err := strconv.Atoi(getEnv("KOP_LIMIT", "10")) if err != nil { panic(err) } @@ -123,7 +119,7 @@ func main() { fmt.Println("starting to produce") - for i := 0; i < nbrMessages; i++ { + for i := 0; i < limit; i++ { msg := &sarama.ProducerMessage{ Topic: topic, Value: sarama.StringEncoder("hello from sarama"), @@ -137,7 +133,7 @@ func main() { panic(err) } } - fmt.Printf("produced all messages successfully (%d) \n", nbrMessages) + fmt.Printf("produced all messages successfully (%d) \n", limit) } waitgroup.Wait() diff --git a/integrations/rustlang-rdkafka/src/main.rs b/integrations/rustlang-rdkafka/src/main.rs index 91d7f08394..f0608c57d4 100644 --- a/integrations/rustlang-rdkafka/src/main.rs +++ b/integrations/rustlang-rdkafka/src/main.rs @@ -33,11 +33,7 @@ use std::env; fn main() { let brokers = env::var("KOP_BROKER").unwrap_or_else(|_| String::from("localhost:9092")); let topic = env::var("KOP_TOPIC").unwrap_or_else(|_| String::from("rustlang")); - let limit: i8 = env::var("KOP_EXPECT_MESSAGES") - .unwrap_or_else(|_| String::from("10")) - .parse() - .unwrap_or(10); - let nbr_messages_produced: i8 = env::var("KOP_NBR_MESSAGES") + let limit: i8 = env::var("KOP_LIMIT") .unwrap_or_else(|_| String::from("10")) .parse() .unwrap_or(10); @@ -57,7 +53,7 @@ fn main() { if should_produce { println!("starting to produce"); - produce(brokers.as_ref(), topic.as_str(), nbr_messages_produced); + produce(brokers.as_ref(), topic.as_str(), limit); } if should_consume { println!("starting to consume"); @@ -70,7 +66,7 @@ fn main() { } println!("exiting normally"); } -fn produce(brokers: &str, topic_name: &str, nbr_messages_produced: i8) { +fn produce(brokers: &str, topic_name: &str, limit: i8) { println!("starting to produce"); let producer: FutureProducer = ClientConfig::new() .set("bootstrap.servers", brokers) @@ -80,7 +76,7 @@ fn produce(brokers: &str, topic_name: &str, nbr_messages_produced: i8) { // This loop is non blocking: all messages will be sent one after the other, without waiting // for the results. - let futures = (0..nbr_messages_produced) + let futures = (0..limit) .map(|i| { // The send operation on the topic returns a future, that will be completed once the // result or failure from Kafka will be received. @@ -106,7 +102,7 @@ fn produce(brokers: &str, topic_name: &str, nbr_messages_produced: i8) { } println!( "produced all messages successfully ({})", - nbr_messages_produced + limit ) } diff --git a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java index 459572def4..968c914ed8 100644 --- a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java +++ b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java @@ -144,8 +144,7 @@ void simpleProduceAndConsume(final String integration, final Optional to .withEnv("KOP_BROKER", "localhost:" + super.kafkaBrokerPort) .withEnv("KOP_PRODUCE", "true") .withEnv("KOP_TOPIC", topic.orElse(integration)) - .withEnv("KOP_NBR_MESSAGES", "10") - .withEnv("KOP_EXPECT_MESSAGES", "10") + .withEnv("KOP_LIMIT", "10") .withLogConsumer(new org.testcontainers.containers.output.Slf4jLogConsumer(KafkaIntegrationTest.log)) .waitingFor(Wait.forLogMessage("starting to produce\\n", 1)) .withNetworkMode("host"); @@ -155,8 +154,7 @@ void simpleProduceAndConsume(final String integration, final Optional to .withEnv("KOP_BROKER", "localhost:" + super.kafkaBrokerPort) .withEnv("KOP_TOPIC", topic.orElse(integration)) .withEnv("KOP_CONSUME", "true") - .withEnv("KOP_NBR_MESSAGES", "10") - .withEnv("KOP_EXPECT_MESSAGES", "10") + .withEnv("KOP_LIMIT", "10") .withLogConsumer(new org.testcontainers.containers.output.Slf4jLogConsumer(KafkaIntegrationTest.log)) .waitingFor(Wait.forLogMessage("starting to consume\\n", 1)) .withNetworkMode("host"); From 6c8d568d2b940e8a2cbecdd852f5482ebe70ad70 Mon Sep 17 00:00:00 2001 From: Pierre Zemb Date: Wed, 8 Jan 2020 11:41:47 +0100 Subject: [PATCH 11/13] mvn license:format --- integrations/golang-confluent-kafka/Dockerfile | 14 ++++++++++++++ integrations/golang-confluent-kafka/main.go | 14 ++++++++++++++ integrations/golang-sarama/Dockerfile | 14 ++++++++++++++ integrations/golang-sarama/main.go | 14 ++++++++++++++ integrations/node-kafka-node/Dockerfile | 14 ++++++++++++++ integrations/node-kafka-node/main.js | 13 +++++++++++++ integrations/node-rdkafka/Dockerfile | 14 ++++++++++++++ integrations/node-rdkafka/main.js | 13 +++++++++++++ integrations/rustlang-rdkafka/Dockerfile | 14 ++++++++++++++ .../io/streamnative/kop/KafkaIntegrationTest.java | 13 +++++++++++++ 10 files changed, 137 insertions(+) diff --git a/integrations/golang-confluent-kafka/Dockerfile b/integrations/golang-confluent-kafka/Dockerfile index be8e224df8..08e203df84 100644 --- a/integrations/golang-confluent-kafka/Dockerfile +++ b/integrations/golang-confluent-kafka/Dockerfile @@ -1,3 +1,17 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/integrations/golang-confluent-kafka/main.go b/integrations/golang-confluent-kafka/main.go index 6763ea60ad..e3941f4668 100644 --- a/integrations/golang-confluent-kafka/main.go +++ b/integrations/golang-confluent-kafka/main.go @@ -1,3 +1,17 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/integrations/golang-sarama/Dockerfile b/integrations/golang-sarama/Dockerfile index d15800a409..4a10bc2280 100644 --- a/integrations/golang-sarama/Dockerfile +++ b/integrations/golang-sarama/Dockerfile @@ -1,3 +1,17 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/integrations/golang-sarama/main.go b/integrations/golang-sarama/main.go index c5290c47ef..ce24337472 100644 --- a/integrations/golang-sarama/main.go +++ b/integrations/golang-sarama/main.go @@ -1,3 +1,17 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/integrations/node-kafka-node/Dockerfile b/integrations/node-kafka-node/Dockerfile index b0b5863f3b..11092a9727 100644 --- a/integrations/node-kafka-node/Dockerfile +++ b/integrations/node-kafka-node/Dockerfile @@ -1,3 +1,17 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/integrations/node-kafka-node/main.js b/integrations/node-kafka-node/main.js index 15793ca121..0ab0e411f6 100644 --- a/integrations/node-kafka-node/main.js +++ b/integrations/node-kafka-node/main.js @@ -1,3 +1,16 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ 'use strict'; diff --git a/integrations/node-rdkafka/Dockerfile b/integrations/node-rdkafka/Dockerfile index b357c1d231..7187ebaf08 100644 --- a/integrations/node-rdkafka/Dockerfile +++ b/integrations/node-rdkafka/Dockerfile @@ -1,3 +1,17 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/integrations/node-rdkafka/main.js b/integrations/node-rdkafka/main.js index 216c973973..42f7292dcb 100644 --- a/integrations/node-rdkafka/main.js +++ b/integrations/node-rdkafka/main.js @@ -1,3 +1,16 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/integrations/rustlang-rdkafka/Dockerfile b/integrations/rustlang-rdkafka/Dockerfile index 994953741c..7370767088 100644 --- a/integrations/rustlang-rdkafka/Dockerfile +++ b/integrations/rustlang-rdkafka/Dockerfile @@ -1,3 +1,17 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java index 968c914ed8..a4bf956161 100644 --- a/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java +++ b/src/test/java/io/streamnative/kop/KafkaIntegrationTest.java @@ -1,3 +1,16 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 6987810a75065ebf1dd57cb5007cc8d7e1e94189 Mon Sep 17 00:00:00 2001 From: Pierre Zemb Date: Thu, 16 Jan 2020 10:15:31 +0100 Subject: [PATCH 12/13] Bump rust-rdkafka version --- integrations/rustlang-rdkafka/Cargo.lock | 266 +++++++++++++++++++++- integrations/rustlang-rdkafka/Cargo.toml | 6 +- integrations/rustlang-rdkafka/src/main.rs | 59 ++--- 3 files changed, 289 insertions(+), 42 deletions(-) diff --git a/integrations/rustlang-rdkafka/Cargo.lock b/integrations/rustlang-rdkafka/Cargo.lock index cbc41472ab..66ce8efc50 100644 --- a/integrations/rustlang-rdkafka/Cargo.lock +++ b/integrations/rustlang-rdkafka/Cargo.lock @@ -1,5 +1,10 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "bytes" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "cc" version = "1.0.48" @@ -18,10 +23,97 @@ dependencies = [ "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "derivative" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "futures" -version = "0.1.29" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-executor 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-channel" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-executor" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-io" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-macro" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-sink" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-task" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-macro 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "itoa" @@ -52,11 +144,77 @@ dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "memchr" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "num_enum" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "derivative 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "num_enum_derive 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num_enum_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "pin-utils" +version = "0.1.0-alpha.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "pkg-config" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "proc-macro-crate" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "toml 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro-nested" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "proc-macro2" version = "1.0.6" @@ -65,6 +223,14 @@ dependencies = [ "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "quote" version = "1.0.2" @@ -75,13 +241,13 @@ dependencies = [ [[package]] name = "rdkafka" -version = "0.22.0" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "rdkafka-sys 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rdkafka-sys 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)", @@ -89,12 +255,13 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "1.2.2" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cmake 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "num_enum 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -102,14 +269,14 @@ dependencies = [ name = "rustlang-rdkafka" version = "0.1.0" dependencies = [ - "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "rdkafka 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rdkafka-sys 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rdkafka 0.23.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -121,6 +288,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "serde" version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "serde_derive" @@ -142,6 +312,21 @@ dependencies = [ "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "syn" version = "1.0.11" @@ -152,6 +337,39 @@ dependencies = [ "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tokio" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project-lite 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-macros 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "toml" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unicode-xid" version = "0.2.0" @@ -163,23 +381,49 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] +"checksum bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "10004c15deb332055f7a4a208190aed362cf9a7c2f6ab70a305fba50e1105f38" "checksum cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)" = "f52a465a666ca3d838ebbf08b241383421412fe7ebb463527bba275526d89f76" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" "checksum cmake 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "81fb25b677f8bf1eb325017cb6bb8452f87969db0fedb4f757b297bee78a7c62" -"checksum futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" +"checksum derivative 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "942ca430eef7a3806595a6737bc388bf51adb888d3fc0dd1b50f1c170167ee3a" +"checksum futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b6f16056ecbb57525ff698bb955162d0cd03bee84e6241c27ff75c08d8ca5987" +"checksum futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fcae98ca17d102fd8a3603727b9259fcf7fa4239b603d2142926189bc8999b86" +"checksum futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "79564c427afefab1dfb3298535b21eda083ef7935b4f0ecbfcb121f0aec10866" +"checksum futures-executor 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1e274736563f686a837a0568b478bdabfeaec2dca794b5649b04e2fe1627c231" +"checksum futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e676577d229e70952ab25f3945795ba5b16d63ca794ca9d2c860e5595d20b5ff" +"checksum futures-macro 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "52e7c56c15537adb4f76d0b7a76ad131cb4d2f4f32d3b0bcabcbe1c7c5e87764" +"checksum futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "171be33efae63c2d59e6dbba34186fe0d6394fb378069a76dfd80fdcffd43c16" +"checksum futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bae52d6b29cf440e298856fec3965ee6fa71b06aa7495178615953fd669e5f9" +"checksum futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d66274fb76985d3c62c886d1da7ac4c0903a8c9f754e8fe0f35a6a6cc39e76" "checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f" "checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" "checksum libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +"checksum memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223" +"checksum num_enum 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "be601e38e20a6f3d01049d85801cb9b7a34a8da7a0da70df507bbde7735058c8" +"checksum num_enum_derive 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b59f30f6a043f2606adbd0addbf1eef6f2e28e8c4968918b63b7ff97ac0db2a7" +"checksum pin-project-lite 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e8822eb8bb72452f038ebf6048efa02c3fe22bf83f76519c9583e47fc194a422" +"checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" "checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" +"checksum proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e" +"checksum proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" +"checksum proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" +"checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" "checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27" +"checksum quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" "checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" -"checksum rdkafka 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9c39cf211c5e933622b600e892ae6fa7bc9e08cb56c4129424e07d91cc3c9088" -"checksum rdkafka-sys 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "78d4b56497b669efc00f1741cc39763e94f712f07817dfc3d96181fe41b535ea" +"checksum rdkafka 0.23.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d455ac2a07a27d87b4f0e321dfd8d9a5b574bd96f55e42e6c594712a08051222" +"checksum rdkafka-sys 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7d770343fbbc6089c750000711a17a906e8b3f7831afcd752d9667d38833e578" "checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" "checksum serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)" = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" "checksum serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)" = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" "checksum serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)" = "48c575e0cc52bdd09b47f330f646cf59afc586e9c4e3ccd6fc1f625b8ea1dad7" +"checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +"checksum syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)" = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" "checksum syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238" +"checksum tokio 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "ffa2fdcfa937b20cb3c822a635ceecd5fc1a27a6a474527e5516aa24b8c8820a" +"checksum tokio-macros 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "50a61f268a3db2acee8dcab514efc813dc6dbe8a00e86076f935f94304b59a7a" +"checksum toml 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "01d1404644c8b12b16bfcffa4322403a91a451584daaaa7c28d3152e6cbc98cf" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" "checksum vcpkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168" diff --git a/integrations/rustlang-rdkafka/Cargo.toml b/integrations/rustlang-rdkafka/Cargo.toml index 3496eb2381..8917ec8c1f 100644 --- a/integrations/rustlang-rdkafka/Cargo.toml +++ b/integrations/rustlang-rdkafka/Cargo.toml @@ -7,9 +7,9 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -rdkafka = { version = "0.22", features = ["cmake-build"] } -rdkafka-sys = { version = "1.2.1", default-features = false } -futures = "0.1.21" +rdkafka = { version = "0.23.1", features = ["cmake-build"] } +futures = "0.3" +tokio = { version = "0.2", features = ["blocking", "macros", "rt-core", "time"] } libc = "0.2.0" log = "0.4.8" serde = "1.0.0" diff --git a/integrations/rustlang-rdkafka/src/main.rs b/integrations/rustlang-rdkafka/src/main.rs index f0608c57d4..145260d07c 100644 --- a/integrations/rustlang-rdkafka/src/main.rs +++ b/integrations/rustlang-rdkafka/src/main.rs @@ -1,3 +1,4 @@ +extern crate futures; // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,23 +15,26 @@ // #[macro_use] extern crate log; -extern crate futures; extern crate rdkafka; +extern crate tokio; + +use std::env; -use crate::futures::Future; -use futures::Stream; +use futures::StreamExt; +use rdkafka::ClientContext; use rdkafka::config::{ClientConfig, RDKafkaLogLevel}; -use rdkafka::consumer::stream_consumer::StreamConsumer; use rdkafka::consumer::{CommitMode, Consumer, ConsumerContext, Rebalance}; +use rdkafka::consumer::stream_consumer::StreamConsumer; use rdkafka::error::KafkaResult; -use rdkafka::message::OwnedHeaders; use rdkafka::message::{Headers, Message}; +use rdkafka::message::OwnedHeaders; use rdkafka::producer::{FutureProducer, FutureRecord}; -use rdkafka::ClientContext; -use rdkafka_sys; -use std::env; +use rdkafka::topic_partition_list::TopicPartitionList; + +use crate::futures::FutureExt; -fn main() { +#[tokio::main] +async fn main() -> Result<(), std::io::Error> { let brokers = env::var("KOP_BROKER").unwrap_or_else(|_| String::from("localhost:9092")); let topic = env::var("KOP_TOPIC").unwrap_or_else(|_| String::from("rustlang")); let limit: i8 = env::var("KOP_LIMIT") @@ -53,7 +57,7 @@ fn main() { if should_produce { println!("starting to produce"); - produce(brokers.as_ref(), topic.as_str(), limit); + produce(brokers.as_ref(), topic.as_str(), limit).await?; } if should_consume { println!("starting to consume"); @@ -62,11 +66,14 @@ fn main() { "rustlang-librdkafka", vec![topic.as_str()], limit, - ); + ).await?; } println!("exiting normally"); + + Ok(()) } -fn produce(brokers: &str, topic_name: &str, limit: i8) { + +async fn produce(brokers: &str, topic_name: &str, limit: i8) -> Result<(), std::io::Error> { println!("starting to produce"); let producer: FutureProducer = ClientConfig::new() .set("bootstrap.servers", brokers) @@ -98,12 +105,13 @@ fn produce(brokers: &str, topic_name: &str, limit: i8) { // This loop will wait until all delivery statuses have been received received. for future in futures { - println!("Future completed. Result: {:?}", future.wait()); + info!("Future completed. Result: {:?}", future.await); } println!( "produced all messages successfully ({})", limit - ) + ); + Ok(()) } // A context can be used to change the behavior of producers and consumers by adding callbacks @@ -122,11 +130,7 @@ impl ConsumerContext for CustomContext { println!("Post rebalance {:?}", rebalance); } - fn commit_callback( - &self, - result: KafkaResult<()>, - _offsets: *mut rdkafka_sys::RDKafkaTopicPartitionList, - ) { + fn commit_callback(&self, result: KafkaResult<()>, _offsets: &TopicPartitionList) { info!("Committing offsets: {:?}", result); } } @@ -134,7 +138,7 @@ impl ConsumerContext for CustomContext { // A type alias with your custom consumer can be created for convenience. type LoggingConsumer = StreamConsumer; -fn consume_and_print(brokers: &str, group_id: &str, topics: Vec<&str>, limit: i8) { +async fn consume_and_print(brokers: &str, group_id: &str, topics: Vec<&str>, limit: i8) -> Result<(), std::io::Error> { let context = CustomContext; let consumer: LoggingConsumer = ClientConfig::new() @@ -155,25 +159,23 @@ fn consume_and_print(brokers: &str, group_id: &str, topics: Vec<&str>, limit: i8 // consumer.start() returns a stream. The stream can be used ot chain together expensive steps, // such as complex computations on a thread pool or asynchronous IO. - let message_stream = consumer.start(); + let mut message_stream = consumer.start(); let mut i = 0; - for message in message_stream.wait() { + while let Some(message) = message_stream.next().await { match message { - Err(_) => panic!("Error while reading from stream."), - Ok(Err(e)) => panic!("Kafka error: {}", e), - Ok(Ok(m)) => { + Err(e) => panic!("Kafka error: {}", e), + Ok(m) => { let payload = match m.payload_view::() { None => "", Some(Ok(s)) => s, Some(Err(e)) => { panic!("Error while deserializing message payload: {:?}", e); - "" } }; println!("key: '{:?}', payload: '{}', topic: {}, partition: {}, offset: {}, timestamp: {:?}", - m.key(), payload, m.topic(), m.partition(), m.offset(), m.timestamp()); + m.key(), payload, m.topic(), m.partition(), m.offset(), m.timestamp()); if let Some(headers) = m.headers() { for i in 0..headers.count() { let header = headers.get(i).unwrap(); @@ -185,9 +187,10 @@ fn consume_and_print(brokers: &str, group_id: &str, topics: Vec<&str>, limit: i8 println!("received msg"); if i == limit { println!("consumed all messages successfully"); - return; + return Ok(()); } } }; } + Ok(()) } From 3f58072d4fa2f233fb7ac77dc69940e2e2ee4ab5 Mon Sep 17 00:00:00 2001 From: Pierre Zemb Date: Thu, 16 Jan 2020 10:45:58 +0100 Subject: [PATCH 13/13] rust-integration: Ensure the produce future is polled --- integrations/rustlang-rdkafka/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/rustlang-rdkafka/src/main.rs b/integrations/rustlang-rdkafka/src/main.rs index 145260d07c..0f0620bd8f 100644 --- a/integrations/rustlang-rdkafka/src/main.rs +++ b/integrations/rustlang-rdkafka/src/main.rs @@ -105,7 +105,7 @@ async fn produce(brokers: &str, topic_name: &str, limit: i8) -> Result<(), std:: // This loop will wait until all delivery statuses have been received received. for future in futures { - info!("Future completed. Result: {:?}", future.await); + println!("Future completed. Result: {:?}", future.await); } println!( "produced all messages successfully ({})",