diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index a244e15f94237..026c1cca7aae8 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -95,5 +95,11 @@ HdrHistogram + + com.fasterxml.jackson.core + jackson-databind + + + diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/DefaultMessageFormatter.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/DefaultMessageFormatter.java new file mode 100644 index 0000000000000..a84fb2d7e6f4d --- /dev/null +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/DefaultMessageFormatter.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.testclient; + +import java.nio.charset.StandardCharsets; +import java.util.Random; + +public class DefaultMessageFormatter implements IMessageFormatter { + Random r = new Random(); + + + @Override + public byte[] formatMessage(String producerName, long msgId, byte[] message) { + String sMessage = new String(message, StandardCharsets.UTF_8); + if (producerName != null && !producerName.isEmpty()) { + sMessage = sMessage.replaceAll("%p", producerName); + } + sMessage = sMessage.replaceAll("%i", String.valueOf(msgId)); + sMessage = sMessage.replaceAll("%t", String.valueOf(System.nanoTime())); + + int idx = sMessage.indexOf("%"); + while (idx > 0) { + + float size = 0; + int i=1; + for (; idx+i < sMessage.length(); i++) { + char c = sMessage.charAt(idx + i); + if (Character.isDigit(c) && c != '.') { + continue; + } + if (c == '.' || c == '-') { + continue; + } + break; + } + if (i != 1) { + size = Float.valueOf(new String(sMessage.substring(idx+1,idx+i))); + } + + String sub = sMessage.substring(idx, idx+i+1); + + if (sMessage.charAt(idx+i) == 'f') { + sMessage=sMessage.replaceFirst(sub, getFloatValue(size)); + } else if (sMessage.charAt(idx+i) == 'l') { + sMessage=sMessage.replaceFirst(sub, getLongValue(size)); + } else if (sMessage.charAt(idx+i) == 'd') { + sMessage = sMessage.replaceFirst(sub, getIntValue(size)); + } else if (sMessage.charAt(idx+i) == 's') { + sMessage = sMessage.replaceFirst(sub, getStringValue(size)); + } + idx = sMessage.indexOf("%", idx); + } + return sMessage.getBytes(StandardCharsets.UTF_8); + } + + private float _getFloatValue(float size) { + float f = r.nextFloat(); + int mag = (int) Math.abs(size); + f = f * (float) Math.pow(10, mag); + if (size < 0 && ((int) f) % 2 == 1) { + return f * -1; + } + return f; + } + + private String getStringValue(float size) { + int s = (int) size; + if (size == 0) { + size = 20; + }; + String result = ""; + for(int i = 0; i < s; i++) { + result = result + (char) ((int) 'a' + (int) (r.nextFloat() * 26)); + } + return result; + } + + private String getFloatValue(float size) { + if (size == 0) { + return String.valueOf(r.nextFloat()); + } + String format = "%" + String.valueOf(size) + "f"; + + return String.format(format, _getFloatValue(size)); + } + + private String getIntValue(float size) { + if (size == 0) { + return String.valueOf(r.nextInt()); + } + return String.valueOf((int) _getFloatValue(size)); + } + private String getLongValue(float size) { + if (size == 0) { + return String.valueOf(r.nextLong()); + } + return String.valueOf((long) _getFloatValue(size)); + } +} diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/IMessageFormatter.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/IMessageFormatter.java new file mode 100644 index 0000000000000..e80ae50484c27 --- /dev/null +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/IMessageFormatter.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.testclient; + +public interface IMessageFormatter { + byte[] formatMessage(String producerName, long msg, byte[] message); +} \ No newline at end of file diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java index ece6853d29cae..94aef5f48b2e1 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java @@ -94,6 +94,8 @@ public class PerformanceProducer { private static Recorder recorder = new Recorder(TimeUnit.SECONDS.toMicros(120000), 5); private static Recorder cumulativeRecorder = new Recorder(TimeUnit.SECONDS.toMicros(120000), 5); + private static IMessageFormatter messageFormatter = null; + @Parameters(commandDescription = "Test pulsar producer performance.") static class Arguments { @@ -121,6 +123,15 @@ static class Arguments { @Parameter(names = { "-n", "--num-producers" }, description = "Number of producers (per topic)") public int numProducers = 1; + @Parameter(names = {"--separator"}, description = "Separator between the topic and topic number") + public String separator = "-"; + + @Parameter(names = {"--send-timeout"}, description = "Set the sendTimeout value default 0 to keep compatibility with previous version of pulsar-perf") + public int sendTimeout = 0; + + @Parameter(names = { "-pn", "--producer-name" }, description = "Producer Name") + public String producerName = null; + @Parameter(names = { "-u", "--service-url" }, description = "Pulsar Service URL") public String serviceURL; @@ -229,6 +240,13 @@ static class Arguments { @Parameter(names = { "-am", "--access-mode" }, description = "Producer access mode") public ProducerAccessMode producerAccessMode = ProducerAccessMode.Shared; + + @Parameter(names = { "-fp", "--format-payload" }, + description = "Format %i as a message index in the stream from producer and/or %t as the timestamp nanoseconds.") + public boolean formatPayload = false; + + @Parameter(names = {"-fc", "--format-class"}, description="Custom Formatter class name") + public String formatterClass = "org.apache.pulsar.testclient.DefaultMessageFormatter"; } public static void main(String[] args) throws Exception { @@ -256,7 +274,7 @@ public static void main(String[] args) throws Exception { String prefixTopicName = arguments.topics.get(0); List defaultTopics = Lists.newArrayList(); for (int i = 0; i < arguments.numTopics; i++) { - defaultTopics.add(String.format("%s-%d", prefixTopicName, i)); + defaultTopics.add(String.format("%s%s%d", prefixTopicName, arguments.separator, i)); } arguments.topics = defaultTopics; } else { @@ -332,6 +350,10 @@ public static void main(String[] args) throws Exception { for (String payload : payloadList) { payloadByteList.add(payload.getBytes(StandardCharsets.UTF_8)); } + + if (arguments.formatPayload) { + messageFormatter = getMessageFormatter(arguments.formatterClass); + } } else { for (int i = 0; i < payloadBytes.length; ++i) { payloadBytes[i] = (byte) (random.nextInt(26) + 65); @@ -388,6 +410,7 @@ public static void main(String[] args) throws Exception { executor.submit(() -> { log.info("Started performance test thread {}", threadIdx); runProducer( + threadIdx, arguments, numMessagesPerThread, msgRatePerThread, @@ -453,7 +476,18 @@ public static void main(String[] args) throws Exception { } } - private static void runProducer(Arguments arguments, + static IMessageFormatter getMessageFormatter(String formatterClass) { + try { + ClassLoader classLoader = PerformanceProducer.class.getClassLoader(); + Class clz = classLoader.loadClass(formatterClass); + return (IMessageFormatter) clz.newInstance(); + } catch (Exception e) { + return null; + } + } + + private static void runProducer(int producerId, + Arguments arguments, long numMessages, int msgRate, List payloadByteList, @@ -486,7 +520,7 @@ private static void runProducer(Arguments arguments, client = clientBuilder.build(); ProducerBuilder producerBuilder = client.newProducer() // - .sendTimeout(0, TimeUnit.SECONDS) // + .sendTimeout(arguments.sendTimeout, TimeUnit.SECONDS) // .compressionType(arguments.compression) // .maxPendingMessages(arguments.maxOutstanding) // .maxPendingMessagesAcrossPartitions(arguments.maxPendingMessagesAcrossPartitions) @@ -494,6 +528,11 @@ private static void runProducer(Arguments arguments, // enable round robin message routing if it is a partitioned topic .messageRoutingMode(MessageRoutingMode.RoundRobinPartition); + if (arguments.producerName != null) { + String producerName = String.format("%s%s%d", arguments.producerName, arguments.separator, producerId); + producerBuilder.producerName(producerName); + } + if (arguments.batchTimeMillis == 0.0 && arguments.batchMaxMessages == 0) { producerBuilder.enableBatching(false); } else { @@ -516,6 +555,7 @@ private static void runProducer(Arguments arguments, } for (int i = 0; i < arguments.numTopics; i++) { + String topic = arguments.topics.get(i); log.info("Adding {} publishers on topic {}", arguments.numProducers, topic); @@ -580,7 +620,12 @@ private static void runProducer(Arguments arguments, byte[] payloadData; if (arguments.payloadFilename != null) { - payloadData = payloadByteList.get(random.nextInt(payloadByteList.size())); + if (messageFormatter != null) { + payloadData = messageFormatter.formatMessage(arguments.producerName, totalSent, + payloadByteList.get(random.nextInt(payloadByteList.size()))); + } else { + payloadData = payloadByteList.get(random.nextInt(payloadByteList.size())); + } } else { payloadData = payloadBytes; } diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java index 716a05a258e80..01588c62743c1 100644 --- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/PerformanceProducerTest.java @@ -176,4 +176,17 @@ public void testCreatePartitions() throws Exception { thread.join(); Assert.assertEquals(10, pulsar.getAdminClient().topics().getPartitionedTopicMetadata(topic).partitions); } + + @Test + public void testNotExistIMessageFormatter() { + IMessageFormatter msgFormatter = PerformanceProducer.getMessageFormatter("org.apache.pulsar.testclient.NonExistentFormatter"); + Assert.assertNull(msgFormatter); + } + + @Test + public void testDefaultIMessageFormatter() { + IMessageFormatter msgFormatter = PerformanceProducer.getMessageFormatter("org.apache.pulsar.testclient.DefaultMessageFormatter"); + Assert.assertTrue(msgFormatter instanceof DefaultMessageFormatter); + } + } diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/TestDefaultMessageFormatter.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/TestDefaultMessageFormatter.java new file mode 100644 index 0000000000000..2d5c4b9a3bfbc --- /dev/null +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/TestDefaultMessageFormatter.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.testclient; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonNode; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.nio.charset.StandardCharsets; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class TestDefaultMessageFormatter { + + @Test + public void testFormatMessage() { + String producerName = "producer-1"; + long msgId = 3; + byte[] message = "{ \"producer\": \"%p\", \"msgId\": %i, \"nanoTime\": %t, \"float1\": %5.2f, \"float2\": %-5.2f, \"long1\": %12l, \"long2\": %l, \"int1\": %d, \"int2\": %1d , \"long3\": %5l, \"str\": \"%5s\" }".getBytes(); + byte[] formatted = new DefaultMessageFormatter().formatMessage(producerName, msgId, message); + String jsonString = new String(formatted, StandardCharsets.UTF_8); + + ObjectMapper objectMapper = new ObjectMapper(); + + JsonNode obj = null; + try { + obj = objectMapper.readValue(jsonString, JsonNode.class); + + } catch(Exception jpe) { + Assert.fail("Exception parsing json"); + } + + String prod = obj.get("producer").asText(); + int mid = obj.get("msgId").asInt(); + long nt = obj.get("nanoTime").asLong(); + float f1 = obj.get("float1").floatValue(); + float f2 = obj.get("float2").floatValue(); + long l1 = obj.get("long1").asLong(); + long l2 = obj.get("long2").asLong(); + long i1 = obj.get("int1").asInt(); + long i2 = obj.get("int2").asInt(); + String str = obj.get("str").asText(); + long l3 = obj.get("long3").asLong(); + Assert.assertEquals(producerName, prod); + Assert.assertEquals(msgId, mid); + Assert.assertTrue( nt > 0); + Assert.assertNotEquals(f1, f2); + Assert.assertNotEquals(l1, l2); + Assert.assertNotEquals(i1, i2); + Assert.assertTrue(l3 > 0); + Assert.assertTrue(l3 <= 99999); + Assert.assertTrue(i2 < 10); + Assert.assertTrue(0 < i2); + Assert.assertTrue(f2 < 100000); + Assert.assertTrue( -100000 < f2); + + } +}