From f49786931dd8c37f1ea08bc684a54b6cda6b13fe Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Mon, 1 Mar 2021 23:29:29 +0800 Subject: [PATCH 01/11] add prometheus metric exporter framework --- docs/configuration.md | 1 + kafka-impl/conf/kop.conf | 895 ++++++++++++++++++ .../handlers/kop/KafkaChannelInitializer.java | 9 +- .../handlers/kop/KafkaProtocolHandler.java | 16 +- .../handlers/kop/KafkaRequestHandler.java | 24 +- .../kop/KafkaServiceConfiguration.java | 6 + .../pulsar/handlers/kop/KopServerStats.java | 19 + .../pulsar/handlers/kop/PendingProduce.java | 21 +- .../pulsar/handlers/kop/RequestStats.java | 53 ++ .../handlers/kop/StatsProviderService.java | 37 + .../kop/stats/DataSketchesOpStatsLogger.java | 196 ++++ .../handlers/kop/stats/LongAdderCounter.java | 55 ++ .../kop/stats/PrometheusMetricsProvider.java | 253 +++++ .../handlers/kop/stats/PrometheusServlet.java | 58 ++ .../kop/stats/PrometheusStatsLogger.java | 73 ++ .../kop/stats/PrometheusTextFormatUtil.java | 148 +++ .../handlers/kop/stats/SimpleGauge.java | 40 + .../handlers/kop/EntryPublishTimeTest.java | 4 +- .../pulsar/handlers/kop/KafkaApisTest.java | 4 +- .../handlers/kop/KafkaRequestHandlerTest.java | 4 +- .../kop/KafkaTopicConsumerManagerTest.java | 4 +- 21 files changed, 1909 insertions(+), 11 deletions(-) create mode 100755 kafka-impl/conf/kop.conf create mode 100644 kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java create mode 100644 kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/RequestStats.java create mode 100644 kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/StatsProviderService.java create mode 100644 kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/DataSketchesOpStatsLogger.java create mode 100644 kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/LongAdderCounter.java create mode 100644 kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java create mode 100644 kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusServlet.java create mode 100644 kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusStatsLogger.java create mode 100644 kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java create mode 100644 kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/SimpleGauge.java diff --git a/docs/configuration.md b/docs/configuration.md index 20c51484e9..fc72ba44f3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -47,4 +47,5 @@ The following table lists all KoP configurations. |kopSslTrustmanagerAlgorithm| Kafka SSL configuration map with: SSL_TRUSTMANAGER_ALGORITHM_CONFIG = ssl.trustmanager.algorithm |SunX509| |kopSslSecureRandomImplementation| Kafka SSL configuration map with: SSL_SECURE_RANDOM_IMPLEMENTATION_CONFIG = ssl.secure.random.implementation | N/A| |saslAllowedMechanisms| Supported SASL mechanisms exposed by the broker |N/A| +|kopPrometheusStatsLatencyRolloverSeconds|Kop metrics expose to prometheus rollover latency | 60 | diff --git a/kafka-impl/conf/kop.conf b/kafka-impl/conf/kop.conf new file mode 100755 index 0000000000..2f8f4c5769 --- /dev/null +++ b/kafka-impl/conf/kop.conf @@ -0,0 +1,895 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### --- Kafka broker settings --- ### + +# The messaging Protocols that avilabale when loaded by Pulsar Broker. +messagingProtocols=kafka + +# ListenersProp for Kafka service(host should follow the advertisedAddress). +# e.g. PLAINTEXT://localhost:9092,SSL://localhost:9093 +# when KoP runs as a plugin in Pulsar, if it is not set, kop will use PLAINTEXT://advertisedAddress:9092 +listeners=PLAINTEXT://127.0.0.1:9092 + +# Kafka on Pulsar Broker tenant +kafkaTenant=public + +# Kafka on Pulsar Broker namespace +kafkaNamespace=default + +# The tenant used for storing Kafka metadata topics +kafkaMetadataTenant=public + +# The namespace used for storing Kafka metadata topics +kafkaMetadataNamespace=__kafka + +# Flag to enable group coordinator +enableGroupCoordinator=true + +# The minimum allowed session timeout for registered consumers. +# Shorter timeouts result in quicker failure detection at the cost +# of more frequent consumer heartbeating, which can overwhelm broker resources. +groupMinSessionTimeoutMs=6000 + +# The maximum allowed session timeout for registered consumers. +# Longer timeouts give consumers more time to process messages in +# between heartbeats at the cost of a longer time to detect failures. +groupMaxSessionTimeoutMs=300000 + +# The amount of time the group coordinator will wait for more consumers +# to join a new group before performing the first rebalance. A longer +# delay means potentially fewer rebalances, but increases the time until +# processing begins +groupInitialRebalanceDelayMs=3000 + +# Compression codec for the offsets topic - compression may be used to achieve "atomic" commits +offsetsTopicCompressionCodec=NONE + +# The maximum size in Bytes for a metadata entry associated with an offset commit +offsetMetadataMaxSize=4096 + +# Offsets older than this retention period will be discarded, default 7 days +offsetsRetentionMinutes=10080 + +# Frequency at which to check for stale offsets +offsetsRetentionCheckIntervalMs=600000 + +# Number of partitions for the offsets topic +offsetsTopicNumPartitions=8 + +# Maximum number of entries that are read from cursor once per time +maxReadEntriesNum=5 + +# The format of an entry. The default value is pulsar. +# Optional values: [pulsar, kafka] +# +# pulsar: +# When KoP receives messages from kafka producer, it will serialize these messages to +# the format so that pulsar consumer can read directly. +# When KoP sends entries to kafka consumer, it will treat each entry as pulsar's +# format and deserialize each entry to kafka's format. +# +# kafka: +# When KoP receives messages from kafka producer, add a header which is PulsarApi.Metadata +# before the messages' bytes, and then write to BK directly. +# When KoP sends entries to kafka consumer, it will treat each entry as kafka's format and +# just discard the pulsar header and send left bytes to Kafka consumer. +# This mode means that current pulsar clients cannot interact with kafka clients, but +# kafka producer works well with kafka consumer. +entryFormat=pulsar + +# Zookeeper path for storing kop consumer group +groupIdZooKeeperPath=/client_group_id + +### --- KoP SSL configs--- ### + +# Kafka ssl configuration map with: SSL_PROTOCOL_CONFIG = ssl.protocol +kopSslProtocol=TLS + +# Kafka ssl configuration map with: SSL_PROVIDER_CONFIG = ssl.provider +kopSslProvider= + +# Kafka ssl configuration map with: SSL_CIPHER_SUITES_CONFIG = ssl.cipher.suites +kopSslCipherSuites= + +# Kafka ssl configuration map with: SSL_ENABLED_PROTOCOLS_CONFIG = ssl.enabled.protocols +kopSslEnabledProtocols=TLSv1.2,TLSv1.1,TLSv1 + +# Kafka ssl configuration map with: SSL_KEYSTORE_TYPE_CONFIG = ssl.keystore.type +kopSslKeystoreType=JKS + +# Kafka ssl configuration map with: SSL_KEYSTORE_LOCATION_CONFIG = ssl.keystore.location +kopSslKeystoreLocation= + +# Kafka ssl configuration map with: SSL_KEYSTORE_PASSWORD_CONFIG = ssl.keystore.password +kopSslKeystorePassword= + +# Kafka ssl configuration map with: SSL_KEY_PASSWORD_CONFIG = ssl.key.password +kopSslKeyPassword= + +# Kafka ssl configuration map with: SSL_TRUSTSTORE_TYPE_CONFIG = ssl.truststore.type +kopSslTruststoreType=JKS + +# Kafka ssl configuration map with: SSL_TRUSTSTORE_LOCATION_CONFIG = ssl.truststore.location +kopSslTruststoreLocation= + +# Kafka ssl configuration map with: SSL_TRUSTSTORE_PASSWORD_CONFIG = ssl.truststore.password +kopSslTruststorePassword= + +# Kafka ssl configuration map with: SSL_KEYMANAGER_ALGORITHM_CONFIG = ssl.keymanager.algorithm +kopSslKeymanagerAlgorithm=SunX509 + +# Kafka ssl configuration map with: SSL_TRUSTMANAGER_ALGORITHM_CONFIG = ssl.trustmanager.algorithm +kopSslTrustmanagerAlgorithm=SunX509 + +# Kafka ssl configuration map with: +# SSL_SECURE_RANDOM_IMPLEMENTATION_CONFIG = ssl.secure.random.implementation +kopSslSecureRandomImplementation= + +# supported SASL mechanisms exposed by broker +saslAllowedMechanisms= + +### --- Changed for KoP --- ### + +# Enable the deletion of inactive topics +brokerDeleteInactiveTopicsEnabled=false + +allowAutoTopicCreation=true + +allowAutoTopicCreationType=partitioned + +# Name of the cluster to which this broker belongs to +clusterName=kafka-cluster + +### --- General broker settings --- ### + +# Max Rate(in 1 seconds) of Message allowed to publish for a broker if broker publish rate limiting enabled +# (Disable message rate limit with value 0) +brokerPublisherThrottlingMaxMessageRate=0 + +# Max Rate(in 1 seconds) of Byte allowed to publish for a broker if broker publish rate limiting enabled. +# (Disable byte rate limit with value 0) +brokerPublisherThrottlingMaxByteRate=0 + +# Zookeeper quorum connection string +zookeeperServers=localhost:2181 + +# Configuration Store connection string +configurationStoreServers= + +# Broker data port +brokerServicePort=6650 + +# Broker data port for TLS - By default TLS is disabled +brokerServicePortTls= + +# Port to use to server HTTP request +webServicePort=8080 + +# Port to use to server HTTPS request - By default TLS is disabled +webServicePortTls= + +# Hostname or IP address the service binds on, default is 0.0.0.0. +bindAddress=0.0.0.0 + +# Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used. +advertisedAddress= + +# Number of threads to use for Netty IO. Default is set to 2 * Runtime.getRuntime().availableProcessors() +numIOThreads= + +# Number of threads to use for HTTP requests processing. Default is set to 2 * Runtime.getRuntime().availableProcessors() +numHttpServerThreads= + +# Flag to control features that are meant to be used when running in standalone mode +isRunningStandalone= + +# Enable cluster's failure-domain which can distribute brokers into logical region +failureDomainsEnabled=false + +# Zookeeper session timeout in milliseconds +zooKeeperSessionTimeoutMillis=30000 + +# ZooKeeper operation timeout in seconds +zooKeeperOperationTimeoutSeconds=30 + +# Time to wait for broker graceful shutdown. After this time elapses, the process will be killed +brokerShutdownTimeoutMs=60000 + +# Enable backlog quota check. Enforces action on topic when the quota is reached +backlogQuotaCheckEnabled=true + +# How often to check for topics that have reached the quota +backlogQuotaCheckIntervalInSeconds=60 + +# Default per-topic backlog quota limit, less than 0 means no limitation. default is -1. +backlogQuotaDefaultLimitGB=-1 + +# Default backlog quota retention policy. Default is producer_request_hold +# 'producer_request_hold' Policy which holds producer's send request until the resource becomes available (or holding times out) +# 'producer_exception' Policy which throws javax.jms.ResourceAllocationException to the producer +# 'consumer_backlog_eviction' Policy which evicts the oldest message from the slowest consumer's backlog +backlogQuotaDefaultRetentionPolicy=producer_request_hold + +# Default ttl for namespaces if ttl is not already configured at namespace policies. (disable default-ttl with value 0) +ttlDurationDefaultInSeconds=0 + +# Enable the deletion of inactive topics +brokerDeleteInactiveTopicsEnabled=true + +# How often to check for inactive topics +brokerDeleteInactiveTopicsFrequencySeconds=60 + +# How frequently to proactively check and purge expired messages +messageExpiryCheckIntervalInMinutes=5 + +# How long to delay rewinding cursor and dispatching messages when active consumer is changed +activeConsumerFailoverDelayTimeMillis=1000 + +# How long to delete inactive subscriptions from last consuming +# When it is 0, inactive subscriptions are not deleted automatically +subscriptionExpirationTimeMinutes=0 + +# Enable subscription message redelivery tracker to send redelivery count to consumer (default is enabled) +subscriptionRedeliveryTrackerEnabled=true + +# How frequently to proactively check and purge expired subscription +subscriptionExpiryCheckIntervalInMinutes=5 + +# Enable Key_Shared subscription (default is enabled) +subscriptionKeySharedEnable=true + +# Set the default behavior for message deduplication in the broker +# This can be overridden per-namespace. If enabled, broker will reject +# messages that were already stored in the topic +brokerDeduplicationEnabled=false + +# Maximum number of producer information that it's going to be +# persisted for deduplication purposes +brokerDeduplicationMaxNumberOfProducers=10000 + +# Number of entries after which a dedup info snapshot is taken. +# A larger interval will lead to fewer snapshots being taken, though it would +# increase the topic recovery time when the entries published after the +# snapshot need to be replayed. +brokerDeduplicationEntriesInterval=1000 + +# Time of inactivity after which the broker will discard the deduplication information +# relative to a disconnected producer. Default is 6 hours. +brokerDeduplicationProducerInactivityTimeoutMinutes=360 + +# When a namespace is created without specifying the number of bundle, this +# value will be used as the default +defaultNumberOfNamespaceBundles=4 + +# Enable check for minimum allowed client library version +clientLibraryVersionCheckEnabled=false + +# Path for the file used to determine the rotation status for the broker when responding +# to service discovery health checks +statusFilePath= + +# If true, (and ModularLoadManagerImpl is being used), the load manager will attempt to +# use only brokers running the latest software version (to minimize impact to bundles) +preferLaterVersions=false + +# Max number of unacknowledged messages allowed to receive messages by a consumer on a shared subscription. Broker will stop sending +# messages to consumer once, this limit reaches until consumer starts acknowledging messages back. +# Using a value of 0, is disabling unackeMessage limit check and consumer can receive messages without any restriction +maxUnackedMessagesPerConsumer=50000 + +# Max number of unacknowledged messages allowed per shared subscription. Broker will stop dispatching messages to +# all consumers of the subscription once this limit reaches until consumer starts acknowledging messages back and +# unack count reaches to limit/2. Using a value of 0, is disabling unackedMessage-limit +# check and dispatcher can dispatch messages without any restriction +maxUnackedMessagesPerSubscription=200000 + +# Max number of unacknowledged messages allowed per broker. Once this limit reaches, broker will stop dispatching +# messages to all shared subscription which has higher number of unack messages until subscriptions start +# acknowledging messages back and unack count reaches to limit/2. Using a value of 0, is disabling +# unackedMessage-limit check and broker doesn't block dispatchers +maxUnackedMessagesPerBroker=0 + +# Once broker reaches maxUnackedMessagesPerBroker limit, it blocks subscriptions which has higher unacked messages +# than this percentage limit and subscription will not receive any new messages until that subscription acks back +# limit/2 messages +maxUnackedMessagesPerSubscriptionOnBrokerBlocked=0.16 + +# Too many subscribe requests from a consumer can cause broker rewinding consumer cursors and loading data from bookies, +# hence causing high network bandwidth usage +# When the positive value is set, broker will throttle the subscribe requests for one consumer. +# Otherwise, the throttling will be disabled. The default value of this setting is 0 - throttling is disabled. +subscribeThrottlingRatePerConsumer=0 + +# Rate period for {subscribeThrottlingRatePerConsumer}. Default is 30s. +subscribeRatePeriodPerConsumerInSecond=30 + +# Default messages per second dispatch throttling-limit for every topic. Using a value of 0, is disabling default +# message dispatch-throttling +dispatchThrottlingRatePerTopicInMsg=0 + +# Default bytes per second dispatch throttling-limit for every topic. Using a value of 0, is disabling +# default message-byte dispatch-throttling +dispatchThrottlingRatePerTopicInByte=0 + +# Default number of message dispatching throttling-limit for a subscription. +# Using a value of 0, is disabling default message dispatch-throttling. +dispatchThrottlingRatePerSubscriptionInMsg=0 + +# Default number of message-bytes dispatching throttling-limit for a subscription. +# Using a value of 0, is disabling default message-byte dispatch-throttling. +dispatchThrottlingRatePerSubscriptionInByte=0 + +# Default messages per second dispatch throttling-limit for every replicator in replication. +# Using a value of 0, is disabling replication message dispatch-throttling +dispatchThrottlingRatePerReplicatorInMsg=0 + +# Default bytes per second dispatch throttling-limit for every replicator in replication. +# Using a value of 0, is disabling replication message-byte dispatch-throttling +dispatchThrottlingRatePerReplicatorInByte=0 + +# By default we enable dispatch-throttling for both caught up consumers as well as consumers who have +# backlog. +dispatchThrottlingOnNonBacklogConsumerEnabled=true + +# Max number of entries to read from bookkeeper. By default it is 100 entries. +dispatcherMaxReadBatchSize=100 + +# Min number of entries to read from bookkeeper. By default it is 1 entries. +# When there is an error occurred on reading entries from bookkeeper, the broker +# will backoff the batch size to this minimum number." +dispatcherMinReadBatchSize=1 + +# Max number of entries to dispatch for a shared subscription. By default it is 20 entries. +dispatcherMaxRoundRobinBatchSize=20 + +# Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic +maxConcurrentLookupRequest=50000 + +# Max number of concurrent topic loading request broker allows to control number of zk-operations +maxConcurrentTopicLoadRequest=5000 + +# Max concurrent non-persistent message can be processed per connection +maxConcurrentNonPersistentMessagePerConnection=1000 + +# Number of worker threads to serve non-persistent topic +numWorkerThreadsForNonPersistentTopic=8 + +# Enable broker to load persistent topics +enablePersistentTopics=true + +# Enable broker to load non-persistent topics +enableNonPersistentTopics=true + +# Enable to run bookie along with broker +enableRunBookieTogether=false + +# Enable to run bookie autorecovery along with broker +enableRunBookieAutoRecoveryTogether=false + +# Max number of producers allowed to connect to topic. Once this limit reaches, Broker will reject new producers +# until the number of connected producers decrease. +# Using a value of 0, is disabling maxProducersPerTopic-limit check. +maxProducersPerTopic=0 + +# Max number of consumers allowed to connect to topic. Once this limit reaches, Broker will reject new consumers +# until the number of connected consumers decrease. +# Using a value of 0, is disabling maxConsumersPerTopic-limit check. +maxConsumersPerTopic=0 + +# Max number of consumers allowed to connect to subscription. Once this limit reaches, Broker will reject new consumers +# until the number of connected consumers decrease. +# Using a value of 0, is disabling maxConsumersPerSubscription-limit check. +maxConsumersPerSubscription=0 + +# Max size of messages. +maxMessageSize=5242880 + +# Interval between checks to see if topics with compaction policies need to be compacted +brokerServiceCompactionMonitorIntervalInSeconds=60 + +# Whether to enable the delayed delivery for messages. +# If disabled, messages will be immediately delivered and there will +# be no tracking overhead. +delayedDeliveryEnabled=true + +# Control the tick time for when retrying on delayed delivery, +# affecting the accuracy of the delivery time compared to the scheduled time. +# Default is 1 second. +delayedDeliveryTickTimeMillis=1000 + +# Enable tracking of replicated subscriptions state across clusters. +enableReplicatedSubscriptions=true + +# Frequency of snapshots for replicated subscriptions tracking. +replicatedSubscriptionsSnapshotFrequencyMillis=1000 + +# Timeout for building a consistent snapshot for tracking replicated subscriptions state. +replicatedSubscriptionsSnapshotTimeoutSeconds=30 + +# Max number of snapshot to be cached per subscription. +replicatedSubscriptionsSnapshotMaxCachedPerSubscription=10 + +### --- Authentication --- ### +# Role names that are treated as "proxy roles". If the broker sees a request with +#role as proxyRoles - it will demand to see a valid original principal. +proxyRoles= + +# If this flag is set then the broker authenticates the original Auth data +# else it just accepts the originalPrincipal and authorizes it (if required). +authenticateOriginalAuthData=false + +# Deprecated - Use webServicePortTls and brokerServicePortTls instead +tlsEnabled=false + +# Tls cert refresh duration in seconds (set 0 to check on every new connection) +tlsCertRefreshCheckDurationSec=300 + +# Path for the TLS certificate file +tlsCertificateFilePath= + +# Path for the TLS private key file +tlsKeyFilePath= + +# Path for the trusted TLS certificate file. +# This cert is used to verify that any certs presented by connecting clients +# are signed by a certificate authority. If this verification +# fails, then the certs are untrusted and the connections are dropped. +tlsTrustCertsFilePath= + +# Accept untrusted TLS certificate from client. +# If true, a client with a cert which cannot be verified with the +# 'tlsTrustCertsFilePath' cert will allowed to connect to the server, +# though the cert will not be used for client authentication. +tlsAllowInsecureConnection=false + +# Specify the tls protocols the broker will use to negotiate during TLS handshake +# (a comma-separated list of protocol names). +# Examples:- [TLSv1.2, TLSv1.1, TLSv1] +tlsProtocols= + +# Specify the tls cipher the broker will use to negotiate during TLS Handshake +# (a comma-separated list of ciphers). +# Examples:- [TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256] +tlsCiphers= + +# Trusted client certificates are required for to connect TLS +# Reject the Connection if the Client Certificate is not trusted. +# In effect, this requires that all connecting clients perform TLS client +# authentication. +tlsRequireTrustedClientCertOnConnect=false + +### --- Authentication --- ### + +# Enable authentication +authenticationEnabled=false + +# Autentication provider name list, which is comma separated list of class names +authenticationProviders= + +# Enforce authorization +authorizationEnabled=false + +# Authorization provider fully qualified class-name +authorizationProvider=org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider + +# Allow wildcard matching in authorization +# (wildcard matching only applicable if wildcard-char: +# * presents at first or last position eg: *.pulsar.service, pulsar.service.*) +authorizationAllowWildcardsMatching=false + +# Role names that are treated as "super-user", meaning they will be able to do all admin +# operations and publish/consume from all topics +superUserRoles= + +# Authentication settings of the broker itself. Used when the broker connects to other brokers, +# either in same or other clusters +brokerClientTlsEnabled=false +brokerClientAuthenticationPlugin= +brokerClientAuthenticationParameters= +brokerClientTrustCertsFilePath= + +# Supported Athenz provider domain names(comma separated) for authentication +athenzDomainNames= + +# When this parameter is not empty, unauthenticated users perform as anonymousUserRole +anonymousUserRole= + +### --- Token Authentication Provider --- ### + +## Symmetric key +# Configure the secret key to be used to validate auth tokens +# The key can be specified like: +# tokenSecretKey=data:base64,xxxxxxxxx +# tokenSecretKey=file:///my/secret.key +tokenSecretKey= + +## Asymmetric public/private key pair +# Configure the public key to be used to validate auth tokens +# The key can be specified like: +# tokenPublicKey=data:base64,xxxxxxxxx +# tokenPublicKey=file:///my/public.key +tokenPublicKey= + +# The token "claim" that will be interpreted as the authentication "role" or "principal" by AuthenticationProviderToken (defaults to "sub" if blank) +tokenAuthClaim= + +### --- SASL Authentication Provider --- ### + +# This is a regexp, which limits the range of possible ids which can connect to the Broker using SASL. +# Default value: `SaslConstants.JAAS_CLIENT_ALLOWED_IDS_DEFAULT`, which is ".*pulsar.*", +# so only clients whose id contains 'pulsar' are allowed to connect. +saslJaasClientAllowedIds= + +# Service Principal, for login context name. +# Default value `SaslConstants.JAAS_DEFAULT_BROKER_SECTION_NAME`, which is "Broker". +saslJaasBrokerSectionName= + +### --- BookKeeper Client --- ### + +# Authentication plugin to use when connecting to bookies +bookkeeperClientAuthenticationPlugin= + +# BookKeeper auth plugin implementatation specifics parameters name and values +bookkeeperClientAuthenticationParametersName= +bookkeeperClientAuthenticationParameters= + +# Timeout for BK add / read operations +bookkeeperClientTimeoutInSeconds=30 + +# Speculative reads are initiated if a read request doesn't complete within a certain time +# Using a value of 0, is disabling the speculative reads +bookkeeperClientSpeculativeReadTimeoutInMillis=0 + +# Use older Bookkeeper wire protocol with bookie +bookkeeperUseV2WireProtocol=true + +# Enable bookies health check. Bookies that have more than the configured number of failure within +# the interval will be quarantined for some time. During this period, new ledgers won't be created +# on these bookies +bookkeeperClientHealthCheckEnabled=true +bookkeeperClientHealthCheckIntervalSeconds=60 +bookkeeperClientHealthCheckErrorThresholdPerInterval=5 +bookkeeperClientHealthCheckQuarantineTimeInSeconds=1800 + +# Enable rack-aware bookie selection policy. BK will chose bookies from different racks when +# forming a new bookie ensemble +bookkeeperClientRackawarePolicyEnabled=true + +# Enable region-aware bookie selection policy. BK will chose bookies from +# different regions and racks when forming a new bookie ensemble +# If enabled, the value of bookkeeperClientRackawarePolicyEnabled is ignored +bookkeeperClientRegionawarePolicyEnabled=false + +# Enable/disable reordering read sequence on reading entries. +bookkeeperClientReorderReadSequenceEnabled=false + +# Enable bookie isolation by specifying a list of bookie groups to choose from. Any bookie +# outside the specified groups will not be used by the broker +bookkeeperClientIsolationGroups= + +# Enable bookie secondary-isolation group if bookkeeperClientIsolationGroups doesn't +# have enough bookie available. +bookkeeperClientSecondaryIsolationGroups= + +# Minimum bookies that should be available as part of bookkeeperClientIsolationGroups +# else broker will include bookkeeperClientSecondaryIsolationGroups bookies in isolated list. +bookkeeperClientMinAvailableBookiesInIsolationGroups= + +# Enable/disable having read operations for a ledger to be sticky to a single bookie. +# If this flag is enabled, the client will use one single bookie (by preference) to read +# all entries for a ledger. +# +# Disable Sticy Read until {@link https://github.com/apache/bookkeeper/issues/1970} is fixed +bookkeeperEnableStickyReads=false + +### --- Managed Ledger --- ### + +# Number of bookies to use when creating a ledger +managedLedgerDefaultEnsembleSize=1 + +# Number of copies to store for each message +managedLedgerDefaultWriteQuorum=1 + +# Number of guaranteed copies (acks to wait before write is complete) +managedLedgerDefaultAckQuorum=1 + +# Default type of checksum to use when writing to BookKeeper. Default is "CRC32C" +# Other possible options are "CRC32", "MAC" or "DUMMY" (no checksum). +managedLedgerDigestType=CRC32C + +# Number of threads to be used for managed ledger tasks dispatching +managedLedgerNumWorkerThreads=8 + +# Number of threads to be used for managed ledger scheduled tasks +managedLedgerNumSchedulerThreads=8 + +# Amount of memory to use for caching data payload in managed ledger. This memory +# is allocated from JVM direct memory and it's shared across all the topics +# running in the same broker. By default, uses 1/5th of available direct memory +managedLedgerCacheSizeMB= + +# Whether we should make a copy of the entry payloads when inserting in cache +managedLedgerCacheCopyEntries=false + +# Threshold to which bring down the cache level when eviction is triggered +managedLedgerCacheEvictionWatermark=0.9 + +# Configure the cache eviction frequency for the managed ledger cache (evictions/sec) +managedLedgerCacheEvictionFrequency=100.0 + +# All entries that have stayed in cache for more than the configured time, will be evicted +managedLedgerCacheEvictionTimeThresholdMillis=1000 + +# Configure the threshold (in number of entries) from where a cursor should be considered 'backlogged' +# and thus should be set as inactive. +managedLedgerCursorBackloggedThreshold=1000 + +# Rate limit the amount of writes per second generated by consumer acking the messages +managedLedgerDefaultMarkDeleteRateLimit=1.0 + +# Max number of entries to append to a ledger before triggering a rollover +# A ledger rollover is triggered on these conditions +# * Either the max rollover time has been reached +# * or max entries have been written to the ledged and at least min-time +# has passed +managedLedgerMaxEntriesPerLedger=50000 + +# Minimum time between ledger rollover for a topic +managedLedgerMinLedgerRolloverTimeMinutes=10 + +# Maximum time before forcing a ledger rollover for a topic +managedLedgerMaxLedgerRolloverTimeMinutes=240 + +# Delay between a ledger being successfully offloaded to long term storage +# and the ledger being deleted from bookkeeper (default is 4 hours) +managedLedgerOffloadDeletionLagMs=14400000 + +# Max number of entries to append to a cursor ledger +managedLedgerCursorMaxEntriesPerLedger=50000 + +# Max time before triggering a rollover on a cursor ledger +managedLedgerCursorRolloverTimeInSeconds=14400 + +# Max number of "acknowledgment holes" that are going to be persistently stored. +# When acknowledging out of order, a consumer will leave holes that are supposed +# to be quickly filled by acking all the messages. The information of which +# messages are acknowledged is persisted by compressing in "ranges" of messages +# that were acknowledged. After the max number of ranges is reached, the information +# will only be tracked in memory and messages will be redelivered in case of +# crashes. +managedLedgerMaxUnackedRangesToPersist=10000 + +# Max number of "acknowledgment holes" that can be stored in Zookeeper. If number of unack message range is higher +# than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into +# zookeeper. +managedLedgerMaxUnackedRangesToPersistInZooKeeper=1000 + +# Skip reading non-recoverable/unreadable data-ledger under managed-ledger's list. It helps when data-ledgers gets +# corrupted at bookkeeper and managed-cursor is stuck at that ledger. +autoSkipNonRecoverableData=false + +# operation timeout while updating managed-ledger metadata. +managedLedgerMetadataOperationsTimeoutSeconds=60 + +# Read entries timeout when broker tries to read messages from bookkeeper. +managedLedgerReadEntryTimeoutSeconds=0 + +# Add entry timeout when broker tries to publish message to bookkeeper (0 to disable it). +managedLedgerAddEntryTimeoutSeconds=0 + +### --- Load balancer --- ### + +# Enable load balancer +loadBalancerEnabled=true + +# Percentage of change to trigger load report update +loadBalancerReportUpdateThresholdPercentage=10 + +# maximum interval to update load report +loadBalancerReportUpdateMaxIntervalMinutes=15 + +# Frequency of report to collect +loadBalancerHostUsageCheckIntervalMinutes=1 + +# Enable/disable automatic bundle unloading for load-shedding +loadBalancerSheddingEnabled=true + +# Load shedding interval. Broker periodically checks whether some traffic should be offload from +# some over-loaded broker to other under-loaded brokers +loadBalancerSheddingIntervalMinutes=1 + +# Prevent the same topics to be shed and moved to other broker more that once within this timeframe +loadBalancerSheddingGracePeriodMinutes=30 + +# Usage threshold to allocate max number of topics to broker +loadBalancerBrokerMaxTopics=50000 + +# Usage threshold to determine a broker as over-loaded +loadBalancerBrokerOverloadedThresholdPercentage=85 + +# Interval to flush dynamic resource quota to ZooKeeper +loadBalancerResourceQuotaUpdateIntervalMinutes=15 + +# enable/disable namespace bundle auto split +loadBalancerAutoBundleSplitEnabled=true + +# enable/disable automatic unloading of split bundles +loadBalancerAutoUnloadSplitBundlesEnabled=true + +# maximum topics in a bundle, otherwise bundle split will be triggered +loadBalancerNamespaceBundleMaxTopics=1000 + +# maximum sessions (producers + consumers) in a bundle, otherwise bundle split will be triggered +loadBalancerNamespaceBundleMaxSessions=1000 + +# maximum msgRate (in + out) in a bundle, otherwise bundle split will be triggered +loadBalancerNamespaceBundleMaxMsgRate=30000 + +# maximum bandwidth (in + out) in a bundle, otherwise bundle split will be triggered +loadBalancerNamespaceBundleMaxBandwidthMbytes=100 + +# maximum number of bundles in a namespace +loadBalancerNamespaceMaximumBundles=128 + +# Override the auto-detection of the network interfaces max speed. +# This option is useful in some environments (eg: EC2 VMs) where the max speed +# reported by Linux is not reflecting the real bandwidth available to the broker. +# Since the network usage is employed by the load manager to decide when a broker +# is overloaded, it is important to make sure the info is correct or override it +# with the right value here. The configured value can be a double (eg: 0.8) and that +# can be used to trigger load-shedding even before hitting on NIC limits. +loadBalancerOverrideBrokerNicSpeedGbps= + +# Name of load manager to use +loadManagerClassName=org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl + +### --- Replication --- ### + +# Enable replication metrics +replicationMetricsEnabled=true + +# Max number of connections to open for each broker in a remote cluster +# More connections host-to-host lead to better throughput over high-latency +# links. +replicationConnectionsPerBroker=16 + +# Replicator producer queue size +replicationProducerQueueSize=1000 + +# Replicator prefix used for replicator producer name and cursor name +replicatorPrefix=pulsar.repl + +# Default message retention time +defaultRetentionTimeInMinutes=600 + +# Default retention size +defaultRetentionSizeInMB=1000 + +# How often to check whether the connections are still alive +keepAliveIntervalSeconds=30 + +# How often broker checks for inactive topics to be deleted (topics with no subscriptions and no one connected) +brokerServicePurgeInactiveFrequencyInSeconds=60 + +# bootstrap namespaces +bootstrapNamespaces= + +### --- WebSocket --- ### + +# Enable the WebSocket API service in broker +webSocketServiceEnabled=false + +# Number of IO threads in Pulsar Client used in WebSocket proxy +webSocketNumIoThreads=8 + +# Number of connections per Broker in Pulsar Client used in WebSocket proxy +webSocketConnectionsPerBroker=8 + +# Time in milliseconds that idle WebSocket session times out +webSocketSessionIdleTimeoutMillis=300000 + +### --- Metrics --- ### + +# Enable topic level metrics +exposeTopicLevelMetricsInPrometheus=true + +# Enable consumer level metrics. default is false +exposeConsumerLevelMetricsInPrometheus=false + +# Classname of Pluggable JVM GC metrics logger that can log GC specific metrics +# jvmGCMetricsLoggerClassName= + +### --- Functions --- ### + +# Enable Functions Worker Service in Broker +functionsWorkerEnabled=false + +### --- Broker Web Stats --- ### + +# Enable topic level metrics +exposePublisherStats=true +statsUpdateFrequencyInSecs=60 +statsUpdateInitialDelayInSecs=60 + +### --- Schema storage --- ### +# The schema storage implementation used by this broker +schemaRegistryStorageClassName=org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorageFactory + +# Enforce schema validation on following cases: +# +# - if a producer without a schema attempts to produce to a topic with schema, the producer will be +# failed to connect. PLEASE be carefully on using this, since non-java clients don't support schema. +# if you enable this setting, it will cause non-java clients failed to produce. +isSchemaValidationEnforced=false + +### --- Ledger Offloading --- ### + +# The directory for all the offloader implementations +offloadersDirectory=./offloaders + +# Driver to use to offload old data to long term storage (Possible values: S3, aws-s3, google-cloud-storage) +# When using google-cloud-storage, Make sure both Google Cloud Storage and Google Cloud Storage JSON API are enabled for +# the project (check from Developers Console -> Api&auth -> APIs). +managedLedgerOffloadDriver= + +# Maximum number of thread pool threads for ledger offloading +managedLedgerOffloadMaxThreads=2 + +# Use Open Range-Set to cache unacked messages +managedLedgerUnackedRangesOpenCacheSetEnabled=true + +# For Amazon S3 ledger offload, AWS region +s3ManagedLedgerOffloadRegion= + +# For Amazon S3 ledger offload, Bucket to place offloaded ledger into +s3ManagedLedgerOffloadBucket= + +# For Amazon S3 ledger offload, Alternative endpoint to connect to (useful for testing) +s3ManagedLedgerOffloadServiceEndpoint= + +# For Amazon S3 ledger offload, Max block size in bytes. (64MB by default, 5MB minimum) +s3ManagedLedgerOffloadMaxBlockSizeInBytes=67108864 + +# For Amazon S3 ledger offload, Read buffer size in bytes (1MB by default) +s3ManagedLedgerOffloadReadBufferSizeInBytes=1048576 + +# For Google Cloud Storage ledger offload, region where offload bucket is located. +# reference this page for more details: https://cloud.google.com/storage/docs/bucket-locations +gcsManagedLedgerOffloadRegion= + +# For Google Cloud Storage ledger offload, Bucket to place offloaded ledger into +gcsManagedLedgerOffloadBucket= + +# For Google Cloud Storage ledger offload, Max block size in bytes. (64MB by default, 5MB minimum) +gcsManagedLedgerOffloadMaxBlockSizeInBytes=67108864 + +# For Google Cloud Storage ledger offload, Read buffer size in bytes (1MB by default) +gcsManagedLedgerOffloadReadBufferSizeInBytes=1048576 + +# For Google Cloud Storage, path to json file containing service account credentials. +# For more details, see the "Service Accounts" section of https://support.google.com/googleapi/answer/6158849 +gcsManagedLedgerOffloadServiceAccountKeyFile= + +### --- Deprecated config variables --- ### + +# Deprecated. Use configurationStoreServers +globalZookeeperServers= + +# Deprecated - Enable TLS when talking with other clusters to replicate messages +replicationTlsEnabled=false + +### --- Prometheus metric provider configuration --- ### +# Prometheus stats rollover latency, default: 60s +kopPrometheusStatsLatencyRolloverSeconds=60 diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaChannelInitializer.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaChannelInitializer.java index b4f6a49797..7e5293f26d 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaChannelInitializer.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaChannelInitializer.java @@ -24,6 +24,7 @@ import io.streamnative.pulsar.handlers.kop.coordinator.transaction.TransactionCoordinator; import io.streamnative.pulsar.handlers.kop.utils.ssl.SSLUtils; import lombok.Getter; +import org.apache.bookkeeper.stats.StatsLogger; import org.apache.pulsar.broker.PulsarService; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -48,13 +49,16 @@ public class KafkaChannelInitializer extends ChannelInitializer { private final EndPoint advertisedEndPoint; @Getter private final SslContextFactory sslContextFactory; + @Getter + private final StatsLogger statsLogger; public KafkaChannelInitializer(PulsarService pulsarService, KafkaServiceConfiguration kafkaConfig, GroupCoordinator groupCoordinator, TransactionCoordinator transactionCoordinator, boolean enableTLS, - EndPoint advertisedEndPoint) { + EndPoint advertisedEndPoint, + StatsLogger statsLogger) { super(); this.pulsarService = pulsarService; this.kafkaConfig = kafkaConfig; @@ -62,6 +66,7 @@ public KafkaChannelInitializer(PulsarService pulsarService, this.transactionCoordinator = transactionCoordinator; this.enableTls = enableTLS; this.advertisedEndPoint = advertisedEndPoint; + this.statsLogger = statsLogger; if (enableTls) { sslContextFactory = SSLUtils.createSslContextFactory(kafkaConfig); @@ -80,7 +85,7 @@ protected void initChannel(SocketChannel ch) throws Exception { new LengthFieldBasedFrameDecoder(MAX_FRAME_LENGTH, 0, 4, 0, 4)); ch.pipeline().addLast("handler", new KafkaRequestHandler(pulsarService, kafkaConfig, - groupCoordinator, transactionCoordinator, enableTls, advertisedEndPoint)); + groupCoordinator, transactionCoordinator, enableTls, advertisedEndPoint, statsLogger)); } } diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaProtocolHandler.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaProtocolHandler.java index d0426521a7..42e99518ba 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaProtocolHandler.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaProtocolHandler.java @@ -14,6 +14,7 @@ package io.streamnative.pulsar.handlers.kop; import static com.google.common.base.Preconditions.checkState; +import static io.streamnative.pulsar.handlers.kop.KopServerStats.SERVER_SCOPE; import static io.streamnative.pulsar.handlers.kop.utils.TopicNameUtils.getKafkaTopicNameFromPulsarTopicname; import static org.apache.pulsar.common.naming.TopicName.PARTITIONED_TOPIC_SUFFIX; @@ -39,6 +40,7 @@ import java.util.stream.Collectors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.stats.StatsLogger; import org.apache.kafka.common.internals.Topic; import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.security.auth.SecurityProtocol; @@ -66,6 +68,8 @@ public class KafkaProtocolHandler implements ProtocolHandler { public static final String PROTOCOL_NAME = "kafka"; public static final String TLS_HANDLER = "tls"; + private StatsLogger rootStatsLogger; + private StatsProviderService statsProviderService; /** * Listener for the changing of topic that stores offsets of consumer group. */ @@ -223,6 +227,9 @@ public void initialize(ServiceConfiguration conf) throws Exception { } this.bindAddress = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(kafkaConfig.getBindAddress()); KopTopic.initialize(kafkaConfig.getKafkaTenant() + "/" + kafkaConfig.getKafkaNamespace()); + + statsProviderService = new StatsProviderService(kafkaConfig); + rootStatsLogger = statsProviderService.getStatsProvider().getStatsLogger(""); } // This method is called after initialize @@ -264,6 +271,9 @@ public void start(BrokerService service) { log.error("Initialized transaction coordinator failed.", e); } } + + statsProviderService.start(); + } // this is called after initialize, and with kafkaConfig, brokerService all set. @@ -291,12 +301,14 @@ public Map> newChannelIniti case PLAINTEXT: case SASL_PLAINTEXT: builder.put(endPoint.getInetAddress(), new KafkaChannelInitializer(brokerService.getPulsar(), - kafkaConfig, groupCoordinator, transactionCoordinator, false, advertisedEndPoint)); + kafkaConfig, groupCoordinator, transactionCoordinator, false, + advertisedEndPoint, rootStatsLogger.scope(SERVER_SCOPE))); break; case SSL: case SASL_SSL: builder.put(endPoint.getInetAddress(), new KafkaChannelInitializer(brokerService.getPulsar(), - kafkaConfig, groupCoordinator, transactionCoordinator, true, advertisedEndPoint)); + kafkaConfig, groupCoordinator, transactionCoordinator, true, + advertisedEndPoint, rootStatsLogger.scope(SERVER_SCOPE))); break; } }); diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaRequestHandler.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaRequestHandler.java index 133124d4af..75e75abb88 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaRequestHandler.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaRequestHandler.java @@ -57,6 +57,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -64,11 +65,13 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.common.util.MathUtils; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.bookkeeper.stats.StatsLogger; import org.apache.commons.lang3.NotImplementedException; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; @@ -157,6 +160,7 @@ import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData; import org.apache.pulsar.zookeeper.ZooKeeperCache; import org.apache.pulsar.zookeeper.ZooKeeperCache.Deserializer; +import org.inferred.freebuilder.shaded.org.openjdk.tools.javac.util.MatchingUtils; /** * This class contains all the request handling methods. @@ -190,13 +194,16 @@ public class KafkaRequestHandler extends KafkaCommandDecoder { private final EntryFormatter entryFormatter; private final Map pendingProduceQueueMap = new ConcurrentHashMap<>(); + private final StatsLogger statsLogger; + private final RequestStats requestStats; public KafkaRequestHandler(PulsarService pulsarService, KafkaServiceConfiguration kafkaConfig, GroupCoordinator groupCoordinator, TransactionCoordinator transactionCoordinator, Boolean tlsEnabled, - EndPoint advertisedEndPoint) throws Exception { + EndPoint advertisedEndPoint, + StatsLogger statsLogger) throws Exception { super(); this.pulsarService = pulsarService; this.kafkaConfig = kafkaConfig; @@ -220,6 +227,8 @@ public KafkaRequestHandler(PulsarService pulsarService, this.entryFormatter = EntryFormatterFactory.create(kafkaConfig.getEntryFormat()); this.currentConnectedGroup = new ConcurrentHashMap<>(); this.groupIdStoredPath = kafkaConfig.getGroupIdZooKeeperPath(); + this.statsLogger = statsLogger; + this.requestStats = new RequestStats(statsLogger); } @Override @@ -618,6 +627,8 @@ protected void handleTopicMetadataRequest(KafkaHeaderAndRequest metadataHar, protected void handleProduceRequest(KafkaHeaderAndRequest produceHar, CompletableFuture resultFuture) { + final long startProduceNanos = MathUtils.nowInNano(); + checkArgument(produceHar.getRequest() instanceof ProduceRequest); ProduceRequest produceRequest = (ProduceRequest) produceHar.getRequest(); if (produceRequest.transactionalId() != null) { @@ -650,7 +661,7 @@ protected void handleProduceRequest(KafkaHeaderAndRequest produceHar, String fullPartitionName = KopTopic.toString(topicPartition); PendingProduce pendingProduce = new PendingProduce(partitionResponse, topicManager, fullPartitionName, - entryFormatter, validRecords, executor, transactionCoordinator); + entryFormatter, validRecords, executor, transactionCoordinator, requestStats); PendingProduceQueue queue = pendingProduceQueueMap.computeIfAbsent(topicPartition, ignored -> new PendingProduceQueue()); queue.add(pendingProduce); @@ -675,6 +686,15 @@ protected void handleProduceRequest(KafkaHeaderAndRequest produceHar, log.debug("[{}] Request {}: Complete handle produce.", ctx.channel(), produceHar.toString()); } + + if (ex != null) { + requestStats.getHandleProduceRequestStats() + .registerFailedEvent(MathUtils.elapsedNanos(startProduceNanos), TimeUnit.NANOSECONDS); + } else { + requestStats.getHandleProduceRequestStats() + .registerSuccessfulEvent(MathUtils.elapsedNanos(startProduceNanos), TimeUnit.NANOSECONDS); + } + resultFuture.complete(new ProduceResponse(responses)); }); } diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaServiceConfiguration.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaServiceConfiguration.java index 92040c79c9..0b3735cde6 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaServiceConfiguration.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaServiceConfiguration.java @@ -321,6 +321,12 @@ public class KafkaServiceConfiguration extends ServiceConfiguration { ) private String kopOauth2ConfigFile; + @FieldContext( + category = CATEGORY_KOP, + doc = "KOP Prometheus stats rollover latency" + ) + private int kopPrometheusStatsLatencyRolloverSeconds = 60; + public @NonNull String getKafkaAdvertisedListeners() { if (kafkaAdvertisedListeners != null) { return kafkaAdvertisedListeners; diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java new file mode 100644 index 0000000000..a864c021c5 --- /dev/null +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java @@ -0,0 +1,19 @@ +package io.streamnative.pulsar.handlers.kop; + +public interface KopServerStats { + String CATEGORY_SERVER = "server"; + + String SERVER_SCOPE = "kop_server"; + + String SERVER_STATUS = "SERVER_STATUS"; + + + /** + * PRODUCE STATS + */ + String HANDLE_PRODUCE_REQUEST = "HANDLE_PRODUCE_REQUEST"; + String PRODUCE_ENCODE = "PRODUCE_ENCODE"; + String MESSAGE_PUBLISH = "MESSAGE_PUBLISH"; + String MESSAGE_QUEUED_LATENCY = "MESSAGE_QUEUED_LATENCY"; + +} diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/PendingProduce.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/PendingProduce.java index f0a3248027..980995205b 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/PendingProduce.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/PendingProduce.java @@ -19,7 +19,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.common.util.MathUtils; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.RecordBatch; @@ -44,6 +46,8 @@ public class PendingProduce { private final TransactionCoordinator transactionCoordinator; private long pid; private boolean isTransactional; + private RequestStats requestStats; + private final long enqueueTimestamp; public PendingProduce(CompletableFuture responseFuture, KafkaTopicManager topicManager, @@ -51,7 +55,9 @@ public PendingProduce(CompletableFuture responseFuture, EntryFormatter entryFormatter, MemoryRecords memoryRecords, ExecutorService executor, - TransactionCoordinator transactionCoordinator) { + TransactionCoordinator transactionCoordinator, + RequestStats requestStats) { + this.enqueueTimestamp = MathUtils.nowInNano(); this.responseFuture = responseFuture; this.topicManager = topicManager; this.partitionName = partitionName; @@ -67,12 +73,15 @@ public PendingProduce(CompletableFuture responseFuture, return null; }); executor.execute(() -> byteBufFuture.complete(entryFormatter.encode(memoryRecords, numMessages))); + requestStats.getProduceEncodeStats() + .registerSuccessfulEvent(MathUtils.elapsedNanos(enqueueTimestamp), TimeUnit.NANOSECONDS); this.offsetFuture = new CompletableFuture<>(); RecordBatch batch = memoryRecords.batchIterator().next(); this.transactionCoordinator = transactionCoordinator; this.pid = batch.producerId(); this.isTransactional = batch.isTransactional(); + this.requestStats = requestStats; } public boolean ready() { @@ -103,6 +112,10 @@ public void publishMessages() { if (!ready()) { throw new RuntimeException("Try to send while PendingProduce is not ready"); } + + requestStats.getMessageQueuedLatencyStats() + .registerSuccessfulEvent(MathUtils.elapsedNanos(enqueueTimestamp), TimeUnit.NANOSECONDS); + PersistentTopic persistentTopic; ByteBuf byteBuf; try { @@ -129,9 +142,15 @@ public void publishMessages() { if (this.isTransactional) { transactionCoordinator.addActivePidOffset(TopicName.get(partitionName), pid, offset); } + requestStats.getMessagePublishStats().registerSuccessfulEvent( + MathUtils.elapsedNanos(enqueueTimestamp), TimeUnit.NANOSECONDS); + responseFuture.complete(new PartitionResponse(Errors.NONE, offset, -1L, -1L)); } else { log.error("publishMessages for topic partition: {} failed when write.", partitionName, e); + requestStats.getMessagePublishStats() + .registerFailedEvent(MathUtils.elapsedNanos(enqueueTimestamp), TimeUnit.NANOSECONDS); + responseFuture.complete(new PartitionResponse(Errors.KAFKA_STORAGE_ERROR)); } byteBuf.release(); diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/RequestStats.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/RequestStats.java new file mode 100644 index 0000000000..653f88e24d --- /dev/null +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/RequestStats.java @@ -0,0 +1,53 @@ +package io.streamnative.pulsar.handlers.kop; + +import lombok.Getter; +import org.apache.bookkeeper.stats.OpStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.bookkeeper.stats.annotations.StatsDoc; + +import static io.streamnative.pulsar.handlers.kop.KopServerStats.CATEGORY_SERVER; +import static io.streamnative.pulsar.handlers.kop.KopServerStats.HANDLE_PRODUCE_REQUEST; +import static io.streamnative.pulsar.handlers.kop.KopServerStats.MESSAGE_PUBLISH; +import static io.streamnative.pulsar.handlers.kop.KopServerStats.MESSAGE_QUEUED_LATENCY; +import static io.streamnative.pulsar.handlers.kop.KopServerStats.PRODUCE_ENCODE; +import static io.streamnative.pulsar.handlers.kop.KopServerStats.SERVER_SCOPE; + +@StatsDoc( + name = SERVER_SCOPE, + category = CATEGORY_SERVER, + help = "KOP request stats" +) + +@Getter +public class RequestStats { + @StatsDoc( + name = HANDLE_PRODUCE_REQUEST, + help = "handle produce request stats of Kop" + ) + private final OpStatsLogger handleProduceRequestStats; + + @StatsDoc( + name = PRODUCE_ENCODE, + help = "produce encode stats of Kop" + ) + private final OpStatsLogger produceEncodeStats; + + @StatsDoc( + name = MESSAGE_PUBLISH, + help = "message publish stats from kop to pulsar broker" + ) + private final OpStatsLogger messagePublishStats; + + @StatsDoc( + name = MESSAGE_QUEUED_LATENCY, + help = "message queued stats from kop to pulsar broker" + ) + private final OpStatsLogger messageQueuedLatencyStats; + + public RequestStats(StatsLogger statsLogger) { + this.handleProduceRequestStats = statsLogger.getOpStatsLogger(HANDLE_PRODUCE_REQUEST); + this.produceEncodeStats = statsLogger.getOpStatsLogger(PRODUCE_ENCODE); + this.messagePublishStats = statsLogger.getOpStatsLogger(MESSAGE_PUBLISH); + this.messageQueuedLatencyStats = statsLogger.getOpStatsLogger(MESSAGE_QUEUED_LATENCY); + } +} diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/StatsProviderService.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/StatsProviderService.java new file mode 100644 index 0000000000..402478ad76 --- /dev/null +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/StatsProviderService.java @@ -0,0 +1,37 @@ +package io.streamnative.pulsar.handlers.kop; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.stats.StatsProvider; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.PropertiesConfiguration; + +@Slf4j +public class StatsProviderService { + private final StatsProvider statsProvider; + private final Configuration conf; + + public StatsProviderService(KafkaServiceConfiguration kafkaServiceConfiguration) throws Exception { + Class statsProviderClass = Class.forName(kafkaServiceConfiguration.getKopStatsProviderClass()); + this.statsProvider = (StatsProvider) statsProviderClass.newInstance(); + + conf = new PropertiesConfiguration(); + conf.addProperty("prometheusStatsHttpEnable", kafkaServiceConfiguration.isKopPrometheusStatsHttpEnable()); + conf.addProperty("prometheusStatsHttpPort", kafkaServiceConfiguration.getKopPrometheusPort()); + conf.addProperty("prometheusStatsLatencyRolloverSeconds", + kafkaServiceConfiguration.getKopPrometheusStatsLatencyRolloverSeconds()); + conf.addProperty("prometheusBasicStatsEnable", false); + } + + public StatsProvider getStatsProvider() { + return this.statsProvider; + } + + protected void start() { + statsProvider.start(conf); + } + + protected void stop() { + statsProvider.stop(); + } + +} diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/DataSketchesOpStatsLogger.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/DataSketchesOpStatsLogger.java new file mode 100644 index 0000000000..fdb0331b37 --- /dev/null +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/DataSketchesOpStatsLogger.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.streamnative.pulsar.handlers.kop.stats; + +import com.yahoo.sketches.quantiles.DoublesSketch; +import com.yahoo.sketches.quantiles.DoublesSketchBuilder; +import com.yahoo.sketches.quantiles.DoublesUnion; +import com.yahoo.sketches.quantiles.DoublesUnionBuilder; +import io.netty.util.concurrent.FastThreadLocal; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; +import java.util.concurrent.locks.StampedLock; +import org.apache.bookkeeper.stats.OpStatsData; +import org.apache.bookkeeper.stats.OpStatsLogger; + +/** + * OpStatsLogger implementation that uses DataSketches library to calculate the approximated latency quantiles. + */ +public class DataSketchesOpStatsLogger implements OpStatsLogger { + + /* + * Use 2 rotating thread local accessor so that we can safely swap them. + */ + private volatile ThreadLocalAccessor current; + private volatile ThreadLocalAccessor replacement; + + /* + * These are the sketches where all the aggregated results are published. + */ + private volatile DoublesSketch successResult; + private volatile DoublesSketch failResult; + + private final LongAdder successCountAdder = new LongAdder(); + private final LongAdder failCountAdder = new LongAdder(); + + private final LongAdder successSumAdder = new LongAdder(); + private final LongAdder failSumAdder = new LongAdder(); + + DataSketchesOpStatsLogger() { + this.current = new ThreadLocalAccessor(); + this.replacement = new ThreadLocalAccessor(); + } + + @Override + public void registerFailedEvent(long eventLatency, TimeUnit unit) { + double valueMillis = unit.toMicros(eventLatency) / 1000.0; + + failCountAdder.increment(); + failSumAdder.add((long) valueMillis); + + LocalData localData = current.localData.get(); + + long stamp = localData.lock.readLock(); + try { + localData.failSketch.update(valueMillis); + } finally { + localData.lock.unlockRead(stamp); + } + } + + @Override + public void registerSuccessfulEvent(long eventLatency, TimeUnit unit) { + double valueMillis = unit.toMicros(eventLatency) / 1000.0; + + successCountAdder.increment(); + successSumAdder.add((long) valueMillis); + + LocalData localData = current.localData.get(); + + long stamp = localData.lock.readLock(); + try { + localData.successSketch.update(valueMillis); + } finally { + localData.lock.unlockRead(stamp); + } + } + + @Override + public void registerSuccessfulValue(long value) { + successCountAdder.increment(); + successSumAdder.add(value); + + LocalData localData = current.localData.get(); + + long stamp = localData.lock.readLock(); + try { + localData.successSketch.update(value); + } finally { + localData.lock.unlockRead(stamp); + } + } + + @Override + public void registerFailedValue(long value) { + failCountAdder.increment(); + failSumAdder.add(value); + + LocalData localData = current.localData.get(); + + long stamp = localData.lock.readLock(); + try { + localData.failSketch.update(value); + } finally { + localData.lock.unlockRead(stamp); + } + } + + @Override + public OpStatsData toOpStatsData() { + // Not relevant as we don't use JMX here + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + // Not relevant as we don't use JMX here + throw new UnsupportedOperationException(); + } + + public void rotateLatencyCollection() { + // Swap current with replacement + ThreadLocalAccessor local = current; + current = replacement; + replacement = local; + + final DoublesUnion aggregateSuccesss = new DoublesUnionBuilder().build(); + final DoublesUnion aggregateFail = new DoublesUnionBuilder().build(); + local.map.forEach((localData, b) -> { + long stamp = localData.lock.writeLock(); + try { + aggregateSuccesss.update(localData.successSketch); + localData.successSketch.reset(); + aggregateFail.update(localData.failSketch); + localData.failSketch.reset(); + } finally { + localData.lock.unlockWrite(stamp); + } + }); + + successResult = aggregateSuccesss.getResultAndReset(); + failResult = aggregateFail.getResultAndReset(); + } + + public long getCount(boolean success) { + return success ? successCountAdder.sum() : failCountAdder.sum(); + } + + public long getSum(boolean success) { + return success ? successSumAdder.sum() : failSumAdder.sum(); + } + + public double getQuantileValue(boolean success, double quantile) { + DoublesSketch s = success ? successResult : failResult; + return s != null ? s.getQuantile(quantile) : Double.NaN; + } + + private static class LocalData { + private final DoublesSketch successSketch = new DoublesSketchBuilder().build(); + private final DoublesSketch failSketch = new DoublesSketchBuilder().build(); + private final StampedLock lock = new StampedLock(); + } + + private static class ThreadLocalAccessor { + private final Map map = new ConcurrentHashMap<>(); + private final FastThreadLocal localData = new FastThreadLocal() { + + @Override + protected LocalData initialValue() throws Exception { + LocalData localData = new LocalData(); + map.put(localData, Boolean.TRUE); + return localData; + } + + @Override + protected void onRemoval(LocalData value) throws Exception { + map.remove(value); + } + }; + } +} diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/LongAdderCounter.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/LongAdderCounter.java new file mode 100644 index 0000000000..2d3e35b22f --- /dev/null +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/LongAdderCounter.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.streamnative.pulsar.handlers.kop.stats; + +import java.util.concurrent.atomic.LongAdder; +import org.apache.bookkeeper.stats.Counter; + +/** + * {@link Counter} implementation based on {@link LongAdder}. + * + *

LongAdder keeps a counter per-thread and then aggregates to get the result, in order to avoid contention between + * multiple threads. + */ +public class LongAdderCounter implements Counter { + private final LongAdder counter = new LongAdder(); + + @Override + public void clear() { + counter.reset(); + } + + @Override + public void inc() { + counter.increment(); + } + + @Override + public void dec() { + counter.decrement(); + } + + @Override + public void add(long delta) { + counter.add(delta); + } + + @Override + public Long get() { + return counter.sum(); + } +} diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java new file mode 100644 index 0000000000..cc028c4cf9 --- /dev/null +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.streamnative.pulsar.handlers.kop.stats; + +import com.google.common.annotations.VisibleForTesting; +import io.netty.util.concurrent.DefaultThreadFactory; +import io.netty.util.internal.PlatformDependent; +import io.prometheus.client.Collector; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.Gauge; +import io.prometheus.client.Gauge.Child; +import io.prometheus.client.hotspot.GarbageCollectorExports; +import io.prometheus.client.hotspot.MemoryPoolsExports; +import io.prometheus.client.hotspot.StandardExports; +import io.prometheus.client.hotspot.ThreadExports; +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.bookkeeper.stats.CachingStatsProvider; +import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.bookkeeper.stats.StatsProvider; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.lang.StringUtils; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +//CHECKSTYLE.OFF: IllegalImport +//CHECKSTYLE.ON: IllegalImport + +/** + * A Prometheus based {@link StatsProvider} implementation. + */ +public class PrometheusMetricsProvider implements StatsProvider { + + private ScheduledExecutorService executor; + + public static final String PROMETHEUS_STATS_HTTP_ENABLE = "prometheusStatsHttpEnable"; + public static final boolean DEFAULT_PROMETHEUS_STATS_HTTP_ENABLE = true; + + public static final String PROMETHEUS_STATS_HTTP_ADDRESS = "prometheusStatsHttpAddress"; + public static final String DEFAULT_PROMETHEUS_STATS_HTTP_ADDR = "0.0.0.0"; + + public static final String PROMETHEUS_STATS_HTTP_PORT = "prometheusStatsHttpPort"; + public static final int DEFAULT_PROMETHEUS_STATS_HTTP_PORT = 8000; + + public static final String PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS = "prometheusStatsLatencyRolloverSeconds"; + public static final int DEFAULT_PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS = 60; + + public static final String PROMETHEUS_BASIC_STATS_ENABLE = "prometheusBasicStatsEnable"; + public static final boolean DEFAULT_PROMETHEUS_BASIC_STATS_ENABLE = true; + + final CollectorRegistry registry; + + Server server; + private final CachingStatsProvider cachingStatsProvider; + + /* + * These acts a registry of the metrics defined in this provider + */ + final ConcurrentMap counters = new ConcurrentSkipListMap<>(); + final ConcurrentMap> gauges = new ConcurrentSkipListMap<>(); + final ConcurrentMap opStats = new ConcurrentSkipListMap<>(); + + public PrometheusMetricsProvider() { + this(CollectorRegistry.defaultRegistry); + } + + public PrometheusMetricsProvider(CollectorRegistry registry) { + this.registry = registry; + this.cachingStatsProvider = new CachingStatsProvider(new StatsProvider() { + @Override + public void start(Configuration conf) { + // nop + } + + @Override + public void stop() { + // nop + } + + @Override + public StatsLogger getStatsLogger(String scope) { + return new PrometheusStatsLogger(PrometheusMetricsProvider.this, scope); + } + + @Override + public String getStatsName(String... statsComponents) { + String completeName; + if (statsComponents.length == 0) { + return ""; + } else if (statsComponents[0].isEmpty()) { + completeName = StringUtils.join(statsComponents, '_', 1, statsComponents.length); + } else { + completeName = StringUtils.join(statsComponents, '_'); + } + return Collector.sanitizeMetricName(completeName); + } + }); + } + + @Override + public void start(Configuration conf) { + boolean httpEnabled = conf.getBoolean(PROMETHEUS_STATS_HTTP_ENABLE, DEFAULT_PROMETHEUS_STATS_HTTP_ENABLE); + boolean bkHttpServerEnabled = conf.getBoolean("httpServerEnabled", false); + // only start its own http server when prometheus http is enabled and bk http server is not enabled. + if (httpEnabled && !bkHttpServerEnabled) { + String httpAddr = conf.getString(PROMETHEUS_STATS_HTTP_ADDRESS, DEFAULT_PROMETHEUS_STATS_HTTP_ADDR); + int httpPort = conf.getInt(PROMETHEUS_STATS_HTTP_PORT, DEFAULT_PROMETHEUS_STATS_HTTP_PORT); + InetSocketAddress httpEndpoint = InetSocketAddress.createUnresolved(httpAddr, httpPort); + this.server = new Server(httpEndpoint); + ServletContextHandler context = new ServletContextHandler(); + context.setContextPath("/"); + server.setHandler(context); + + context.addServlet(new ServletHolder(new PrometheusServlet(this)), "/metrics"); + + try { + server.start(); + log.info("Started Prometheus stats endpoint at {}", httpEndpoint); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + boolean basicStatsEnabled = conf.getBoolean(PROMETHEUS_BASIC_STATS_ENABLE, + DEFAULT_PROMETHEUS_BASIC_STATS_ENABLE); + if (basicStatsEnabled) { + // Include standard JVM stats + registerMetrics(new StandardExports()); + registerMetrics(new MemoryPoolsExports()); + registerMetrics(new GarbageCollectorExports()); + registerMetrics(new ThreadExports()); + + // Add direct memory allocated through unsafe + registerMetrics(Gauge.build("jvm_memory_direct_bytes_used", "-").create().setChild(new Child() { + @Override + public double get() { + return directMemoryUsage != null ? directMemoryUsage.longValue() : Double.NaN; + } + })); + + registerMetrics(Gauge.build("jvm_memory_direct_bytes_max", "-").create().setChild(new Child() { + @Override + public double get() { + return PlatformDependent.maxDirectMemory(); + } + })); + } + + executor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("metrics")); + + int latencyRolloverSeconds = conf.getInt(PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS, + DEFAULT_PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS); + + executor.scheduleAtFixedRate(() -> { + rotateLatencyCollection(); + }, 1, latencyRolloverSeconds, TimeUnit.SECONDS); + + } + + @Override + public void stop() { + if (server != null) { + try { + server.stop(); + } catch (Exception e) { + log.warn("Failed to shutdown Jetty server", e); + } + } + } + + @Override + public StatsLogger getStatsLogger(String scope) { + return this.cachingStatsProvider.getStatsLogger(scope); + } + + @Override + public void writeAllMetrics(Writer writer) throws IOException { + PrometheusTextFormatUtil.writeMetricsCollectedByPrometheusClient(writer, registry); + + gauges.forEach((name, gauge) -> PrometheusTextFormatUtil.writeGauge(writer, name, gauge)); + counters.forEach((name, counter) -> PrometheusTextFormatUtil.writeCounter(writer, name, counter)); + opStats.forEach((name, opStatLogger) -> PrometheusTextFormatUtil.writeOpStat(writer, name, opStatLogger)); + } + + @Override + public String getStatsName(String... statsComponents) { + return cachingStatsProvider.getStatsName(statsComponents); + } + + @VisibleForTesting + void rotateLatencyCollection() { + opStats.forEach((name, metric) -> { + metric.rotateLatencyCollection(); + }); + } + + private void registerMetrics(Collector collector) { + try { + collector.register(registry); + } catch (Exception e) { + // Ignore if these were already registered + if (log.isDebugEnabled()) { + log.debug("Failed to register Prometheus collector exports", e); + } + } + } + + + private static final Logger log = LoggerFactory.getLogger(org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider.class); + + /* + * Try to get Netty counter of used direct memory. This will be correct, unlike the JVM values. + */ + private static final AtomicLong directMemoryUsage; + static { + AtomicLong tmpDirectMemoryUsage = null; + + try { + Field field = PlatformDependent.class.getDeclaredField("DIRECT_MEMORY_COUNTER"); + field.setAccessible(true); + tmpDirectMemoryUsage = (AtomicLong) field.get(null); + } catch (Throwable t) { + log.warn("Failed to access netty DIRECT_MEMORY_COUNTER field {}", t.getMessage()); + } + + directMemoryUsage = tmpDirectMemoryUsage; + } +} diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusServlet.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusServlet.java new file mode 100644 index 0000000000..2caef9a22a --- /dev/null +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusServlet.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.streamnative.pulsar.handlers.kop.stats; + +import io.prometheus.client.exporter.common.TextFormat; +import java.io.IOException; +import java.io.Writer; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Servlet used to export metrics in prometheus text format. + */ +public class PrometheusServlet extends HttpServlet { + private static final long serialVersionUID = 1L; + + private final transient PrometheusMetricsProvider provider; + + public PrometheusServlet(PrometheusMetricsProvider provider) { + this.provider = provider; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + resp.setStatus(HttpServletResponse.SC_OK); + resp.setContentType(TextFormat.CONTENT_TYPE_004); + + Writer writer = resp.getWriter(); + try { + provider.writeAllMetrics(writer); + writer.flush(); + } finally { + writer.close(); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + doGet(req, resp); + } + +} diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusStatsLogger.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusStatsLogger.java new file mode 100644 index 0000000000..4fc1911b04 --- /dev/null +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusStatsLogger.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.streamnative.pulsar.handlers.kop.stats; + +import com.google.common.base.Joiner; +import io.prometheus.client.Collector; +import org.apache.bookkeeper.stats.Counter; +import org.apache.bookkeeper.stats.Gauge; +import org.apache.bookkeeper.stats.OpStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; + +/** + * A {@code Prometheus} based {@link StatsLogger} implementation. + */ +public class PrometheusStatsLogger implements StatsLogger { + + private final PrometheusMetricsProvider provider; + private final String scope; + + PrometheusStatsLogger(PrometheusMetricsProvider provider, String scope) { + this.provider = provider; + this.scope = scope; + } + + @Override + public OpStatsLogger getOpStatsLogger(String name) { + return provider.opStats.computeIfAbsent(completeName(name), x -> new DataSketchesOpStatsLogger()); + } + + @Override + public Counter getCounter(String name) { + return provider.counters.computeIfAbsent(completeName(name), x -> new LongAdderCounter()); + } + + @Override + public void registerGauge(String name, Gauge gauge) { + provider.gauges.computeIfAbsent(completeName(name), x -> new SimpleGauge(gauge)); + } + + @Override + public void unregisterGauge(String name, Gauge gauge) { + // no-op + } + + @Override + public void removeScope(String name, StatsLogger statsLogger) { + // no-op + } + + @Override + public StatsLogger scope(String name) { + return new PrometheusStatsLogger(provider, completeName(name)); + } + + private String completeName(String name) { + String completeName = scope.isEmpty() ? name : Joiner.on('_').join(scope, name); + return Collector.sanitizeMetricName(completeName); + } +} diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java new file mode 100644 index 0000000000..bbb7aea8d3 --- /dev/null +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.streamnative.pulsar.handlers.kop.stats; + +import io.prometheus.client.Collector; +import io.prometheus.client.Collector.MetricFamilySamples; +import io.prometheus.client.Collector.MetricFamilySamples.Sample; +import io.prometheus.client.CollectorRegistry; +import java.io.IOException; +import java.io.Writer; +import java.util.Enumeration; +import org.apache.bookkeeper.stats.Counter; + +/** + * Logic to write metrics in Prometheus text format. + */ +public class PrometheusTextFormatUtil { + static void writeGauge(Writer w, String name, SimpleGauge gauge) { + // Example: + // # TYPE bookie_storage_entries_count gauge + // bookie_storage_entries_count 519 + try { + w.append("# TYPE ").append(name).append(" gauge\n"); + w.append(name).append(' ').append(gauge.getSample().toString()).append('\n'); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + static void writeCounter(Writer w, String name, Counter counter) { + // Example: + // # TYPE jvm_threads_started_total counter + // jvm_threads_started_total 59 + try { + w.append("# TYPE ").append(name).append(" counter\n"); + w.append(name).append(' ').append(counter.get().toString()).append('\n'); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + static void writeOpStat(Writer w, String name, DataSketchesOpStatsLogger opStat) { + // Example: + // # TYPE bookie_journal_JOURNAL_ADD_ENTRY summary + // bookie_journal_JOURNAL_ADD_ENTRY{success="false",quantile="0.5",} NaN + // bookie_journal_JOURNAL_ADD_ENTRY{success="false",quantile="0.75",} NaN + // bookie_journal_JOURNAL_ADD_ENTRY{success="false",quantile="0.95",} NaN + // bookie_journal_JOURNAL_ADD_ENTRY{success="false",quantile="0.99",} NaN + // bookie_journal_JOURNAL_ADD_ENTRY{success="false",quantile="0.999",} NaN + // bookie_journal_JOURNAL_ADD_ENTRY{success="false",quantile="0.9999",} NaN + // bookie_journal_JOURNAL_ADD_ENTRY{success="false",quantile="1.0",} NaN + // bookie_journal_JOURNAL_ADD_ENTRY_count{success="false",} 0.0 + // bookie_journal_JOURNAL_ADD_ENTRY_sum{success="false",} 0.0 + // bookie_journal_JOURNAL_ADD_ENTRY{success="true",quantile="0.5",} 1.706 + // bookie_journal_JOURNAL_ADD_ENTRY{success="true",quantile="0.75",} 1.89 + // bookie_journal_JOURNAL_ADD_ENTRY{success="true",quantile="0.95",} 2.121 + // bookie_journal_JOURNAL_ADD_ENTRY{success="true",quantile="0.99",} 10.708 + // bookie_journal_JOURNAL_ADD_ENTRY{success="true",quantile="0.999",} 10.902 + // bookie_journal_JOURNAL_ADD_ENTRY{success="true",quantile="0.9999",} 10.902 + // bookie_journal_JOURNAL_ADD_ENTRY{success="true",quantile="1.0",} 10.902 + // bookie_journal_JOURNAL_ADD_ENTRY_count{success="true",} 658.0 + // bookie_journal_JOURNAL_ADD_ENTRY_sum{success="true",} 1265.0800000000002 + try { + w.append("# TYPE ").append(name).append(" summary\n"); + writeQuantile(w, opStat, name, false, 0.5); + writeQuantile(w, opStat, name, false, 0.75); + writeQuantile(w, opStat, name, false, 0.95); + writeQuantile(w, opStat, name, false, 0.99); + writeQuantile(w, opStat, name, false, 0.999); + writeQuantile(w, opStat, name, false, 0.9999); + writeQuantile(w, opStat, name, false, 1.0); + writeCount(w, opStat, name, false); + writeSum(w, opStat, name, false); + + writeQuantile(w, opStat, name, true, 0.5); + writeQuantile(w, opStat, name, true, 0.75); + writeQuantile(w, opStat, name, true, 0.95); + writeQuantile(w, opStat, name, true, 0.99); + writeQuantile(w, opStat, name, true, 0.999); + writeQuantile(w, opStat, name, true, 0.9999); + writeQuantile(w, opStat, name, true, 1.0); + writeCount(w, opStat, name, true); + writeSum(w, opStat, name, true); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static void writeQuantile(Writer w, DataSketchesOpStatsLogger opStat, String name, Boolean success, + double quantile) throws IOException { + w.append(name).append("{success=\"").append(success.toString()).append("\",quantile=\"") + .append(Double.toString(quantile)).append("\"} ") + .append(Double.toString(opStat.getQuantileValue(success, quantile))).append('\n'); + } + + private static void writeCount(Writer w, DataSketchesOpStatsLogger opStat, String name, Boolean success) + throws IOException { + w.append(name).append("_count{success=\"").append(success.toString()).append("\"} ") + .append(Long.toString(opStat.getCount(success))).append('\n'); + } + + private static void writeSum(Writer w, DataSketchesOpStatsLogger opStat, String name, Boolean success) + throws IOException { + w.append(name).append("_sum{success=\"").append(success.toString()).append("\"} ") + .append(Double.toString(opStat.getSum(success))).append('\n'); + } + + static void writeMetricsCollectedByPrometheusClient(Writer w, CollectorRegistry registry) throws IOException { + Enumeration metricFamilySamples = registry.metricFamilySamples(); + while (metricFamilySamples.hasMoreElements()) { + MetricFamilySamples metricFamily = metricFamilySamples.nextElement(); + + for (int i = 0; i < metricFamily.samples.size(); i++) { + Sample sample = metricFamily.samples.get(i); + w.write(sample.name); + w.write('{'); + for (int j = 0; j < sample.labelNames.size(); j++) { + if (j != 0) { + w.write(", "); + } + w.write(sample.labelNames.get(j)); + w.write("=\""); + w.write(sample.labelValues.get(j)); + w.write('"'); + } + + w.write("} "); + w.write(Collector.doubleToGoString(sample.value)); + w.write('\n'); + } + } + } +} diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/SimpleGauge.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/SimpleGauge.java new file mode 100644 index 0000000000..0331b8ba1c --- /dev/null +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/SimpleGauge.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.streamnative.pulsar.handlers.kop.stats; + +import org.apache.bookkeeper.stats.Gauge; + +/** + * A {@link Gauge} implementation that forwards on the value supplier. + */ +public class SimpleGauge { + + // public SimpleGauge(CollectorRegistry registry, String name) { + // this.gauge = PrometheusUtil.safeRegister(registry, + // Gauge.build().name(Collector.sanitizeMetricName(name)).help("-").create()); + // } + + private final Gauge gauge; + + public SimpleGauge(final Gauge gauge) { + this.gauge = gauge; + } + + Number getSample() { + return gauge.getSample(); + } +} diff --git a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/EntryPublishTimeTest.java b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/EntryPublishTimeTest.java index c24a9b822e..054b5198fd 100644 --- a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/EntryPublishTimeTest.java +++ b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/EntryPublishTimeTest.java @@ -23,6 +23,7 @@ import io.streamnative.pulsar.handlers.kop.coordinator.transaction.TransactionCoordinator; import java.net.InetSocketAddress; import java.net.SocketAddress; +import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.pulsar.broker.protocol.ProtocolHandler; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.RetentionPolicies; @@ -90,7 +91,8 @@ protected void setup() throws Exception { groupCoordinator, transactionCoordinator, false, - getPlainEndPoint()); + getPlainEndPoint(), + NullStatsLogger.INSTANCE); ChannelHandlerContext mockCtx = mock(ChannelHandlerContext.class); Channel mockChannel = mock(Channel.class); doReturn(mockChannel).when(mockCtx).channel(); diff --git a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaApisTest.java b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaApisTest.java index eae817c433..233d797300 100644 --- a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaApisTest.java +++ b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaApisTest.java @@ -44,6 +44,7 @@ import java.util.stream.Collectors; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.commons.lang3.tuple.Pair; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; @@ -141,7 +142,8 @@ protected void setup() throws Exception { groupCoordinator, transactionCoordinator, false, - getPlainEndPoint()); + getPlainEndPoint(), + NullStatsLogger.INSTANCE); ChannelHandlerContext mockCtx = mock(ChannelHandlerContext.class); Channel mockChannel = mock(Channel.class); doReturn(mockChannel).when(mockCtx).channel(); diff --git a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaRequestHandlerTest.java b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaRequestHandlerTest.java index d82578b5d0..4264e591a3 100644 --- a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaRequestHandlerTest.java +++ b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaRequestHandlerTest.java @@ -50,6 +50,7 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.admin.NewTopic; @@ -146,7 +147,8 @@ protected void setup() throws Exception { groupCoordinator, transactionCoordinator, false, - getPlainEndPoint()); + getPlainEndPoint(), + NullStatsLogger.INSTANCE); } @AfterMethod diff --git a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaTopicConsumerManagerTest.java b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaTopicConsumerManagerTest.java index bb3ddb516a..0eca00c80e 100644 --- a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaTopicConsumerManagerTest.java +++ b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/KafkaTopicConsumerManagerTest.java @@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; +import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.commons.lang3.tuple.Pair; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; @@ -69,7 +70,8 @@ protected void setup() throws Exception { groupCoordinator, transactionCoordinator, false, - getPlainEndPoint()); + getPlainEndPoint(), + NullStatsLogger.INSTANCE); ChannelHandlerContext mockCtx = mock(ChannelHandlerContext.class); Channel mockChannel = mock(Channel.class); From 376eb6303ee6f4423fbf055457916dfb367af952 Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Tue, 2 Mar 2021 10:09:20 +0800 Subject: [PATCH 02/11] format code --- .../pulsar/handlers/kop/KopServerStats.java | 13 +++++++++++++ .../pulsar/handlers/kop/RequestStats.java | 13 +++++++++++++ .../handlers/kop/StatsProviderService.java | 13 +++++++++++++ .../kop/stats/DataSketchesOpStatsLogger.java | 17 +++++++---------- .../handlers/kop/stats/LongAdderCounter.java | 17 +++++++---------- .../kop/stats/PrometheusMetricsProvider.java | 17 +++++++---------- .../handlers/kop/stats/PrometheusServlet.java | 17 +++++++---------- .../kop/stats/PrometheusStatsLogger.java | 17 +++++++---------- .../kop/stats/PrometheusTextFormatUtil.java | 17 +++++++---------- .../pulsar/handlers/kop/stats/SimpleGauge.java | 17 +++++++---------- 10 files changed, 88 insertions(+), 70 deletions(-) diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java index a864c021c5..852ae336d6 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java @@ -1,3 +1,16 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.streamnative.pulsar.handlers.kop; public interface KopServerStats { diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/RequestStats.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/RequestStats.java index 653f88e24d..b12c9029be 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/RequestStats.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/RequestStats.java @@ -1,3 +1,16 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.streamnative.pulsar.handlers.kop; import lombok.Getter; diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/StatsProviderService.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/StatsProviderService.java index 402478ad76..3fedf5a287 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/StatsProviderService.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/StatsProviderService.java @@ -1,3 +1,16 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.streamnative.pulsar.handlers.kop; import lombok.extern.slf4j.Slf4j; diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/DataSketchesOpStatsLogger.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/DataSketchesOpStatsLogger.java index fdb0331b37..e78c916f8d 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/DataSketchesOpStatsLogger.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/DataSketchesOpStatsLogger.java @@ -1,18 +1,15 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.streamnative.pulsar.handlers.kop.stats; diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/LongAdderCounter.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/LongAdderCounter.java index 2d3e35b22f..7d678fd0bb 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/LongAdderCounter.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/LongAdderCounter.java @@ -1,18 +1,15 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.streamnative.pulsar.handlers.kop.stats; diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java index cc028c4cf9..65bbbea75d 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java @@ -1,18 +1,15 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.streamnative.pulsar.handlers.kop.stats; diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusServlet.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusServlet.java index 2caef9a22a..21624ffe39 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusServlet.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusServlet.java @@ -1,18 +1,15 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.streamnative.pulsar.handlers.kop.stats; diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusStatsLogger.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusStatsLogger.java index 4fc1911b04..5727fca2fd 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusStatsLogger.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusStatsLogger.java @@ -1,18 +1,15 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.streamnative.pulsar.handlers.kop.stats; diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java index bbb7aea8d3..e6f3d89564 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java @@ -1,18 +1,15 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.streamnative.pulsar.handlers.kop.stats; diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/SimpleGauge.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/SimpleGauge.java index 0331b8ba1c..60b75702b8 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/SimpleGauge.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/SimpleGauge.java @@ -1,18 +1,15 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.streamnative.pulsar.handlers.kop.stats; From aa1dbf104a8314ce10d91806a4610eb90182ddaf Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Wed, 3 Mar 2021 14:56:05 +0800 Subject: [PATCH 03/11] format code --- .../io/streamnative/pulsar/handlers/kop/KafkaRequestHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaRequestHandler.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaRequestHandler.java index 75e75abb88..9c408cd0e4 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaRequestHandler.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaRequestHandler.java @@ -160,7 +160,6 @@ import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData; import org.apache.pulsar.zookeeper.ZooKeeperCache; import org.apache.pulsar.zookeeper.ZooKeeperCache.Deserializer; -import org.inferred.freebuilder.shaded.org.openjdk.tools.javac.util.MatchingUtils; /** * This class contains all the request handling methods. From bc2549703fad4672d515af5de26a3f1367e5f9ae Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Sun, 28 Mar 2021 12:39:03 +0800 Subject: [PATCH 04/11] use pulsar prometheusRawMetricsProvider to expose kop metric to pulsar metrics --- .../handlers/kop/KafkaProtocolHandler.java | 16 ++- .../kop/stats/PrometheusMetricsProvider.java | 103 +--------------- .../handlers/kop/stats/PrometheusServlet.java | 55 --------- .../kop/stats/PrometheusTextFormatUtil.java | 96 +++++++-------- .../handlers/kop/MetricsProviderTest.java | 110 ++++++++++++++++++ 5 files changed, 169 insertions(+), 211 deletions(-) delete mode 100644 kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusServlet.java create mode 100644 tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaProtocolHandler.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaProtocolHandler.java index 42e99518ba..a7409442ac 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaProtocolHandler.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaProtocolHandler.java @@ -26,6 +26,7 @@ import io.streamnative.pulsar.handlers.kop.coordinator.group.OffsetConfig; import io.streamnative.pulsar.handlers.kop.coordinator.transaction.TransactionConfig; import io.streamnative.pulsar.handlers.kop.coordinator.transaction.TransactionCoordinator; +import io.streamnative.pulsar.handlers.kop.stats.PrometheusMetricsProvider; import io.streamnative.pulsar.handlers.kop.utils.ConfigurationUtils; import io.streamnative.pulsar.handlers.kop.utils.KopTopic; import io.streamnative.pulsar.handlers.kop.utils.MetadataUtils; @@ -41,6 +42,8 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.kafka.common.internals.Topic; import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.security.auth.SecurityProtocol; @@ -69,7 +72,7 @@ public class KafkaProtocolHandler implements ProtocolHandler { public static final String TLS_HANDLER = "tls"; private StatsLogger rootStatsLogger; - private StatsProviderService statsProviderService; + private PrometheusMetricsProvider statsProvider; /** * Listener for the changing of topic that stores offsets of consumer group. */ @@ -228,8 +231,8 @@ public void initialize(ServiceConfiguration conf) throws Exception { this.bindAddress = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(kafkaConfig.getBindAddress()); KopTopic.initialize(kafkaConfig.getKafkaTenant() + "/" + kafkaConfig.getKafkaNamespace()); - statsProviderService = new StatsProviderService(kafkaConfig); - rootStatsLogger = statsProviderService.getStatsProvider().getStatsLogger(""); + statsProvider = new PrometheusMetricsProvider(); + rootStatsLogger = statsProvider.getStatsLogger(""); } // This method is called after initialize @@ -272,8 +275,11 @@ public void start(BrokerService service) { } } - statsProviderService.start(); - + Configuration conf = new PropertiesConfiguration(); + conf.addProperty("prometheusStatsLatencyRolloverSeconds", + kafkaConfig.getKopPrometheusStatsLatencyRolloverSeconds()); + statsProvider.start(conf); + brokerService.pulsar().addPrometheusRawMetricsProvider(statsProvider); } // this is called after initialize, and with kafkaConfig, brokerService all set. diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java index 65bbbea75d..3e5addf98b 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java @@ -39,40 +39,26 @@ import org.apache.bookkeeper.stats.StatsProvider; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; +import org.apache.pulsar.broker.stats.prometheus.PrometheusRawMetricsProvider; +import org.apache.pulsar.common.util.SimpleTextOutputStream; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -//CHECKSTYLE.OFF: IllegalImport -//CHECKSTYLE.ON: IllegalImport - /** * A Prometheus based {@link StatsProvider} implementation. */ -public class PrometheusMetricsProvider implements StatsProvider { +public class PrometheusMetricsProvider implements PrometheusRawMetricsProvider { private ScheduledExecutorService executor; - public static final String PROMETHEUS_STATS_HTTP_ENABLE = "prometheusStatsHttpEnable"; - public static final boolean DEFAULT_PROMETHEUS_STATS_HTTP_ENABLE = true; - - public static final String PROMETHEUS_STATS_HTTP_ADDRESS = "prometheusStatsHttpAddress"; - public static final String DEFAULT_PROMETHEUS_STATS_HTTP_ADDR = "0.0.0.0"; - - public static final String PROMETHEUS_STATS_HTTP_PORT = "prometheusStatsHttpPort"; - public static final int DEFAULT_PROMETHEUS_STATS_HTTP_PORT = 8000; - public static final String PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS = "prometheusStatsLatencyRolloverSeconds"; public static final int DEFAULT_PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS = 60; - public static final String PROMETHEUS_BASIC_STATS_ENABLE = "prometheusBasicStatsEnable"; - public static final boolean DEFAULT_PROMETHEUS_BASIC_STATS_ENABLE = true; - final CollectorRegistry registry; - Server server; private final CachingStatsProvider cachingStatsProvider; /* @@ -119,54 +105,7 @@ public String getStatsName(String... statsComponents) { }); } - @Override public void start(Configuration conf) { - boolean httpEnabled = conf.getBoolean(PROMETHEUS_STATS_HTTP_ENABLE, DEFAULT_PROMETHEUS_STATS_HTTP_ENABLE); - boolean bkHttpServerEnabled = conf.getBoolean("httpServerEnabled", false); - // only start its own http server when prometheus http is enabled and bk http server is not enabled. - if (httpEnabled && !bkHttpServerEnabled) { - String httpAddr = conf.getString(PROMETHEUS_STATS_HTTP_ADDRESS, DEFAULT_PROMETHEUS_STATS_HTTP_ADDR); - int httpPort = conf.getInt(PROMETHEUS_STATS_HTTP_PORT, DEFAULT_PROMETHEUS_STATS_HTTP_PORT); - InetSocketAddress httpEndpoint = InetSocketAddress.createUnresolved(httpAddr, httpPort); - this.server = new Server(httpEndpoint); - ServletContextHandler context = new ServletContextHandler(); - context.setContextPath("/"); - server.setHandler(context); - - context.addServlet(new ServletHolder(new PrometheusServlet(this)), "/metrics"); - - try { - server.start(); - log.info("Started Prometheus stats endpoint at {}", httpEndpoint); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - boolean basicStatsEnabled = conf.getBoolean(PROMETHEUS_BASIC_STATS_ENABLE, - DEFAULT_PROMETHEUS_BASIC_STATS_ENABLE); - if (basicStatsEnabled) { - // Include standard JVM stats - registerMetrics(new StandardExports()); - registerMetrics(new MemoryPoolsExports()); - registerMetrics(new GarbageCollectorExports()); - registerMetrics(new ThreadExports()); - - // Add direct memory allocated through unsafe - registerMetrics(Gauge.build("jvm_memory_direct_bytes_used", "-").create().setChild(new Child() { - @Override - public double get() { - return directMemoryUsage != null ? directMemoryUsage.longValue() : Double.NaN; - } - })); - - registerMetrics(Gauge.build("jvm_memory_direct_bytes_max", "-").create().setChild(new Child() { - @Override - public double get() { - return PlatformDependent.maxDirectMemory(); - } - })); - } executor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("metrics")); @@ -179,32 +118,21 @@ public double get() { } - @Override public void stop() { - if (server != null) { - try { - server.stop(); - } catch (Exception e) { - log.warn("Failed to shutdown Jetty server", e); - } - } + executor.shutdown(); } - @Override public StatsLogger getStatsLogger(String scope) { return this.cachingStatsProvider.getStatsLogger(scope); } @Override - public void writeAllMetrics(Writer writer) throws IOException { - PrometheusTextFormatUtil.writeMetricsCollectedByPrometheusClient(writer, registry); - + public void generate(SimpleTextOutputStream writer) { gauges.forEach((name, gauge) -> PrometheusTextFormatUtil.writeGauge(writer, name, gauge)); counters.forEach((name, counter) -> PrometheusTextFormatUtil.writeCounter(writer, name, counter)); opStats.forEach((name, opStatLogger) -> PrometheusTextFormatUtil.writeOpStat(writer, name, opStatLogger)); } - @Override public String getStatsName(String... statsComponents) { return cachingStatsProvider.getStatsName(statsComponents); } @@ -227,24 +155,5 @@ private void registerMetrics(Collector collector) { } } - - private static final Logger log = LoggerFactory.getLogger(org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider.class); - - /* - * Try to get Netty counter of used direct memory. This will be correct, unlike the JVM values. - */ - private static final AtomicLong directMemoryUsage; - static { - AtomicLong tmpDirectMemoryUsage = null; - - try { - Field field = PlatformDependent.class.getDeclaredField("DIRECT_MEMORY_COUNTER"); - field.setAccessible(true); - tmpDirectMemoryUsage = (AtomicLong) field.get(null); - } catch (Throwable t) { - log.warn("Failed to access netty DIRECT_MEMORY_COUNTER field {}", t.getMessage()); - } - - directMemoryUsage = tmpDirectMemoryUsage; - } + private static final Logger log = LoggerFactory.getLogger(PrometheusMetricsProvider.class); } diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusServlet.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusServlet.java deleted file mode 100644 index 21624ffe39..0000000000 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusServlet.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.streamnative.pulsar.handlers.kop.stats; - -import io.prometheus.client.exporter.common.TextFormat; -import java.io.IOException; -import java.io.Writer; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -/** - * Servlet used to export metrics in prometheus text format. - */ -public class PrometheusServlet extends HttpServlet { - private static final long serialVersionUID = 1L; - - private final transient PrometheusMetricsProvider provider; - - public PrometheusServlet(PrometheusMetricsProvider provider) { - this.provider = provider; - } - - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - resp.setStatus(HttpServletResponse.SC_OK); - resp.setContentType(TextFormat.CONTENT_TYPE_004); - - Writer writer = resp.getWriter(); - try { - provider.writeAllMetrics(writer); - writer.flush(); - } finally { - writer.close(); - } - } - - @Override - protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - doGet(req, resp); - } - -} diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java index e6f3d89564..199c2aa512 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java @@ -17,40 +17,32 @@ import io.prometheus.client.Collector.MetricFamilySamples; import io.prometheus.client.Collector.MetricFamilySamples.Sample; import io.prometheus.client.CollectorRegistry; -import java.io.IOException; -import java.io.Writer; import java.util.Enumeration; import org.apache.bookkeeper.stats.Counter; +import org.apache.pulsar.common.util.SimpleTextOutputStream; /** * Logic to write metrics in Prometheus text format. */ public class PrometheusTextFormatUtil { - static void writeGauge(Writer w, String name, SimpleGauge gauge) { + static void writeGauge(SimpleTextOutputStream w, String name, SimpleGauge gauge) { // Example: // # TYPE bookie_storage_entries_count gauge // bookie_storage_entries_count 519 - try { - w.append("# TYPE ").append(name).append(" gauge\n"); - w.append(name).append(' ').append(gauge.getSample().toString()).append('\n'); - } catch (IOException e) { - throw new RuntimeException(e); - } + w.write("# TYPE ").write(name).write(" gauge\n"); + w.write(name).write(' ').write(gauge.getSample().toString()).write('\n'); + } - static void writeCounter(Writer w, String name, Counter counter) { + static void writeCounter(SimpleTextOutputStream w, String name, Counter counter) { // Example: // # TYPE jvm_threads_started_total counter // jvm_threads_started_total 59 - try { - w.append("# TYPE ").append(name).append(" counter\n"); - w.append(name).append(' ').append(counter.get().toString()).append('\n'); - } catch (IOException e) { - throw new RuntimeException(e); - } + w.write("# TYPE ").write(name).write(" counter\n"); + w.write(name).write(' ').write(counter.get().toString()).write('\n'); } - static void writeOpStat(Writer w, String name, DataSketchesOpStatsLogger opStat) { + static void writeOpStat(SimpleTextOutputStream w, String name, DataSketchesOpStatsLogger opStat) { // Example: // # TYPE bookie_journal_JOURNAL_ADD_ENTRY summary // bookie_journal_JOURNAL_ADD_ENTRY{success="false",quantile="0.5",} NaN @@ -71,53 +63,49 @@ static void writeOpStat(Writer w, String name, DataSketchesOpStatsLogger opStat) // bookie_journal_JOURNAL_ADD_ENTRY{success="true",quantile="1.0",} 10.902 // bookie_journal_JOURNAL_ADD_ENTRY_count{success="true",} 658.0 // bookie_journal_JOURNAL_ADD_ENTRY_sum{success="true",} 1265.0800000000002 - try { - w.append("# TYPE ").append(name).append(" summary\n"); - writeQuantile(w, opStat, name, false, 0.5); - writeQuantile(w, opStat, name, false, 0.75); - writeQuantile(w, opStat, name, false, 0.95); - writeQuantile(w, opStat, name, false, 0.99); - writeQuantile(w, opStat, name, false, 0.999); - writeQuantile(w, opStat, name, false, 0.9999); - writeQuantile(w, opStat, name, false, 1.0); - writeCount(w, opStat, name, false); - writeSum(w, opStat, name, false); + w.write("# TYPE ").write(name).write(" summary\n"); + writeQuantile(w, opStat, name, false, 0.5); + writeQuantile(w, opStat, name, false, 0.75); + writeQuantile(w, opStat, name, false, 0.95); + writeQuantile(w, opStat, name, false, 0.99); + writeQuantile(w, opStat, name, false, 0.999); + writeQuantile(w, opStat, name, false, 0.9999); + writeQuantile(w, opStat, name, false, 1.0); + writeCount(w, opStat, name, false); + writeSum(w, opStat, name, false); - writeQuantile(w, opStat, name, true, 0.5); - writeQuantile(w, opStat, name, true, 0.75); - writeQuantile(w, opStat, name, true, 0.95); - writeQuantile(w, opStat, name, true, 0.99); - writeQuantile(w, opStat, name, true, 0.999); - writeQuantile(w, opStat, name, true, 0.9999); - writeQuantile(w, opStat, name, true, 1.0); - writeCount(w, opStat, name, true); - writeSum(w, opStat, name, true); + writeQuantile(w, opStat, name, true, 0.5); + writeQuantile(w, opStat, name, true, 0.75); + writeQuantile(w, opStat, name, true, 0.95); + writeQuantile(w, opStat, name, true, 0.99); + writeQuantile(w, opStat, name, true, 0.999); + writeQuantile(w, opStat, name, true, 0.9999); + writeQuantile(w, opStat, name, true, 1.0); + writeCount(w, opStat, name, true); + writeSum(w, opStat, name, true); - } catch (IOException e) { - throw new RuntimeException(e); - } } - private static void writeQuantile(Writer w, DataSketchesOpStatsLogger opStat, String name, Boolean success, - double quantile) throws IOException { - w.append(name).append("{success=\"").append(success.toString()).append("\",quantile=\"") - .append(Double.toString(quantile)).append("\"} ") - .append(Double.toString(opStat.getQuantileValue(success, quantile))).append('\n'); + private static void writeQuantile(SimpleTextOutputStream w, DataSketchesOpStatsLogger opStat, String name, + Boolean success, double quantile) { + w.write(name).write("{success=\"").write(success.toString()).write("\",quantile=\"") + .write(Double.toString(quantile)).write("\"} ") + .write(Double.toString(opStat.getQuantileValue(success, quantile))).write('\n'); } - private static void writeCount(Writer w, DataSketchesOpStatsLogger opStat, String name, Boolean success) - throws IOException { - w.append(name).append("_count{success=\"").append(success.toString()).append("\"} ") - .append(Long.toString(opStat.getCount(success))).append('\n'); + private static void writeCount(SimpleTextOutputStream w, DataSketchesOpStatsLogger opStat, String name, + Boolean success) { + w.write(name).write("_count{success=\"").write(success.toString()).write("\"} ") + .write(Long.toString(opStat.getCount(success))).write('\n'); } - private static void writeSum(Writer w, DataSketchesOpStatsLogger opStat, String name, Boolean success) - throws IOException { - w.append(name).append("_sum{success=\"").append(success.toString()).append("\"} ") - .append(Double.toString(opStat.getSum(success))).append('\n'); + private static void writeSum(SimpleTextOutputStream w, DataSketchesOpStatsLogger opStat, String name, + Boolean success) { + w.write(name).write("_sum{success=\"").write(success.toString()).write("\"} ") + .write(Double.toString(opStat.getSum(success))).write('\n'); } - static void writeMetricsCollectedByPrometheusClient(Writer w, CollectorRegistry registry) throws IOException { + static void writeMetricsCollectedByPrometheusClient(SimpleTextOutputStream w, CollectorRegistry registry) { Enumeration metricFamilySamples = registry.metricFamilySamples(); while (metricFamilySamples.hasMoreElements()) { MetricFamilySamples metricFamily = metricFamilySamples.nextElement(); diff --git a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java new file mode 100644 index 0000000000..21a4662aa2 --- /dev/null +++ b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java @@ -0,0 +1,110 @@ +package io.streamnative.pulsar.handlers.kop; + +import com.google.common.collect.Sets; +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +public class MetricsProviderTest extends KopProtocolHandlerTestBase{ + + @BeforeMethod + @Override + protected void setup() throws Exception { + super.internalSetup(); + log.info("success internal setup"); + + if (!admin.clusters().getClusters().contains(configClusterName)) { + // so that clients can test short names + admin.clusters().createCluster(configClusterName, + new ClusterData("http://127.0.0.1:" + brokerWebservicePort)); + } else { + admin.clusters().updateCluster(configClusterName, + new ClusterData("http://127.0.0.1:" + brokerWebservicePort)); + } + + if (!admin.tenants().getTenants().contains("public")) { + admin.tenants().createTenant("public", + new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); + } else { + admin.tenants().updateTenant("public", + new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test"))); + } + if (!admin.namespaces().getNamespaces("public").contains("public/default")) { + admin.namespaces().createNamespace("public/default"); + admin.namespaces().setNamespaceReplicationClusters("public/default", Sets.newHashSet("test")); + admin.namespaces().setRetention("public/default", + new RetentionPolicies(60, 1000)); + } + if (!admin.namespaces().getNamespaces("public").contains("public/__kafka")) { + admin.namespaces().createNamespace("public/__kafka"); + admin.namespaces().setNamespaceReplicationClusters("public/__kafka", Sets.newHashSet("test")); + admin.namespaces().setRetention("public/__kafka", + new RetentionPolicies(-1, -1)); + } + } + + @AfterMethod + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + + @Test(timeOut = 30000) + public void testMetricsProvider() throws Exception { + int partitionNumber = 3; + String kafkaTopicName = "kopKafkaProducePulsarMetrics" + partitionNumber; + + // create partitioned topic. + admin.topics().createPartitionedTopic(kafkaTopicName, partitionNumber); + + // 1. produce message with Kafka producer. + KProducer kProducer = new KProducer(kafkaTopicName, false, getKafkaBrokerPort()); + + int totalMsgs = 10; + + String messageStrPrefix = "Message_Kop_KafkaProducePulsarConsume_" + partitionNumber + "_"; + + for (int i = 0; i < totalMsgs; i++) { + String messageStr = messageStrPrefix + i; + ProducerRecord record = new ProducerRecord<>( + kafkaTopicName, + i, + messageStr); + + kProducer.getProducer().send(record).get(); + + if (log.isDebugEnabled()) { + log.debug("Kafka Producer Sent message: ({}, {})", i, messageStr); + } + } + + + HttpClient httpClient = HttpClientBuilder.create().build(); + final String metricsEndPoint = pulsar.getWebServiceAddress() + "/metrics"; + HttpResponse response = httpClient.execute(new HttpGet(metricsEndPoint)); + InputStream inputStream = response.getEntity().getContent(); + InputStreamReader isReader = new InputStreamReader(inputStream); + BufferedReader reader = new BufferedReader(isReader); + StringBuffer sb = new StringBuffer(); + String str; + while((str = reader.readLine()) != null){ + sb.append(str); + } + Assert.assertTrue(sb.toString().contains("kop_server_HANDLE_PRODUCE_REQUEST")); + } +} From f86147f1c691cbc22454aed04a144709a45644aa Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Sun, 28 Mar 2021 13:04:40 +0800 Subject: [PATCH 05/11] format code --- .../pulsar/handlers/kop/KopServerStats.java | 6 ++- .../pulsar/handlers/kop/RequestStats.java | 13 +++-- .../handlers/kop/StatsProviderService.java | 50 ------------------- .../kop/stats/PrometheusMetricsProvider.java | 15 ------ .../handlers/kop/stats/package-info.java | 19 +++++++ .../handlers/kop/MetricsProviderTest.java | 13 +++++ 6 files changed, 44 insertions(+), 72 deletions(-) delete mode 100644 kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/StatsProviderService.java create mode 100644 kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/package-info.java diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java index 852ae336d6..4149b0f334 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java @@ -13,6 +13,9 @@ */ package io.streamnative.pulsar.handlers.kop; +/** + * Kop server stats for prometheus metrics. + */ public interface KopServerStats { String CATEGORY_SERVER = "server"; @@ -20,9 +23,8 @@ public interface KopServerStats { String SERVER_STATUS = "SERVER_STATUS"; - /** - * PRODUCE STATS + * PRODUCE STATS. */ String HANDLE_PRODUCE_REQUEST = "HANDLE_PRODUCE_REQUEST"; String PRODUCE_ENCODE = "PRODUCE_ENCODE"; diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/RequestStats.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/RequestStats.java index b12c9029be..e9b9ff0e4f 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/RequestStats.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/RequestStats.java @@ -13,11 +13,6 @@ */ package io.streamnative.pulsar.handlers.kop; -import lombok.Getter; -import org.apache.bookkeeper.stats.OpStatsLogger; -import org.apache.bookkeeper.stats.StatsLogger; -import org.apache.bookkeeper.stats.annotations.StatsDoc; - import static io.streamnative.pulsar.handlers.kop.KopServerStats.CATEGORY_SERVER; import static io.streamnative.pulsar.handlers.kop.KopServerStats.HANDLE_PRODUCE_REQUEST; import static io.streamnative.pulsar.handlers.kop.KopServerStats.MESSAGE_PUBLISH; @@ -25,6 +20,14 @@ import static io.streamnative.pulsar.handlers.kop.KopServerStats.PRODUCE_ENCODE; import static io.streamnative.pulsar.handlers.kop.KopServerStats.SERVER_SCOPE; +import lombok.Getter; +import org.apache.bookkeeper.stats.OpStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.bookkeeper.stats.annotations.StatsDoc; + +/** + * Kop request stats metric for prometheus metrics. + */ @StatsDoc( name = SERVER_SCOPE, category = CATEGORY_SERVER, diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/StatsProviderService.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/StatsProviderService.java deleted file mode 100644 index 3fedf5a287..0000000000 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/StatsProviderService.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.streamnative.pulsar.handlers.kop; - -import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.stats.StatsProvider; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.PropertiesConfiguration; - -@Slf4j -public class StatsProviderService { - private final StatsProvider statsProvider; - private final Configuration conf; - - public StatsProviderService(KafkaServiceConfiguration kafkaServiceConfiguration) throws Exception { - Class statsProviderClass = Class.forName(kafkaServiceConfiguration.getKopStatsProviderClass()); - this.statsProvider = (StatsProvider) statsProviderClass.newInstance(); - - conf = new PropertiesConfiguration(); - conf.addProperty("prometheusStatsHttpEnable", kafkaServiceConfiguration.isKopPrometheusStatsHttpEnable()); - conf.addProperty("prometheusStatsHttpPort", kafkaServiceConfiguration.getKopPrometheusPort()); - conf.addProperty("prometheusStatsLatencyRolloverSeconds", - kafkaServiceConfiguration.getKopPrometheusStatsLatencyRolloverSeconds()); - conf.addProperty("prometheusBasicStatsEnable", false); - } - - public StatsProvider getStatsProvider() { - return this.statsProvider; - } - - protected void start() { - statsProvider.start(conf); - } - - protected void stop() { - statsProvider.stop(); - } - -} diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java index 3e5addf98b..cbfc0e2b76 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java @@ -15,25 +15,13 @@ import com.google.common.annotations.VisibleForTesting; import io.netty.util.concurrent.DefaultThreadFactory; -import io.netty.util.internal.PlatformDependent; import io.prometheus.client.Collector; import io.prometheus.client.CollectorRegistry; -import io.prometheus.client.Gauge; -import io.prometheus.client.Gauge.Child; -import io.prometheus.client.hotspot.GarbageCollectorExports; -import io.prometheus.client.hotspot.MemoryPoolsExports; -import io.prometheus.client.hotspot.StandardExports; -import io.prometheus.client.hotspot.ThreadExports; -import java.io.IOException; -import java.io.Writer; -import java.lang.reflect.Field; -import java.net.InetSocketAddress; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; import org.apache.bookkeeper.stats.CachingStatsProvider; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.stats.StatsProvider; @@ -41,9 +29,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.pulsar.broker.stats.prometheus.PrometheusRawMetricsProvider; import org.apache.pulsar.common.util.SimpleTextOutputStream; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/package-info.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/package-info.java new file mode 100644 index 0000000000..c2eddc6adb --- /dev/null +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/package-info.java @@ -0,0 +1,19 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Timer related classes. + * + *

The classes under this package are ported from Kafka. + */ +package io.streamnative.pulsar.handlers.kop.stats; diff --git a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java index 21a4662aa2..eaa327a7a0 100644 --- a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java +++ b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java @@ -1,3 +1,16 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.streamnative.pulsar.handlers.kop; import com.google.common.collect.Sets; From 1ea71e465f0a7debac76097532ecd86362682307 Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Sun, 28 Mar 2021 13:36:42 +0800 Subject: [PATCH 06/11] format code --- .../pulsar/handlers/kop/KafkaProtocolHandler.java | 1 + .../handlers/kop/stats/PrometheusMetricsProvider.java | 11 +++++------ .../handlers/kop/stats/PrometheusTextFormatUtil.java | 8 ++++---- .../pulsar/handlers/kop/stats/SimpleGauge.java | 2 +- .../pulsar/handlers/kop/MetricsProviderTest.java | 5 ++++- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaProtocolHandler.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaProtocolHandler.java index a7409442ac..fe43cb8514 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaProtocolHandler.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KafkaProtocolHandler.java @@ -332,6 +332,7 @@ public void close() { groupCoordinator.shutdown(); } KafkaTopicManager.LOOKUP_CACHE.clear(); + statsProvider.stop(); } public void initGroupCoordinator(BrokerService service) throws Exception { diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java index cbfc0e2b76..5d1bde92e1 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java @@ -36,22 +36,23 @@ * A Prometheus based {@link StatsProvider} implementation. */ public class PrometheusMetricsProvider implements PrometheusRawMetricsProvider { + private static final Logger log = LoggerFactory.getLogger(PrometheusMetricsProvider.class); private ScheduledExecutorService executor; public static final String PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS = "prometheusStatsLatencyRolloverSeconds"; public static final int DEFAULT_PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS = 60; - final CollectorRegistry registry; + private final CollectorRegistry registry; private final CachingStatsProvider cachingStatsProvider; /* * These acts a registry of the metrics defined in this provider */ - final ConcurrentMap counters = new ConcurrentSkipListMap<>(); - final ConcurrentMap> gauges = new ConcurrentSkipListMap<>(); - final ConcurrentMap opStats = new ConcurrentSkipListMap<>(); + private final ConcurrentMap counters = new ConcurrentSkipListMap<>(); + private final ConcurrentMap> gauges = new ConcurrentSkipListMap<>(); + private final ConcurrentMap opStats = new ConcurrentSkipListMap<>(); public PrometheusMetricsProvider() { this(CollectorRegistry.defaultRegistry); @@ -139,6 +140,4 @@ private void registerMetrics(Collector collector) { } } } - - private static final Logger log = LoggerFactory.getLogger(PrometheusMetricsProvider.class); } diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java index 199c2aa512..2cc21dbd75 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusTextFormatUtil.java @@ -25,7 +25,7 @@ * Logic to write metrics in Prometheus text format. */ public class PrometheusTextFormatUtil { - static void writeGauge(SimpleTextOutputStream w, String name, SimpleGauge gauge) { + public static void writeGauge(SimpleTextOutputStream w, String name, SimpleGauge gauge) { // Example: // # TYPE bookie_storage_entries_count gauge // bookie_storage_entries_count 519 @@ -34,7 +34,7 @@ static void writeGauge(SimpleTextOutputStream w, String name, SimpleGauge metricFamilySamples = registry.metricFamilySamples(); while (metricFamilySamples.hasMoreElements()) { MetricFamilySamples metricFamily = metricFamilySamples.nextElement(); diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/SimpleGauge.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/SimpleGauge.java index 60b75702b8..3290d00ffd 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/SimpleGauge.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/SimpleGauge.java @@ -31,7 +31,7 @@ public SimpleGauge(final Gauge gauge) { this.gauge = gauge; } - Number getSample() { + public Number getSample() { return gauge.getSample(); } } diff --git a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java index eaa327a7a0..b9b32e6331 100644 --- a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java +++ b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java @@ -31,6 +31,9 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +/** + * test for kop prometheus metrics. + */ @Slf4j public class MetricsProviderTest extends KopProtocolHandlerTestBase{ @@ -115,7 +118,7 @@ public void testMetricsProvider() throws Exception { BufferedReader reader = new BufferedReader(isReader); StringBuffer sb = new StringBuffer(); String str; - while((str = reader.readLine()) != null){ + while ((str = reader.readLine()) != null) { sb.append(str); } Assert.assertTrue(sb.toString().contains("kop_server_HANDLE_PRODUCE_REQUEST")); From 31622d5c653e2f1f2be59b52e41ce8ab26ab5691 Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Sun, 28 Mar 2021 13:45:36 +0800 Subject: [PATCH 07/11] format code --- .../handlers/kop/stats/PrometheusMetricsProvider.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java index 5d1bde92e1..941396b892 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java @@ -50,9 +50,9 @@ public class PrometheusMetricsProvider implements PrometheusRawMetricsProvider { /* * These acts a registry of the metrics defined in this provider */ - private final ConcurrentMap counters = new ConcurrentSkipListMap<>(); - private final ConcurrentMap> gauges = new ConcurrentSkipListMap<>(); - private final ConcurrentMap opStats = new ConcurrentSkipListMap<>(); + public final ConcurrentMap counters = new ConcurrentSkipListMap<>(); + public final ConcurrentMap> gauges = new ConcurrentSkipListMap<>(); + public final ConcurrentMap opStats = new ConcurrentSkipListMap<>(); public PrometheusMetricsProvider() { this(CollectorRegistry.defaultRegistry); From 531c4203a3d8cee908374454fac294b9afdc37ad Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Sun, 28 Mar 2021 13:49:00 +0800 Subject: [PATCH 08/11] format code --- .../handlers/kop/stats/PrometheusMetricsProvider.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java index 941396b892..003a9aa187 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java @@ -129,15 +129,4 @@ void rotateLatencyCollection() { metric.rotateLatencyCollection(); }); } - - private void registerMetrics(Collector collector) { - try { - collector.register(registry); - } catch (Exception e) { - // Ignore if these were already registered - if (log.isDebugEnabled()) { - log.debug("Failed to register Prometheus collector exports", e); - } - } - } } From 7f3fc24001d6d4e6ce72f18d47301172edc7c5ab Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Sun, 28 Mar 2021 14:00:53 +0800 Subject: [PATCH 09/11] format code --- .../kop/stats/PrometheusMetricsProvider.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java index 003a9aa187..492da3a52f 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java @@ -29,22 +29,16 @@ import org.apache.commons.lang.StringUtils; import org.apache.pulsar.broker.stats.prometheus.PrometheusRawMetricsProvider; import org.apache.pulsar.common.util.SimpleTextOutputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * A Prometheus based {@link StatsProvider} implementation. */ public class PrometheusMetricsProvider implements PrometheusRawMetricsProvider { - private static final Logger log = LoggerFactory.getLogger(PrometheusMetricsProvider.class); - private ScheduledExecutorService executor; public static final String PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS = "prometheusStatsLatencyRolloverSeconds"; public static final int DEFAULT_PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS = 60; - private final CollectorRegistry registry; - private final CachingStatsProvider cachingStatsProvider; /* @@ -54,12 +48,8 @@ public class PrometheusMetricsProvider implements PrometheusRawMetricsProvider { public final ConcurrentMap> gauges = new ConcurrentSkipListMap<>(); public final ConcurrentMap opStats = new ConcurrentSkipListMap<>(); - public PrometheusMetricsProvider() { - this(CollectorRegistry.defaultRegistry); - } - public PrometheusMetricsProvider(CollectorRegistry registry) { - this.registry = registry; + public PrometheusMetricsProvider() { this.cachingStatsProvider = new CachingStatsProvider(new StatsProvider() { @Override public void start(Configuration conf) { From 02ee742d5bcb98c118f5b4e7d1b5d094528d1da2 Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Sun, 28 Mar 2021 14:19:55 +0800 Subject: [PATCH 10/11] format code --- .../pulsar/handlers/kop/stats/PrometheusMetricsProvider.java | 1 - 1 file changed, 1 deletion(-) diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java index 492da3a52f..eb42bbe728 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/stats/PrometheusMetricsProvider.java @@ -16,7 +16,6 @@ import com.google.common.annotations.VisibleForTesting; import io.netty.util.concurrent.DefaultThreadFactory; import io.prometheus.client.Collector; -import io.prometheus.client.CollectorRegistry; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.Executors; From fd72cc5c5f4c8b9bc4534570fbc792af814dde03 Mon Sep 17 00:00:00 2001 From: hangc0276 Date: Wed, 31 Mar 2021 22:57:55 +0800 Subject: [PATCH 11/11] update test --- kafka-impl/conf/kop.conf | 895 ------------------ .../pulsar/handlers/kop/KopServerStats.java | 2 - .../handlers/kop/MetricsProviderTest.java | 3 + 3 files changed, 3 insertions(+), 897 deletions(-) delete mode 100755 kafka-impl/conf/kop.conf diff --git a/kafka-impl/conf/kop.conf b/kafka-impl/conf/kop.conf deleted file mode 100755 index 2f8f4c5769..0000000000 --- a/kafka-impl/conf/kop.conf +++ /dev/null @@ -1,895 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -### --- Kafka broker settings --- ### - -# The messaging Protocols that avilabale when loaded by Pulsar Broker. -messagingProtocols=kafka - -# ListenersProp for Kafka service(host should follow the advertisedAddress). -# e.g. PLAINTEXT://localhost:9092,SSL://localhost:9093 -# when KoP runs as a plugin in Pulsar, if it is not set, kop will use PLAINTEXT://advertisedAddress:9092 -listeners=PLAINTEXT://127.0.0.1:9092 - -# Kafka on Pulsar Broker tenant -kafkaTenant=public - -# Kafka on Pulsar Broker namespace -kafkaNamespace=default - -# The tenant used for storing Kafka metadata topics -kafkaMetadataTenant=public - -# The namespace used for storing Kafka metadata topics -kafkaMetadataNamespace=__kafka - -# Flag to enable group coordinator -enableGroupCoordinator=true - -# The minimum allowed session timeout for registered consumers. -# Shorter timeouts result in quicker failure detection at the cost -# of more frequent consumer heartbeating, which can overwhelm broker resources. -groupMinSessionTimeoutMs=6000 - -# The maximum allowed session timeout for registered consumers. -# Longer timeouts give consumers more time to process messages in -# between heartbeats at the cost of a longer time to detect failures. -groupMaxSessionTimeoutMs=300000 - -# The amount of time the group coordinator will wait for more consumers -# to join a new group before performing the first rebalance. A longer -# delay means potentially fewer rebalances, but increases the time until -# processing begins -groupInitialRebalanceDelayMs=3000 - -# Compression codec for the offsets topic - compression may be used to achieve "atomic" commits -offsetsTopicCompressionCodec=NONE - -# The maximum size in Bytes for a metadata entry associated with an offset commit -offsetMetadataMaxSize=4096 - -# Offsets older than this retention period will be discarded, default 7 days -offsetsRetentionMinutes=10080 - -# Frequency at which to check for stale offsets -offsetsRetentionCheckIntervalMs=600000 - -# Number of partitions for the offsets topic -offsetsTopicNumPartitions=8 - -# Maximum number of entries that are read from cursor once per time -maxReadEntriesNum=5 - -# The format of an entry. The default value is pulsar. -# Optional values: [pulsar, kafka] -# -# pulsar: -# When KoP receives messages from kafka producer, it will serialize these messages to -# the format so that pulsar consumer can read directly. -# When KoP sends entries to kafka consumer, it will treat each entry as pulsar's -# format and deserialize each entry to kafka's format. -# -# kafka: -# When KoP receives messages from kafka producer, add a header which is PulsarApi.Metadata -# before the messages' bytes, and then write to BK directly. -# When KoP sends entries to kafka consumer, it will treat each entry as kafka's format and -# just discard the pulsar header and send left bytes to Kafka consumer. -# This mode means that current pulsar clients cannot interact with kafka clients, but -# kafka producer works well with kafka consumer. -entryFormat=pulsar - -# Zookeeper path for storing kop consumer group -groupIdZooKeeperPath=/client_group_id - -### --- KoP SSL configs--- ### - -# Kafka ssl configuration map with: SSL_PROTOCOL_CONFIG = ssl.protocol -kopSslProtocol=TLS - -# Kafka ssl configuration map with: SSL_PROVIDER_CONFIG = ssl.provider -kopSslProvider= - -# Kafka ssl configuration map with: SSL_CIPHER_SUITES_CONFIG = ssl.cipher.suites -kopSslCipherSuites= - -# Kafka ssl configuration map with: SSL_ENABLED_PROTOCOLS_CONFIG = ssl.enabled.protocols -kopSslEnabledProtocols=TLSv1.2,TLSv1.1,TLSv1 - -# Kafka ssl configuration map with: SSL_KEYSTORE_TYPE_CONFIG = ssl.keystore.type -kopSslKeystoreType=JKS - -# Kafka ssl configuration map with: SSL_KEYSTORE_LOCATION_CONFIG = ssl.keystore.location -kopSslKeystoreLocation= - -# Kafka ssl configuration map with: SSL_KEYSTORE_PASSWORD_CONFIG = ssl.keystore.password -kopSslKeystorePassword= - -# Kafka ssl configuration map with: SSL_KEY_PASSWORD_CONFIG = ssl.key.password -kopSslKeyPassword= - -# Kafka ssl configuration map with: SSL_TRUSTSTORE_TYPE_CONFIG = ssl.truststore.type -kopSslTruststoreType=JKS - -# Kafka ssl configuration map with: SSL_TRUSTSTORE_LOCATION_CONFIG = ssl.truststore.location -kopSslTruststoreLocation= - -# Kafka ssl configuration map with: SSL_TRUSTSTORE_PASSWORD_CONFIG = ssl.truststore.password -kopSslTruststorePassword= - -# Kafka ssl configuration map with: SSL_KEYMANAGER_ALGORITHM_CONFIG = ssl.keymanager.algorithm -kopSslKeymanagerAlgorithm=SunX509 - -# Kafka ssl configuration map with: SSL_TRUSTMANAGER_ALGORITHM_CONFIG = ssl.trustmanager.algorithm -kopSslTrustmanagerAlgorithm=SunX509 - -# Kafka ssl configuration map with: -# SSL_SECURE_RANDOM_IMPLEMENTATION_CONFIG = ssl.secure.random.implementation -kopSslSecureRandomImplementation= - -# supported SASL mechanisms exposed by broker -saslAllowedMechanisms= - -### --- Changed for KoP --- ### - -# Enable the deletion of inactive topics -brokerDeleteInactiveTopicsEnabled=false - -allowAutoTopicCreation=true - -allowAutoTopicCreationType=partitioned - -# Name of the cluster to which this broker belongs to -clusterName=kafka-cluster - -### --- General broker settings --- ### - -# Max Rate(in 1 seconds) of Message allowed to publish for a broker if broker publish rate limiting enabled -# (Disable message rate limit with value 0) -brokerPublisherThrottlingMaxMessageRate=0 - -# Max Rate(in 1 seconds) of Byte allowed to publish for a broker if broker publish rate limiting enabled. -# (Disable byte rate limit with value 0) -brokerPublisherThrottlingMaxByteRate=0 - -# Zookeeper quorum connection string -zookeeperServers=localhost:2181 - -# Configuration Store connection string -configurationStoreServers= - -# Broker data port -brokerServicePort=6650 - -# Broker data port for TLS - By default TLS is disabled -brokerServicePortTls= - -# Port to use to server HTTP request -webServicePort=8080 - -# Port to use to server HTTPS request - By default TLS is disabled -webServicePortTls= - -# Hostname or IP address the service binds on, default is 0.0.0.0. -bindAddress=0.0.0.0 - -# Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used. -advertisedAddress= - -# Number of threads to use for Netty IO. Default is set to 2 * Runtime.getRuntime().availableProcessors() -numIOThreads= - -# Number of threads to use for HTTP requests processing. Default is set to 2 * Runtime.getRuntime().availableProcessors() -numHttpServerThreads= - -# Flag to control features that are meant to be used when running in standalone mode -isRunningStandalone= - -# Enable cluster's failure-domain which can distribute brokers into logical region -failureDomainsEnabled=false - -# Zookeeper session timeout in milliseconds -zooKeeperSessionTimeoutMillis=30000 - -# ZooKeeper operation timeout in seconds -zooKeeperOperationTimeoutSeconds=30 - -# Time to wait for broker graceful shutdown. After this time elapses, the process will be killed -brokerShutdownTimeoutMs=60000 - -# Enable backlog quota check. Enforces action on topic when the quota is reached -backlogQuotaCheckEnabled=true - -# How often to check for topics that have reached the quota -backlogQuotaCheckIntervalInSeconds=60 - -# Default per-topic backlog quota limit, less than 0 means no limitation. default is -1. -backlogQuotaDefaultLimitGB=-1 - -# Default backlog quota retention policy. Default is producer_request_hold -# 'producer_request_hold' Policy which holds producer's send request until the resource becomes available (or holding times out) -# 'producer_exception' Policy which throws javax.jms.ResourceAllocationException to the producer -# 'consumer_backlog_eviction' Policy which evicts the oldest message from the slowest consumer's backlog -backlogQuotaDefaultRetentionPolicy=producer_request_hold - -# Default ttl for namespaces if ttl is not already configured at namespace policies. (disable default-ttl with value 0) -ttlDurationDefaultInSeconds=0 - -# Enable the deletion of inactive topics -brokerDeleteInactiveTopicsEnabled=true - -# How often to check for inactive topics -brokerDeleteInactiveTopicsFrequencySeconds=60 - -# How frequently to proactively check and purge expired messages -messageExpiryCheckIntervalInMinutes=5 - -# How long to delay rewinding cursor and dispatching messages when active consumer is changed -activeConsumerFailoverDelayTimeMillis=1000 - -# How long to delete inactive subscriptions from last consuming -# When it is 0, inactive subscriptions are not deleted automatically -subscriptionExpirationTimeMinutes=0 - -# Enable subscription message redelivery tracker to send redelivery count to consumer (default is enabled) -subscriptionRedeliveryTrackerEnabled=true - -# How frequently to proactively check and purge expired subscription -subscriptionExpiryCheckIntervalInMinutes=5 - -# Enable Key_Shared subscription (default is enabled) -subscriptionKeySharedEnable=true - -# Set the default behavior for message deduplication in the broker -# This can be overridden per-namespace. If enabled, broker will reject -# messages that were already stored in the topic -brokerDeduplicationEnabled=false - -# Maximum number of producer information that it's going to be -# persisted for deduplication purposes -brokerDeduplicationMaxNumberOfProducers=10000 - -# Number of entries after which a dedup info snapshot is taken. -# A larger interval will lead to fewer snapshots being taken, though it would -# increase the topic recovery time when the entries published after the -# snapshot need to be replayed. -brokerDeduplicationEntriesInterval=1000 - -# Time of inactivity after which the broker will discard the deduplication information -# relative to a disconnected producer. Default is 6 hours. -brokerDeduplicationProducerInactivityTimeoutMinutes=360 - -# When a namespace is created without specifying the number of bundle, this -# value will be used as the default -defaultNumberOfNamespaceBundles=4 - -# Enable check for minimum allowed client library version -clientLibraryVersionCheckEnabled=false - -# Path for the file used to determine the rotation status for the broker when responding -# to service discovery health checks -statusFilePath= - -# If true, (and ModularLoadManagerImpl is being used), the load manager will attempt to -# use only brokers running the latest software version (to minimize impact to bundles) -preferLaterVersions=false - -# Max number of unacknowledged messages allowed to receive messages by a consumer on a shared subscription. Broker will stop sending -# messages to consumer once, this limit reaches until consumer starts acknowledging messages back. -# Using a value of 0, is disabling unackeMessage limit check and consumer can receive messages without any restriction -maxUnackedMessagesPerConsumer=50000 - -# Max number of unacknowledged messages allowed per shared subscription. Broker will stop dispatching messages to -# all consumers of the subscription once this limit reaches until consumer starts acknowledging messages back and -# unack count reaches to limit/2. Using a value of 0, is disabling unackedMessage-limit -# check and dispatcher can dispatch messages without any restriction -maxUnackedMessagesPerSubscription=200000 - -# Max number of unacknowledged messages allowed per broker. Once this limit reaches, broker will stop dispatching -# messages to all shared subscription which has higher number of unack messages until subscriptions start -# acknowledging messages back and unack count reaches to limit/2. Using a value of 0, is disabling -# unackedMessage-limit check and broker doesn't block dispatchers -maxUnackedMessagesPerBroker=0 - -# Once broker reaches maxUnackedMessagesPerBroker limit, it blocks subscriptions which has higher unacked messages -# than this percentage limit and subscription will not receive any new messages until that subscription acks back -# limit/2 messages -maxUnackedMessagesPerSubscriptionOnBrokerBlocked=0.16 - -# Too many subscribe requests from a consumer can cause broker rewinding consumer cursors and loading data from bookies, -# hence causing high network bandwidth usage -# When the positive value is set, broker will throttle the subscribe requests for one consumer. -# Otherwise, the throttling will be disabled. The default value of this setting is 0 - throttling is disabled. -subscribeThrottlingRatePerConsumer=0 - -# Rate period for {subscribeThrottlingRatePerConsumer}. Default is 30s. -subscribeRatePeriodPerConsumerInSecond=30 - -# Default messages per second dispatch throttling-limit for every topic. Using a value of 0, is disabling default -# message dispatch-throttling -dispatchThrottlingRatePerTopicInMsg=0 - -# Default bytes per second dispatch throttling-limit for every topic. Using a value of 0, is disabling -# default message-byte dispatch-throttling -dispatchThrottlingRatePerTopicInByte=0 - -# Default number of message dispatching throttling-limit for a subscription. -# Using a value of 0, is disabling default message dispatch-throttling. -dispatchThrottlingRatePerSubscriptionInMsg=0 - -# Default number of message-bytes dispatching throttling-limit for a subscription. -# Using a value of 0, is disabling default message-byte dispatch-throttling. -dispatchThrottlingRatePerSubscriptionInByte=0 - -# Default messages per second dispatch throttling-limit for every replicator in replication. -# Using a value of 0, is disabling replication message dispatch-throttling -dispatchThrottlingRatePerReplicatorInMsg=0 - -# Default bytes per second dispatch throttling-limit for every replicator in replication. -# Using a value of 0, is disabling replication message-byte dispatch-throttling -dispatchThrottlingRatePerReplicatorInByte=0 - -# By default we enable dispatch-throttling for both caught up consumers as well as consumers who have -# backlog. -dispatchThrottlingOnNonBacklogConsumerEnabled=true - -# Max number of entries to read from bookkeeper. By default it is 100 entries. -dispatcherMaxReadBatchSize=100 - -# Min number of entries to read from bookkeeper. By default it is 1 entries. -# When there is an error occurred on reading entries from bookkeeper, the broker -# will backoff the batch size to this minimum number." -dispatcherMinReadBatchSize=1 - -# Max number of entries to dispatch for a shared subscription. By default it is 20 entries. -dispatcherMaxRoundRobinBatchSize=20 - -# Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic -maxConcurrentLookupRequest=50000 - -# Max number of concurrent topic loading request broker allows to control number of zk-operations -maxConcurrentTopicLoadRequest=5000 - -# Max concurrent non-persistent message can be processed per connection -maxConcurrentNonPersistentMessagePerConnection=1000 - -# Number of worker threads to serve non-persistent topic -numWorkerThreadsForNonPersistentTopic=8 - -# Enable broker to load persistent topics -enablePersistentTopics=true - -# Enable broker to load non-persistent topics -enableNonPersistentTopics=true - -# Enable to run bookie along with broker -enableRunBookieTogether=false - -# Enable to run bookie autorecovery along with broker -enableRunBookieAutoRecoveryTogether=false - -# Max number of producers allowed to connect to topic. Once this limit reaches, Broker will reject new producers -# until the number of connected producers decrease. -# Using a value of 0, is disabling maxProducersPerTopic-limit check. -maxProducersPerTopic=0 - -# Max number of consumers allowed to connect to topic. Once this limit reaches, Broker will reject new consumers -# until the number of connected consumers decrease. -# Using a value of 0, is disabling maxConsumersPerTopic-limit check. -maxConsumersPerTopic=0 - -# Max number of consumers allowed to connect to subscription. Once this limit reaches, Broker will reject new consumers -# until the number of connected consumers decrease. -# Using a value of 0, is disabling maxConsumersPerSubscription-limit check. -maxConsumersPerSubscription=0 - -# Max size of messages. -maxMessageSize=5242880 - -# Interval between checks to see if topics with compaction policies need to be compacted -brokerServiceCompactionMonitorIntervalInSeconds=60 - -# Whether to enable the delayed delivery for messages. -# If disabled, messages will be immediately delivered and there will -# be no tracking overhead. -delayedDeliveryEnabled=true - -# Control the tick time for when retrying on delayed delivery, -# affecting the accuracy of the delivery time compared to the scheduled time. -# Default is 1 second. -delayedDeliveryTickTimeMillis=1000 - -# Enable tracking of replicated subscriptions state across clusters. -enableReplicatedSubscriptions=true - -# Frequency of snapshots for replicated subscriptions tracking. -replicatedSubscriptionsSnapshotFrequencyMillis=1000 - -# Timeout for building a consistent snapshot for tracking replicated subscriptions state. -replicatedSubscriptionsSnapshotTimeoutSeconds=30 - -# Max number of snapshot to be cached per subscription. -replicatedSubscriptionsSnapshotMaxCachedPerSubscription=10 - -### --- Authentication --- ### -# Role names that are treated as "proxy roles". If the broker sees a request with -#role as proxyRoles - it will demand to see a valid original principal. -proxyRoles= - -# If this flag is set then the broker authenticates the original Auth data -# else it just accepts the originalPrincipal and authorizes it (if required). -authenticateOriginalAuthData=false - -# Deprecated - Use webServicePortTls and brokerServicePortTls instead -tlsEnabled=false - -# Tls cert refresh duration in seconds (set 0 to check on every new connection) -tlsCertRefreshCheckDurationSec=300 - -# Path for the TLS certificate file -tlsCertificateFilePath= - -# Path for the TLS private key file -tlsKeyFilePath= - -# Path for the trusted TLS certificate file. -# This cert is used to verify that any certs presented by connecting clients -# are signed by a certificate authority. If this verification -# fails, then the certs are untrusted and the connections are dropped. -tlsTrustCertsFilePath= - -# Accept untrusted TLS certificate from client. -# If true, a client with a cert which cannot be verified with the -# 'tlsTrustCertsFilePath' cert will allowed to connect to the server, -# though the cert will not be used for client authentication. -tlsAllowInsecureConnection=false - -# Specify the tls protocols the broker will use to negotiate during TLS handshake -# (a comma-separated list of protocol names). -# Examples:- [TLSv1.2, TLSv1.1, TLSv1] -tlsProtocols= - -# Specify the tls cipher the broker will use to negotiate during TLS Handshake -# (a comma-separated list of ciphers). -# Examples:- [TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256] -tlsCiphers= - -# Trusted client certificates are required for to connect TLS -# Reject the Connection if the Client Certificate is not trusted. -# In effect, this requires that all connecting clients perform TLS client -# authentication. -tlsRequireTrustedClientCertOnConnect=false - -### --- Authentication --- ### - -# Enable authentication -authenticationEnabled=false - -# Autentication provider name list, which is comma separated list of class names -authenticationProviders= - -# Enforce authorization -authorizationEnabled=false - -# Authorization provider fully qualified class-name -authorizationProvider=org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider - -# Allow wildcard matching in authorization -# (wildcard matching only applicable if wildcard-char: -# * presents at first or last position eg: *.pulsar.service, pulsar.service.*) -authorizationAllowWildcardsMatching=false - -# Role names that are treated as "super-user", meaning they will be able to do all admin -# operations and publish/consume from all topics -superUserRoles= - -# Authentication settings of the broker itself. Used when the broker connects to other brokers, -# either in same or other clusters -brokerClientTlsEnabled=false -brokerClientAuthenticationPlugin= -brokerClientAuthenticationParameters= -brokerClientTrustCertsFilePath= - -# Supported Athenz provider domain names(comma separated) for authentication -athenzDomainNames= - -# When this parameter is not empty, unauthenticated users perform as anonymousUserRole -anonymousUserRole= - -### --- Token Authentication Provider --- ### - -## Symmetric key -# Configure the secret key to be used to validate auth tokens -# The key can be specified like: -# tokenSecretKey=data:base64,xxxxxxxxx -# tokenSecretKey=file:///my/secret.key -tokenSecretKey= - -## Asymmetric public/private key pair -# Configure the public key to be used to validate auth tokens -# The key can be specified like: -# tokenPublicKey=data:base64,xxxxxxxxx -# tokenPublicKey=file:///my/public.key -tokenPublicKey= - -# The token "claim" that will be interpreted as the authentication "role" or "principal" by AuthenticationProviderToken (defaults to "sub" if blank) -tokenAuthClaim= - -### --- SASL Authentication Provider --- ### - -# This is a regexp, which limits the range of possible ids which can connect to the Broker using SASL. -# Default value: `SaslConstants.JAAS_CLIENT_ALLOWED_IDS_DEFAULT`, which is ".*pulsar.*", -# so only clients whose id contains 'pulsar' are allowed to connect. -saslJaasClientAllowedIds= - -# Service Principal, for login context name. -# Default value `SaslConstants.JAAS_DEFAULT_BROKER_SECTION_NAME`, which is "Broker". -saslJaasBrokerSectionName= - -### --- BookKeeper Client --- ### - -# Authentication plugin to use when connecting to bookies -bookkeeperClientAuthenticationPlugin= - -# BookKeeper auth plugin implementatation specifics parameters name and values -bookkeeperClientAuthenticationParametersName= -bookkeeperClientAuthenticationParameters= - -# Timeout for BK add / read operations -bookkeeperClientTimeoutInSeconds=30 - -# Speculative reads are initiated if a read request doesn't complete within a certain time -# Using a value of 0, is disabling the speculative reads -bookkeeperClientSpeculativeReadTimeoutInMillis=0 - -# Use older Bookkeeper wire protocol with bookie -bookkeeperUseV2WireProtocol=true - -# Enable bookies health check. Bookies that have more than the configured number of failure within -# the interval will be quarantined for some time. During this period, new ledgers won't be created -# on these bookies -bookkeeperClientHealthCheckEnabled=true -bookkeeperClientHealthCheckIntervalSeconds=60 -bookkeeperClientHealthCheckErrorThresholdPerInterval=5 -bookkeeperClientHealthCheckQuarantineTimeInSeconds=1800 - -# Enable rack-aware bookie selection policy. BK will chose bookies from different racks when -# forming a new bookie ensemble -bookkeeperClientRackawarePolicyEnabled=true - -# Enable region-aware bookie selection policy. BK will chose bookies from -# different regions and racks when forming a new bookie ensemble -# If enabled, the value of bookkeeperClientRackawarePolicyEnabled is ignored -bookkeeperClientRegionawarePolicyEnabled=false - -# Enable/disable reordering read sequence on reading entries. -bookkeeperClientReorderReadSequenceEnabled=false - -# Enable bookie isolation by specifying a list of bookie groups to choose from. Any bookie -# outside the specified groups will not be used by the broker -bookkeeperClientIsolationGroups= - -# Enable bookie secondary-isolation group if bookkeeperClientIsolationGroups doesn't -# have enough bookie available. -bookkeeperClientSecondaryIsolationGroups= - -# Minimum bookies that should be available as part of bookkeeperClientIsolationGroups -# else broker will include bookkeeperClientSecondaryIsolationGroups bookies in isolated list. -bookkeeperClientMinAvailableBookiesInIsolationGroups= - -# Enable/disable having read operations for a ledger to be sticky to a single bookie. -# If this flag is enabled, the client will use one single bookie (by preference) to read -# all entries for a ledger. -# -# Disable Sticy Read until {@link https://github.com/apache/bookkeeper/issues/1970} is fixed -bookkeeperEnableStickyReads=false - -### --- Managed Ledger --- ### - -# Number of bookies to use when creating a ledger -managedLedgerDefaultEnsembleSize=1 - -# Number of copies to store for each message -managedLedgerDefaultWriteQuorum=1 - -# Number of guaranteed copies (acks to wait before write is complete) -managedLedgerDefaultAckQuorum=1 - -# Default type of checksum to use when writing to BookKeeper. Default is "CRC32C" -# Other possible options are "CRC32", "MAC" or "DUMMY" (no checksum). -managedLedgerDigestType=CRC32C - -# Number of threads to be used for managed ledger tasks dispatching -managedLedgerNumWorkerThreads=8 - -# Number of threads to be used for managed ledger scheduled tasks -managedLedgerNumSchedulerThreads=8 - -# Amount of memory to use for caching data payload in managed ledger. This memory -# is allocated from JVM direct memory and it's shared across all the topics -# running in the same broker. By default, uses 1/5th of available direct memory -managedLedgerCacheSizeMB= - -# Whether we should make a copy of the entry payloads when inserting in cache -managedLedgerCacheCopyEntries=false - -# Threshold to which bring down the cache level when eviction is triggered -managedLedgerCacheEvictionWatermark=0.9 - -# Configure the cache eviction frequency for the managed ledger cache (evictions/sec) -managedLedgerCacheEvictionFrequency=100.0 - -# All entries that have stayed in cache for more than the configured time, will be evicted -managedLedgerCacheEvictionTimeThresholdMillis=1000 - -# Configure the threshold (in number of entries) from where a cursor should be considered 'backlogged' -# and thus should be set as inactive. -managedLedgerCursorBackloggedThreshold=1000 - -# Rate limit the amount of writes per second generated by consumer acking the messages -managedLedgerDefaultMarkDeleteRateLimit=1.0 - -# Max number of entries to append to a ledger before triggering a rollover -# A ledger rollover is triggered on these conditions -# * Either the max rollover time has been reached -# * or max entries have been written to the ledged and at least min-time -# has passed -managedLedgerMaxEntriesPerLedger=50000 - -# Minimum time between ledger rollover for a topic -managedLedgerMinLedgerRolloverTimeMinutes=10 - -# Maximum time before forcing a ledger rollover for a topic -managedLedgerMaxLedgerRolloverTimeMinutes=240 - -# Delay between a ledger being successfully offloaded to long term storage -# and the ledger being deleted from bookkeeper (default is 4 hours) -managedLedgerOffloadDeletionLagMs=14400000 - -# Max number of entries to append to a cursor ledger -managedLedgerCursorMaxEntriesPerLedger=50000 - -# Max time before triggering a rollover on a cursor ledger -managedLedgerCursorRolloverTimeInSeconds=14400 - -# Max number of "acknowledgment holes" that are going to be persistently stored. -# When acknowledging out of order, a consumer will leave holes that are supposed -# to be quickly filled by acking all the messages. The information of which -# messages are acknowledged is persisted by compressing in "ranges" of messages -# that were acknowledged. After the max number of ranges is reached, the information -# will only be tracked in memory and messages will be redelivered in case of -# crashes. -managedLedgerMaxUnackedRangesToPersist=10000 - -# Max number of "acknowledgment holes" that can be stored in Zookeeper. If number of unack message range is higher -# than this limit then broker will persist unacked ranges into bookkeeper to avoid additional data overhead into -# zookeeper. -managedLedgerMaxUnackedRangesToPersistInZooKeeper=1000 - -# Skip reading non-recoverable/unreadable data-ledger under managed-ledger's list. It helps when data-ledgers gets -# corrupted at bookkeeper and managed-cursor is stuck at that ledger. -autoSkipNonRecoverableData=false - -# operation timeout while updating managed-ledger metadata. -managedLedgerMetadataOperationsTimeoutSeconds=60 - -# Read entries timeout when broker tries to read messages from bookkeeper. -managedLedgerReadEntryTimeoutSeconds=0 - -# Add entry timeout when broker tries to publish message to bookkeeper (0 to disable it). -managedLedgerAddEntryTimeoutSeconds=0 - -### --- Load balancer --- ### - -# Enable load balancer -loadBalancerEnabled=true - -# Percentage of change to trigger load report update -loadBalancerReportUpdateThresholdPercentage=10 - -# maximum interval to update load report -loadBalancerReportUpdateMaxIntervalMinutes=15 - -# Frequency of report to collect -loadBalancerHostUsageCheckIntervalMinutes=1 - -# Enable/disable automatic bundle unloading for load-shedding -loadBalancerSheddingEnabled=true - -# Load shedding interval. Broker periodically checks whether some traffic should be offload from -# some over-loaded broker to other under-loaded brokers -loadBalancerSheddingIntervalMinutes=1 - -# Prevent the same topics to be shed and moved to other broker more that once within this timeframe -loadBalancerSheddingGracePeriodMinutes=30 - -# Usage threshold to allocate max number of topics to broker -loadBalancerBrokerMaxTopics=50000 - -# Usage threshold to determine a broker as over-loaded -loadBalancerBrokerOverloadedThresholdPercentage=85 - -# Interval to flush dynamic resource quota to ZooKeeper -loadBalancerResourceQuotaUpdateIntervalMinutes=15 - -# enable/disable namespace bundle auto split -loadBalancerAutoBundleSplitEnabled=true - -# enable/disable automatic unloading of split bundles -loadBalancerAutoUnloadSplitBundlesEnabled=true - -# maximum topics in a bundle, otherwise bundle split will be triggered -loadBalancerNamespaceBundleMaxTopics=1000 - -# maximum sessions (producers + consumers) in a bundle, otherwise bundle split will be triggered -loadBalancerNamespaceBundleMaxSessions=1000 - -# maximum msgRate (in + out) in a bundle, otherwise bundle split will be triggered -loadBalancerNamespaceBundleMaxMsgRate=30000 - -# maximum bandwidth (in + out) in a bundle, otherwise bundle split will be triggered -loadBalancerNamespaceBundleMaxBandwidthMbytes=100 - -# maximum number of bundles in a namespace -loadBalancerNamespaceMaximumBundles=128 - -# Override the auto-detection of the network interfaces max speed. -# This option is useful in some environments (eg: EC2 VMs) where the max speed -# reported by Linux is not reflecting the real bandwidth available to the broker. -# Since the network usage is employed by the load manager to decide when a broker -# is overloaded, it is important to make sure the info is correct or override it -# with the right value here. The configured value can be a double (eg: 0.8) and that -# can be used to trigger load-shedding even before hitting on NIC limits. -loadBalancerOverrideBrokerNicSpeedGbps= - -# Name of load manager to use -loadManagerClassName=org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl - -### --- Replication --- ### - -# Enable replication metrics -replicationMetricsEnabled=true - -# Max number of connections to open for each broker in a remote cluster -# More connections host-to-host lead to better throughput over high-latency -# links. -replicationConnectionsPerBroker=16 - -# Replicator producer queue size -replicationProducerQueueSize=1000 - -# Replicator prefix used for replicator producer name and cursor name -replicatorPrefix=pulsar.repl - -# Default message retention time -defaultRetentionTimeInMinutes=600 - -# Default retention size -defaultRetentionSizeInMB=1000 - -# How often to check whether the connections are still alive -keepAliveIntervalSeconds=30 - -# How often broker checks for inactive topics to be deleted (topics with no subscriptions and no one connected) -brokerServicePurgeInactiveFrequencyInSeconds=60 - -# bootstrap namespaces -bootstrapNamespaces= - -### --- WebSocket --- ### - -# Enable the WebSocket API service in broker -webSocketServiceEnabled=false - -# Number of IO threads in Pulsar Client used in WebSocket proxy -webSocketNumIoThreads=8 - -# Number of connections per Broker in Pulsar Client used in WebSocket proxy -webSocketConnectionsPerBroker=8 - -# Time in milliseconds that idle WebSocket session times out -webSocketSessionIdleTimeoutMillis=300000 - -### --- Metrics --- ### - -# Enable topic level metrics -exposeTopicLevelMetricsInPrometheus=true - -# Enable consumer level metrics. default is false -exposeConsumerLevelMetricsInPrometheus=false - -# Classname of Pluggable JVM GC metrics logger that can log GC specific metrics -# jvmGCMetricsLoggerClassName= - -### --- Functions --- ### - -# Enable Functions Worker Service in Broker -functionsWorkerEnabled=false - -### --- Broker Web Stats --- ### - -# Enable topic level metrics -exposePublisherStats=true -statsUpdateFrequencyInSecs=60 -statsUpdateInitialDelayInSecs=60 - -### --- Schema storage --- ### -# The schema storage implementation used by this broker -schemaRegistryStorageClassName=org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorageFactory - -# Enforce schema validation on following cases: -# -# - if a producer without a schema attempts to produce to a topic with schema, the producer will be -# failed to connect. PLEASE be carefully on using this, since non-java clients don't support schema. -# if you enable this setting, it will cause non-java clients failed to produce. -isSchemaValidationEnforced=false - -### --- Ledger Offloading --- ### - -# The directory for all the offloader implementations -offloadersDirectory=./offloaders - -# Driver to use to offload old data to long term storage (Possible values: S3, aws-s3, google-cloud-storage) -# When using google-cloud-storage, Make sure both Google Cloud Storage and Google Cloud Storage JSON API are enabled for -# the project (check from Developers Console -> Api&auth -> APIs). -managedLedgerOffloadDriver= - -# Maximum number of thread pool threads for ledger offloading -managedLedgerOffloadMaxThreads=2 - -# Use Open Range-Set to cache unacked messages -managedLedgerUnackedRangesOpenCacheSetEnabled=true - -# For Amazon S3 ledger offload, AWS region -s3ManagedLedgerOffloadRegion= - -# For Amazon S3 ledger offload, Bucket to place offloaded ledger into -s3ManagedLedgerOffloadBucket= - -# For Amazon S3 ledger offload, Alternative endpoint to connect to (useful for testing) -s3ManagedLedgerOffloadServiceEndpoint= - -# For Amazon S3 ledger offload, Max block size in bytes. (64MB by default, 5MB minimum) -s3ManagedLedgerOffloadMaxBlockSizeInBytes=67108864 - -# For Amazon S3 ledger offload, Read buffer size in bytes (1MB by default) -s3ManagedLedgerOffloadReadBufferSizeInBytes=1048576 - -# For Google Cloud Storage ledger offload, region where offload bucket is located. -# reference this page for more details: https://cloud.google.com/storage/docs/bucket-locations -gcsManagedLedgerOffloadRegion= - -# For Google Cloud Storage ledger offload, Bucket to place offloaded ledger into -gcsManagedLedgerOffloadBucket= - -# For Google Cloud Storage ledger offload, Max block size in bytes. (64MB by default, 5MB minimum) -gcsManagedLedgerOffloadMaxBlockSizeInBytes=67108864 - -# For Google Cloud Storage ledger offload, Read buffer size in bytes (1MB by default) -gcsManagedLedgerOffloadReadBufferSizeInBytes=1048576 - -# For Google Cloud Storage, path to json file containing service account credentials. -# For more details, see the "Service Accounts" section of https://support.google.com/googleapi/answer/6158849 -gcsManagedLedgerOffloadServiceAccountKeyFile= - -### --- Deprecated config variables --- ### - -# Deprecated. Use configurationStoreServers -globalZookeeperServers= - -# Deprecated - Enable TLS when talking with other clusters to replicate messages -replicationTlsEnabled=false - -### --- Prometheus metric provider configuration --- ### -# Prometheus stats rollover latency, default: 60s -kopPrometheusStatsLatencyRolloverSeconds=60 diff --git a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java index 4149b0f334..c2a5507561 100644 --- a/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java +++ b/kafka-impl/src/main/java/io/streamnative/pulsar/handlers/kop/KopServerStats.java @@ -21,8 +21,6 @@ public interface KopServerStats { String SERVER_SCOPE = "kop_server"; - String SERVER_STATUS = "SERVER_STATUS"; - /** * PRODUCE STATS. */ diff --git a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java index b9b32e6331..332416f360 100644 --- a/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java +++ b/tests/src/test/java/io/streamnative/pulsar/handlers/kop/MetricsProviderTest.java @@ -122,5 +122,8 @@ public void testMetricsProvider() throws Exception { sb.append(str); } Assert.assertTrue(sb.toString().contains("kop_server_HANDLE_PRODUCE_REQUEST")); + Assert.assertTrue(sb.toString().contains("kop_server_PRODUCE_ENCODE")); + Assert.assertTrue(sb.toString().contains("kop_server_MESSAGE_PUBLISH")); + Assert.assertTrue(sb.toString().contains("kop_server_MESSAGE_QUEUED_LATENCY")); } }