From 5d47a21521ae77ac0eb39a68ae4146e94ccc5da0 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Fri, 29 Apr 2022 08:38:55 +0800 Subject: [PATCH 1/2] [ISSUE 3247] Fix jvm_memory_direct_bytes_used metrics when using jdk11+ --- .../prometheus/PrometheusMetricsProvider.java | 26 +++++------- .../TestPrometheusMetricsProvider.java | 40 +++++++++++++++++++ 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java index 974d914e3e3..5c781234460 100644 --- a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java +++ b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java @@ -30,19 +30,19 @@ import io.prometheus.client.hotspot.MemoryPoolsExports; import io.prometheus.client.hotspot.StandardExports; import io.prometheus.client.hotspot.ThreadExports; - import java.io.IOException; import java.io.Writer; -import java.lang.reflect.Field; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ManagementFactory; import java.net.InetSocketAddress; import java.util.Collections; +import java.util.List; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.stats.StatsProvider; import org.apache.bookkeeper.stats.ThreadRegistry; @@ -130,7 +130,7 @@ public void start(Configuration conf) { registerMetrics(Gauge.build("jvm_memory_direct_bytes_used", "-").create().setChild(new Child() { @Override public double get() { - return directMemoryUsage != null ? directMemoryUsage.longValue() : Double.NaN; + return poolMxBeanOp.isPresent() ? poolMxBeanOp.get().getMemoryUsed() : Double.NaN; } })); @@ -218,18 +218,10 @@ private void registerMetrics(Collector collector) { /* * Try to get Netty counter of used direct memory. This will be correct, unlike the JVM values. */ - private static final AtomicLong directMemoryUsage; + private static final Optional poolMxBeanOp; static { - AtomicLong tmpDirectMemoryUsage = null; - - try { - Field field = PlatformDependent.class.getDeclaredField("DIRECT_MEMORY_COUNTER"); - field.setAccessible(true); - tmpDirectMemoryUsage = (AtomicLong) field.get(null); - } catch (Throwable t) { - log.warn("Failed to access netty DIRECT_MEMORY_COUNTER field {}", t.getMessage()); - } - - directMemoryUsage = tmpDirectMemoryUsage; + List platformMXBeans = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); + poolMxBeanOp = platformMXBeans.stream() + .filter(bufferPoolMXBean -> bufferPoolMXBean.getName().equals("direct")).findAny(); } } diff --git a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/test/java/org/apache/bookkeeper/stats/prometheus/TestPrometheusMetricsProvider.java b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/test/java/org/apache/bookkeeper/stats/prometheus/TestPrometheusMetricsProvider.java index df954e64d98..999be26cbb7 100644 --- a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/test/java/org/apache/bookkeeper/stats/prometheus/TestPrometheusMetricsProvider.java +++ b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/test/java/org/apache/bookkeeper/stats/prometheus/TestPrometheusMetricsProvider.java @@ -21,11 +21,16 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; +import java.io.StringWriter; +import java.nio.ByteBuffer; import java.util.Collections; +import java.util.HashMap; + import lombok.Cleanup; import org.apache.bookkeeper.stats.Counter; import org.apache.bookkeeper.stats.StatsLogger; import org.apache.commons.configuration.PropertiesConfiguration; +import org.junit.Assert; import org.junit.Test; /** @@ -111,4 +116,39 @@ public void testTwoCounters() throws Exception { assertEquals(1, provider.counters.size()); } + @Test + public void testJvmDirectMemoryMetrics() throws Exception { + PropertiesConfiguration config = new PropertiesConfiguration(); + config.setProperty(PrometheusMetricsProvider.PROMETHEUS_STATS_HTTP_ENABLE, true); + config.setProperty(PrometheusMetricsProvider.PROMETHEUS_STATS_HTTP_PORT, 0); + config.setProperty(PrometheusMetricsProvider.PROMETHEUS_STATS_HTTP_ADDRESS, "127.0.0.1"); + ByteBuffer byteBuffer = ByteBuffer.allocateDirect(25); + PrometheusMetricsProvider provider = new PrometheusMetricsProvider(); + try { + provider.start(config); + assertNotNull(provider.server); + StringWriter writer = new StringWriter(); + provider.writeAllMetrics(writer); + String s = writer.toString(); + String[] split = s.split(System.lineSeparator()); + HashMap map = new HashMap<>(); + for (String str : split) { + String[] aux = str.split(" "); + map.put(aux[0], aux[1]); + } + String directBytesMax = map.get("jvm_memory_direct_bytes_max{}"); + Assert.assertNotNull(directBytesMax); + Assert.assertNotEquals("Nan", directBytesMax); + Assert.assertNotEquals("-1", directBytesMax); + String directBytesUsed = map.get("jvm_memory_direct_bytes_used{}"); + Assert.assertNotNull(directBytesUsed); + Assert.assertNotEquals("Nan", directBytesUsed); + Assert.assertTrue(Double.parseDouble(directBytesUsed) > 25); + // ensure byteBuffer doesn't gc + byteBuffer.clear(); + } finally { + provider.stop(); + } + } + } From b5a8189b0a050b5f6922939ce8ddd8e9bbfc9edc Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Thu, 5 May 2022 14:37:15 +0800 Subject: [PATCH 2/2] remove annotation --- .../stats/prometheus/PrometheusMetricsProvider.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java index 5c781234460..ab81992531b 100644 --- a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java +++ b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/PrometheusMetricsProvider.java @@ -215,10 +215,8 @@ private void registerMetrics(Collector collector) { private static final Logger log = LoggerFactory.getLogger(PrometheusMetricsProvider.class); - /* - * Try to get Netty counter of used direct memory. This will be correct, unlike the JVM values. - */ private static final Optional poolMxBeanOp; + static { List platformMXBeans = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class); poolMxBeanOp = platformMXBeans.stream()