From 82c274d31fd5591d96325ff4c09aaa859674a80a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 5 May 2026 10:48:35 +0300 Subject: [PATCH] Migrate from Yahoo DataSketches to Apache DataSketches 7.0.1 (KLL) The com.yahoo.datasketches:sketches-core:0.8.3 dependency has been unmaintained since the project moved to Apache. Switch to the current org.apache.datasketches:datasketches-java:7.0.1 and adopt the KLL quantiles sketch, which the upstream maintainers recommend over the classic DoublesSketch for new code (faster, smaller, comparable accuracy at default K=200). DataSketchesOpStatsLogger changes: - On-heap throughout, matching the previous code (no off-heap usage). - KLL has no reset(); per-thread sketches are reassigned to a fresh newHeapInstance() under the existing StampedLock write lock. - KLL has no DoublesUnion; aggregation uses merge() directly into a freshly allocated KllDoublesSketch each rotation. - KllDoublesSketch.getQuantile() throws SketchesArgumentException on an empty sketch, so getQuantileValue() now guards with isEmpty() to preserve the prior NaN return. LICENSE bundles list datasketches-java-7.0.1.jar and the transitively required datasketches-memory-4.1.0.jar separately, since the two jars come from distinct upstream repositories. --- .../src/main/resources/LICENSE-all.bin.txt | 7 ++-- .../src/main/resources/LICENSE-server.bin.txt | 7 ++-- pom.xml | 6 ++-- .../prometheus-metrics-provider/pom.xml | 4 +-- .../prometheus/DataSketchesOpStatsLogger.java | 33 +++++++++---------- 5 files changed, 28 insertions(+), 29 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 207660d466d..7817e000899 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -279,8 +279,8 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar [22] - lib/org.rocksdb-rocksdbjni-9.9.3.jar [23] - lib/com.beust-jcommander-1.82.jar [24] -- lib/com.yahoo.datasketches-memory-0.8.3.jar [25] -- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] +- lib/org.apache.datasketches-datasketches-java-7.0.1.jar [25] +- lib/org.apache.datasketches-datasketches-memory-4.1.0.jar [27] - lib/at.yawk.lz4-lz4-java-1.10.2.jar [26] - lib/com.google.api-api-common-2.53.0.jar [63] - lib/com.google.api.grpc-proto-google-common-protos-2.63.2.jar [28] @@ -387,8 +387,9 @@ Apache Software License, Version 2. [22] Source available at https://github.com/jetty/jetty.project/tree/jetty-12.1.7 [23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82 -[25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 +[25] Source available at https://github.com/apache/datasketches-java/tree/7.0.1 [26] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2 +[27] Source available at https://github.com/apache/datasketches-memory/tree/4.1.0 [28] Source available at https://github.com/googleapis/sdk-platform-java/tree/v2.63.2/java-common-protos [29] Source available at https://github.com/google/gson/tree/gson-parent-2.12.1 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.31.1 diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index b9f76c2ef34..98ffd619ef8 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -279,8 +279,8 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar [22] - lib/org.rocksdb-rocksdbjni-9.9.3.jar [23] - lib/com.beust-jcommander-1.82.jar [24] -- lib/com.yahoo.datasketches-memory-0.8.3.jar [25] -- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] +- lib/org.apache.datasketches-datasketches-java-7.0.1.jar [25] +- lib/org.apache.datasketches-datasketches-memory-4.1.0.jar [27] - lib/at.yawk.lz4-lz4-java-1.10.2.jar [26] - lib/com.google.api-api-common-2.53.0.jar [62] - lib/com.google.api.grpc-proto-google-common-protos-2.63.2.jar [28] @@ -383,8 +383,9 @@ Apache Software License, Version 2. [22] Source available at https://github.com/jetty/jetty.project/tree/jetty-12.1.7 [23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82 -[25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 +[25] Source available at https://github.com/apache/datasketches-java/tree/7.0.1 [26] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2 +[27] Source available at https://github.com/apache/datasketches-memory/tree/4.1.0 [28] Source available at https://github.com/googleapis/sdk-platform-java/tree/v2.63.2/java-common-protos [29] Source available at https://github.com/google/gson/tree/gson-parent-2.12.1 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.31.1 diff --git a/pom.xml b/pom.xml index 68d5e0ecb40..bc6daa3912d 100644 --- a/pom.xml +++ b/pom.xml @@ -182,7 +182,7 @@ 4.11.0 4.2.12.Final 0.15.0 - 0.8.3 + 7.0.1 4.5.13 4.4.15 4.34.0 @@ -666,8 +666,8 @@ - com.yahoo.datasketches - sketches-core + org.apache.datasketches + datasketches-java ${datasketches.version} diff --git a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml index c36284757e8..57af90075c6 100644 --- a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml +++ b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml @@ -70,8 +70,8 @@ - com.yahoo.datasketches - sketches-core + org.apache.datasketches + datasketches-java diff --git a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/DataSketchesOpStatsLogger.java b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/DataSketchesOpStatsLogger.java index 1abe85088c6..f9a069d2831 100644 --- a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/DataSketchesOpStatsLogger.java +++ b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/DataSketchesOpStatsLogger.java @@ -16,10 +16,6 @@ */ package org.apache.bookkeeper.stats.prometheus; -import com.yahoo.sketches.quantiles.DoublesSketch; -import com.yahoo.sketches.quantiles.DoublesSketchBuilder; -import com.yahoo.sketches.quantiles.DoublesUnion; -import com.yahoo.sketches.quantiles.DoublesUnionBuilder; import io.netty.util.concurrent.FastThreadLocal; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -28,6 +24,7 @@ import java.util.concurrent.locks.StampedLock; import org.apache.bookkeeper.stats.OpStatsData; import org.apache.bookkeeper.stats.OpStatsLogger; +import org.apache.datasketches.kll.KllDoublesSketch; /** * OpStatsLogger implementation that uses DataSketches library to calculate the approximated latency quantiles. @@ -43,8 +40,8 @@ public class DataSketchesOpStatsLogger implements OpStatsLogger { /* * These are the sketches where all the aggregated results are published. */ - private volatile DoublesSketch successResult; - private volatile DoublesSketch failResult; + private volatile KllDoublesSketch successResult; + private volatile KllDoublesSketch failResult; private final LongAdder successCountAdder = new LongAdder(); private final LongAdder failCountAdder = new LongAdder(); @@ -145,22 +142,22 @@ public void rotateLatencyCollection() { current = replacement; replacement = local; - final DoublesUnion aggregateSuccess = new DoublesUnionBuilder().build(); - final DoublesUnion aggregateFail = new DoublesUnionBuilder().build(); + final KllDoublesSketch aggregateSuccess = KllDoublesSketch.newHeapInstance(); + final KllDoublesSketch aggregateFail = KllDoublesSketch.newHeapInstance(); local.map.forEach((localData, b) -> { long stamp = localData.lock.writeLock(); try { - aggregateSuccess.update(localData.successSketch); - localData.successSketch.reset(); - aggregateFail.update(localData.failSketch); - localData.failSketch.reset(); + aggregateSuccess.merge(localData.successSketch); + aggregateFail.merge(localData.failSketch); + localData.successSketch = KllDoublesSketch.newHeapInstance(); + localData.failSketch = KllDoublesSketch.newHeapInstance(); } finally { localData.lock.unlockWrite(stamp); } }); - successResult = aggregateSuccess.getResultAndReset(); - failResult = aggregateFail.getResultAndReset(); + successResult = aggregateSuccess; + failResult = aggregateFail; } public long getCount(boolean success) { @@ -172,8 +169,8 @@ public long getSum(boolean success) { } public double getQuantileValue(boolean success, double quantile) { - DoublesSketch s = success ? successResult : failResult; - return s != null ? s.getQuantile(quantile) : Double.NaN; + KllDoublesSketch s = success ? successResult : failResult; + return (s != null && !s.isEmpty()) ? s.getQuantile(quantile) : Double.NaN; } public Map getLabels() { @@ -190,8 +187,8 @@ public void initializeThread(Map labels) { } private static class LocalData { - private final DoublesSketch successSketch = new DoublesSketchBuilder().build(); - private final DoublesSketch failSketch = new DoublesSketchBuilder().build(); + private KllDoublesSketch successSketch = KllDoublesSketch.newHeapInstance(); + private KllDoublesSketch failSketch = KllDoublesSketch.newHeapInstance(); private final StampedLock lock = new StampedLock(); }