Migrate from Yahoo DataSketches to Apache DataSketches 7.0.1 (KLL)

bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt

@@ -279,8 +279,8 @@ Apache Software License, Version 2.
 - lib/org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar [22]
 - lib/org.rocksdb-rocksdbjni-9.9.3.jar [23]
 - lib/com.beust-jcommander-1.82.jar [24]
-- lib/com.yahoo.datasketches-memory-0.8.3.jar [25]
-- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25]
+- lib/org.apache.datasketches-datasketches-java-7.0.1.jar [25]
+- lib/org.apache.datasketches-datasketches-memory-4.1.0.jar [27]
 - lib/at.yawk.lz4-lz4-java-1.10.2.jar [26]
 - lib/com.google.api-api-common-2.53.0.jar [63]
 - lib/com.google.api.grpc-proto-google-common-protos-2.63.2.jar [28]
@@ -387,8 +387,9 @@ Apache Software License, Version 2.
 [22] Source available at https://github.com/jetty/jetty.project/tree/jetty-12.1.7
 [23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3
 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82
-[25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
+[25] Source available at https://github.com/apache/datasketches-java/tree/7.0.1
 [26] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2
+[27] Source available at https://github.com/apache/datasketches-memory/tree/4.1.0
 [28] Source available at https://github.com/googleapis/sdk-platform-java/tree/v2.63.2/java-common-protos
 [29] Source available at https://github.com/google/gson/tree/gson-parent-2.12.1
 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.31.1

bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt

@@ -279,8 +279,8 @@ Apache Software License, Version 2.
 - lib/org.eclipse.jetty.toolchain-jetty-servlet-api-4.0.9.jar [22]
 - lib/org.rocksdb-rocksdbjni-9.9.3.jar [23]
 - lib/com.beust-jcommander-1.82.jar [24]
-- lib/com.yahoo.datasketches-memory-0.8.3.jar [25]
-- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25]
+- lib/org.apache.datasketches-datasketches-java-7.0.1.jar [25]
+- lib/org.apache.datasketches-datasketches-memory-4.1.0.jar [27]
 - lib/at.yawk.lz4-lz4-java-1.10.2.jar [26]
 - lib/com.google.api-api-common-2.53.0.jar [62]
 - lib/com.google.api.grpc-proto-google-common-protos-2.63.2.jar [28]
@@ -383,8 +383,9 @@ Apache Software License, Version 2.
 [22] Source available at https://github.com/jetty/jetty.project/tree/jetty-12.1.7
 [23] Source available at https://github.com/facebook/rocksdb/tree/v9.9.3
 [24] Source available at https://github.com/cbeust/jcommander/tree/1.82
-[25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
+[25] Source available at https://github.com/apache/datasketches-java/tree/7.0.1
 [26] Source available at https://github.com/yawkat/lz4-java/tree/v1.10.2
+[27] Source available at https://github.com/apache/datasketches-memory/tree/4.1.0
 [28] Source available at https://github.com/googleapis/sdk-platform-java/tree/v2.63.2/java-common-protos
 [29] Source available at https://github.com/google/gson/tree/gson-parent-2.12.1
 [30] Source available at https://github.com/census-instrumentation/opencensus-java/tree/v0.31.1

pom.xml

@@ -182,7 +182,7 @@
     <mockito.version>4.11.0</mockito.version>
     <netty.version>4.2.12.Final</netty.version>
     <prometheus.version>0.15.0</prometheus.version>
-    <datasketches.version>0.8.3</datasketches.version>
+    <datasketches.version>7.0.1</datasketches.version>
     <httpclient.version>4.5.13</httpclient.version>
     <httpcore.version>4.4.15</httpcore.version>
     <protobuf.version>4.34.0</protobuf.version>
@@ -666,8 +666,8 @@
       </dependency>
       <!-- data-sketches -->
       <dependency>
-        <groupId>com.yahoo.datasketches</groupId>
-        <artifactId>sketches-core</artifactId>
+        <groupId>org.apache.datasketches</groupId>
+        <artifactId>datasketches-java</artifactId>
         <version>${datasketches.version}</version>
       </dependency>
       <!-- opentelemetry -->

stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml

@@ -70,8 +70,8 @@
     </dependency>
 
     <dependency>
-      <groupId>com.yahoo.datasketches</groupId>
-      <artifactId>sketches-core</artifactId>
+      <groupId>org.apache.datasketches</groupId>
+      <artifactId>datasketches-java</artifactId>
     </dependency>
 
     <dependency>

stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/DataSketchesOpStatsLogger.java

@@ -16,10 +16,6 @@
  */
 package org.apache.bookkeeper.stats.prometheus;
 
-import com.yahoo.sketches.quantiles.DoublesSketch;
-import com.yahoo.sketches.quantiles.DoublesSketchBuilder;
-import com.yahoo.sketches.quantiles.DoublesUnion;
-import com.yahoo.sketches.quantiles.DoublesUnionBuilder;
 import io.netty.util.concurrent.FastThreadLocal;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -28,6 +24,7 @@
 import java.util.concurrent.locks.StampedLock;
 import org.apache.bookkeeper.stats.OpStatsData;
 import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.apache.datasketches.kll.KllDoublesSketch;
 
 /**
  * OpStatsLogger implementation that uses DataSketches library to calculate the approximated latency quantiles.
@@ -43,8 +40,8 @@ public class DataSketchesOpStatsLogger implements OpStatsLogger {
     /*
      * These are the sketches where all the aggregated results are published.
      */
-    private volatile DoublesSketch successResult;
-    private volatile DoublesSketch failResult;
+    private volatile KllDoublesSketch successResult;
+    private volatile KllDoublesSketch failResult;
 
     private final LongAdder successCountAdder = new LongAdder();
     private final LongAdder failCountAdder = new LongAdder();
@@ -145,22 +142,22 @@ public void rotateLatencyCollection() {
         current = replacement;
         replacement = local;
 
-        final DoublesUnion aggregateSuccess = new DoublesUnionBuilder().build();
-        final DoublesUnion aggregateFail = new DoublesUnionBuilder().build();
+        final KllDoublesSketch aggregateSuccess = KllDoublesSketch.newHeapInstance();
+        final KllDoublesSketch aggregateFail = KllDoublesSketch.newHeapInstance();
         local.map.forEach((localData, b) -> {
             long stamp = localData.lock.writeLock();
             try {
-                aggregateSuccess.update(localData.successSketch);
-                localData.successSketch.reset();
-                aggregateFail.update(localData.failSketch);
-                localData.failSketch.reset();
+                aggregateSuccess.merge(localData.successSketch);
+                aggregateFail.merge(localData.failSketch);
+                localData.successSketch = KllDoublesSketch.newHeapInstance();
+                localData.failSketch = KllDoublesSketch.newHeapInstance();
             } finally {
                 localData.lock.unlockWrite(stamp);
             }
         });
 
-        successResult = aggregateSuccess.getResultAndReset();
-        failResult = aggregateFail.getResultAndReset();
+        successResult = aggregateSuccess;
+        failResult = aggregateFail;
     }
 
     public long getCount(boolean success) {
@@ -172,8 +169,8 @@ public long getSum(boolean success) {
     }
 
     public double getQuantileValue(boolean success, double quantile) {
-        DoublesSketch s = success ? successResult : failResult;
-        return s != null ? s.getQuantile(quantile) : Double.NaN;
+        KllDoublesSketch s = success ? successResult : failResult;
+        return (s != null && !s.isEmpty()) ? s.getQuantile(quantile) : Double.NaN;
     }
 
     public Map<String, String> getLabels() {
@@ -190,8 +187,8 @@ public void initializeThread(Map<String, String> labels) {
     }
 
     private static class LocalData {
-        private final DoublesSketch successSketch = new DoublesSketchBuilder().build();
-        private final DoublesSketch failSketch = new DoublesSketchBuilder().build();
+        private KllDoublesSketch successSketch = KllDoublesSketch.newHeapInstance();
+        private KllDoublesSketch failSketch = KllDoublesSketch.newHeapInstance();
         private final StampedLock lock = new StampedLock();
     }