From 666f8a8fe5f9d21a282a74d003c5e91475962f28 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Mon, 9 May 2022 18:58:17 +0800 Subject: [PATCH 1/4] Update datasketches version --- .../src/main/resources/LICENSE-all.bin.txt | 6 +++--- .../src/main/resources/LICENSE-server.bin.txt | 6 +++--- dependencies.gradle | 4 ++-- pom.xml | 6 +++--- .../prometheus-metrics-provider/pom.xml | 6 +++--- .../prometheus/DataSketchesOpStatsLogger.java | 14 +++++++------- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index e75feaff686..d8476ddf6c9 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -266,8 +266,8 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-util-ajax-9.4.46.v20220331.jar [22] - lib/org.rocksdb-rocksdbjni-6.29.4.1.jar [23] - lib/com.beust-jcommander-1.78.jar [24] -- lib/com.yahoo.datasketches-memory-0.8.3.jar [25] -- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] +- lib/org.apache.datasketches.datasketches-java-2.0.0.jar [25] +- lib/org.apache.datasketches.datasketches-memory-2.0.0.jar [25] - lib/net.jpountz.lz4-lz4-1.3.0.jar [26] - lib/com.google.api.grpc-proto-google-common-protos-2.0.1.jar [28] - lib/com.google.code.gson-gson-2.8.9.jar [29] @@ -338,7 +338,7 @@ Apache Software License, Version 2. [22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.46.v20220331 [23] Source available at https://github.com/facebook/rocksdb/tree/v6.22.1 [24] Source available at https://github.com/cbeust/jcommander/tree/1.78 -[25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 +[25] Source available at https://github.com/apache/datasketches-java/tree/2.0.0 [26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 [28] Source available at https://github.com/googleapis/googleapis [29] Source available at https://github.com/google/gson/tree/gson-parent-2.8.9 diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 7fb754fc3ba..1e4cfcf8f3c 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -266,8 +266,8 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-util-ajax-9.4.46.v20220331.jar [22] - lib/org.rocksdb-rocksdbjni-6.29.4.1.jar [23] - lib/com.beust-jcommander-1.78.jar [24] -- lib/com.yahoo.datasketches-memory-0.8.3.jar [25] -- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] +- lib/org.apache.datasketches.datasketches-java-2.0.0.jar [25] +- lib/org.apache.datasketches.datasketches-memory-2.0.0.jar [25] - lib/net.jpountz.lz4-lz4-1.3.0.jar [26] - lib/com.google.api.grpc-proto-google-common-protos-2.0.1.jar [28] - lib/com.google.code.gson-gson-2.8.9.jar [29] @@ -335,7 +335,7 @@ Apache Software License, Version 2. [22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.46.v20220331 [23] Source available at https://github.com/facebook/rocksdb/tree/v6.16.4 [24] Source available at https://github.com/cbeust/jcommander/tree/1.78 -[25] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3 +[25] Source available at https://github.com/apache/datasketches-java/tree/2.0.0 [26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 [28] Source available at https://github.com/googleapis/googleapis [29] Source available at https://github.com/google/gson/tree/gson-parent-2.8.9 diff --git a/dependencies.gradle b/dependencies.gradle index efc1615f976..84f601c4385 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -86,7 +86,7 @@ depVersions = [ thrift: "0.14.2", testcontainers: "1.16.3", vertx: "3.9.8", - yahooDatasketches: "0.8.3", + yahooDatasketches: "2.0.0", zookeeper: "3.8.0", ] @@ -235,7 +235,7 @@ depLibs = [ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-core' exclude group: 'io.netty' }, - yahooDatasketches: "com.yahoo.datasketches:sketches-core:${depVersions.yahooDatasketches}", + yahooDatasketches: "org.apache.datasketches:sketches-java:${depVersions.yahooDatasketches}", zookeeper: dependencies.create("org.apache.zookeeper:zookeeper:${depVersions.zookeeper}") { exclude group: 'io.netty' exclude group: 'ch.qos.logback' diff --git a/pom.xml b/pom.xml index 3fc2e1d74eb..65ef5b4181b 100644 --- a/pom.xml +++ b/pom.xml @@ -157,7 +157,7 @@ 9.1.3 2.0.9 0.8.1 - 0.8.3 + 2.0.0 4.5.13 4.4.15 3.19.2 @@ -626,8 +626,8 @@ - com.yahoo.datasketches - sketches-core + org.apache.datasketches + datasketches-java ${datasketches.version} diff --git a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml index 0956d579c13..40eaa2a203c 100644 --- a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml +++ b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml @@ -61,10 +61,10 @@ com.google.guava guava - + - com.yahoo.datasketches - sketches-core + org.apache.datasketches + datasketches-java diff --git a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/DataSketchesOpStatsLogger.java b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/DataSketchesOpStatsLogger.java index 403cb18ac85..5bd51ceb233 100644 --- a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/DataSketchesOpStatsLogger.java +++ b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/prometheus/DataSketchesOpStatsLogger.java @@ -16,11 +16,6 @@ */ package org.apache.bookkeeper.stats.prometheus; -import com.yahoo.sketches.quantiles.DoublesSketch; -import com.yahoo.sketches.quantiles.DoublesSketchBuilder; -import com.yahoo.sketches.quantiles.DoublesUnion; -import com.yahoo.sketches.quantiles.DoublesUnionBuilder; - import io.netty.util.concurrent.FastThreadLocal; import java.util.Map; @@ -31,6 +26,11 @@ import org.apache.bookkeeper.stats.OpStatsData; import org.apache.bookkeeper.stats.OpStatsLogger; +import org.apache.datasketches.quantiles.DoublesSketch; +import org.apache.datasketches.quantiles.DoublesSketchBuilder; +import org.apache.datasketches.quantiles.DoublesUnion; +import org.apache.datasketches.quantiles.DoublesUnionBuilder; +import org.apache.datasketches.quantiles.UpdateDoublesSketch; /** * OpStatsLogger implementation that uses DataSketches library to calculate the approximated latency quantiles. @@ -193,8 +193,8 @@ public void initializeThread(Map labels) { } private static class LocalData { - private final DoublesSketch successSketch = new DoublesSketchBuilder().build(); - private final DoublesSketch failSketch = new DoublesSketchBuilder().build(); + private final UpdateDoublesSketch successSketch = new DoublesSketchBuilder().build(); + private final UpdateDoublesSketch failSketch = new DoublesSketchBuilder().build(); private final StampedLock lock = new StampedLock(); } From ebd5b4a2498858b5a2445611b5c1725bb616ce7d Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Mon, 9 May 2022 18:58:24 +0800 Subject: [PATCH 2/4] Update datasketches version --- bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt | 4 ++-- bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt | 4 ++-- pom.xml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index d8476ddf6c9..6e7fb74c4bb 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -266,7 +266,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-util-ajax-9.4.46.v20220331.jar [22] - lib/org.rocksdb-rocksdbjni-6.29.4.1.jar [23] - lib/com.beust-jcommander-1.78.jar [24] -- lib/org.apache.datasketches.datasketches-java-2.0.0.jar [25] +- lib/org.apache.datasketches.datasketches-java-3.2.0.jar [25] - lib/org.apache.datasketches.datasketches-memory-2.0.0.jar [25] - lib/net.jpountz.lz4-lz4-1.3.0.jar [26] - lib/com.google.api.grpc-proto-google-common-protos-2.0.1.jar [28] @@ -338,7 +338,7 @@ Apache Software License, Version 2. [22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.46.v20220331 [23] Source available at https://github.com/facebook/rocksdb/tree/v6.22.1 [24] Source available at https://github.com/cbeust/jcommander/tree/1.78 -[25] Source available at https://github.com/apache/datasketches-java/tree/2.0.0 +[25] Source available at https://github.com/apache/datasketches-java/tree/3.2.0 [26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 [28] Source available at https://github.com/googleapis/googleapis [29] Source available at https://github.com/google/gson/tree/gson-parent-2.8.9 diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 1e4cfcf8f3c..6ac5dc5b046 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -266,7 +266,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-util-ajax-9.4.46.v20220331.jar [22] - lib/org.rocksdb-rocksdbjni-6.29.4.1.jar [23] - lib/com.beust-jcommander-1.78.jar [24] -- lib/org.apache.datasketches.datasketches-java-2.0.0.jar [25] +- lib/org.apache.datasketches.datasketches-java-3.2.0.jar [25] - lib/org.apache.datasketches.datasketches-memory-2.0.0.jar [25] - lib/net.jpountz.lz4-lz4-1.3.0.jar [26] - lib/com.google.api.grpc-proto-google-common-protos-2.0.1.jar [28] @@ -335,7 +335,7 @@ Apache Software License, Version 2. [22] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.46.v20220331 [23] Source available at https://github.com/facebook/rocksdb/tree/v6.16.4 [24] Source available at https://github.com/cbeust/jcommander/tree/1.78 -[25] Source available at https://github.com/apache/datasketches-java/tree/2.0.0 +[25] Source available at https://github.com/apache/datasketches-java/tree/3.2.0 [26] Source available at https://github.com/lz4/lz4-java/tree/1.3.0 [28] Source available at https://github.com/googleapis/googleapis [29] Source available at https://github.com/google/gson/tree/gson-parent-2.8.9 diff --git a/pom.xml b/pom.xml index 65ef5b4181b..3c1429d735d 100644 --- a/pom.xml +++ b/pom.xml @@ -157,7 +157,7 @@ 9.1.3 2.0.9 0.8.1 - 2.0.0 + 3.2.0 4.5.13 4.4.15 3.19.2 From dbd8ca5e411ea7f08c037fd7fa161e8f223b6975 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Mon, 9 May 2022 19:59:57 +0800 Subject: [PATCH 3/4] fix gradle script --- dependencies.gradle | 4 ++-- .../prometheus-metrics-provider/build.gradle | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dependencies.gradle b/dependencies.gradle index 84f601c4385..764ff371073 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -36,6 +36,7 @@ depVersions = [ commonsLang3: "3.6", commonsBeanutils: "1.9.3", curator: "5.1.0", + datasketches: "3.2.0", dockerJava: "3.2.13", dropwizard: "4.1.12.1", errorprone: "2.4.0", @@ -86,7 +87,6 @@ depVersions = [ thrift: "0.14.2", testcontainers: "1.16.3", vertx: "3.9.8", - yahooDatasketches: "2.0.0", zookeeper: "3.8.0", ] @@ -115,6 +115,7 @@ depLibs = [ curatorRecipes: dependencies.create("org.apache.curator:curator-recipes:${depVersions.curator}") { exclude group: 'org.apache.curator', module: 'curator-framework' }, + datasketches: "org.apache.datasketches:datasketches-java:${depVersions.datasketches}", dockerJava: "com.github.docker-java:docker-java:${depVersions.dockerJava}", errorprone: "com.google.errorprone:error_prone_annotations:${depVersions.errorprone}", etcd: dependencies.create("io.etcd:jetcd-core:${depVersions.etcd}") { @@ -235,7 +236,6 @@ depLibs = [ exclude group: 'com.fasterxml.jackson.core', module: 'jackson-core' exclude group: 'io.netty' }, - yahooDatasketches: "org.apache.datasketches:sketches-java:${depVersions.yahooDatasketches}", zookeeper: dependencies.create("org.apache.zookeeper:zookeeper:${depVersions.zookeeper}") { exclude group: 'io.netty' exclude group: 'ch.qos.logback' diff --git a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/build.gradle b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/build.gradle index d99f1a3c974..4c181b1f5a4 100644 --- a/stats/bookkeeper-stats-providers/prometheus-metrics-provider/build.gradle +++ b/stats/bookkeeper-stats-providers/prometheus-metrics-provider/build.gradle @@ -32,7 +32,7 @@ dependencies { implementation depLibs.prometheusSimpleClientHotspot implementation depLibs.prometheusSimpleClientServlet implementation depLibs.slf4j - implementation depLibs.yahooDatasketches + implementation depLibs.datasketches testCompileOnly depLibs.lombok testImplementation depLibs.junit From 35e3e3343bb29f1c4f622e3009e967632e643006 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Wed, 11 May 2022 14:39:59 +0800 Subject: [PATCH 4/4] add cve suppressions --- bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt | 4 ++-- bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt | 4 ++-- src/owasp-dependency-check-suppressions.xml | 8 ++++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 6e7fb74c4bb..fc95dbdb7c4 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -266,8 +266,8 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-util-ajax-9.4.46.v20220331.jar [22] - lib/org.rocksdb-rocksdbjni-6.29.4.1.jar [23] - lib/com.beust-jcommander-1.78.jar [24] -- lib/org.apache.datasketches.datasketches-java-3.2.0.jar [25] -- lib/org.apache.datasketches.datasketches-memory-2.0.0.jar [25] +- lib/org.apache.datasketches-datasketches-java-3.2.0.jar [25] +- lib/org.apache.datasketches-datasketches-memory-2.0.0.jar [25] - lib/net.jpountz.lz4-lz4-1.3.0.jar [26] - lib/com.google.api.grpc-proto-google-common-protos-2.0.1.jar [28] - lib/com.google.code.gson-gson-2.8.9.jar [29] diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 6ac5dc5b046..6736b046a0b 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -266,8 +266,8 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-util-ajax-9.4.46.v20220331.jar [22] - lib/org.rocksdb-rocksdbjni-6.29.4.1.jar [23] - lib/com.beust-jcommander-1.78.jar [24] -- lib/org.apache.datasketches.datasketches-java-3.2.0.jar [25] -- lib/org.apache.datasketches.datasketches-memory-2.0.0.jar [25] +- lib/org.apache.datasketches-datasketches-java-3.2.0.jar [25] +- lib/org.apache.datasketches-datasketches-memory-2.0.0.jar [25] - lib/net.jpountz.lz4-lz4-1.3.0.jar [26] - lib/com.google.api.grpc-proto-google-common-protos-2.0.1.jar [28] - lib/com.google.code.gson-gson-2.8.9.jar [29] diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index 082dfae0364..3ac9ad8c80e 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -335,5 +335,13 @@ file name: netty-tcnative-boringssl-static-2.0.50.Final-windows-x86_64.jar b6211fb3224bfd2dadf10273a15211f592174652 CVE-2015-1205 + + + + ceb2774dcd51210437a2f139b64d33213bf7f523 + CVE-2021-40531 +