diff --git a/.github/ISSUE_TEMPLATE/flaky_test.md b/.github/ISSUE_TEMPLATE/flaky_test.md new file mode 100644 index 00000000000..fc8159776c7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/flaky_test.md @@ -0,0 +1,31 @@ +--- +name: Flaky test +about: Report a flaky test failure +title: 'Flaky-test: test_class.test_method' +labels: ["area/tests", "flaky-tests"] +assignees: '' +--- + +test_class.test_method is flaky. It fails sporadically. + +``` +[relevant parts of the exception stacktrace here] +``` + + - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - org.apache.bookkeeper - bookkeeper-benchmark - Apache BookKeeper :: Benchmark - - UTF-8 - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - target/latencyDump.dat - - - - - - - - org.apache.zookeeper - zookeeper - test-jar - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - test - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - compile - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test-jar - test - - - diff --git a/bookkeeper-common-allocator/pom.xml b/bookkeeper-common-allocator/pom.xml deleted file mode 100644 index 31315773665..00000000000 --- a/bookkeeper-common-allocator/pom.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-common-allocator - Apache BookKeeper :: Common :: Allocator - - - io.netty - netty-buffer - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - diff --git a/bookkeeper-common/pom.xml b/bookkeeper-common/pom.xml deleted file mode 100644 index e7709f4597a..00000000000 --- a/bookkeeper-common/pom.xml +++ /dev/null @@ -1,129 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-common - Apache BookKeeper :: Common - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.parent.version} - - - org.apache.bookkeeper - cpu-affinity - ${project.parent.version} - - - com.google.guava - guava - - - io.netty - netty-common - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-annotations - - - org.jctools - jctools-core - - - com.google.code.findbugs - jsr305 - provided - - - com.google.errorprone - error_prone_annotations - provided - - - org.apache.commons - commons-lang3 - test - - - org.slf4j - slf4j-api - ${slf4j.version} - - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - test - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - diff --git a/bookkeeper-dist/all/build.gradle b/bookkeeper-dist/all/build.gradle index 68f1043eff8..bd1e73565bd 100644 --- a/bookkeeper-dist/all/build.gradle +++ b/bookkeeper-dist/all/build.gradle @@ -60,7 +60,7 @@ def depLicences = [ "javax.servlet-api-4.0.0/CDDL+GPL-1.1", "bouncycastle-1.0.2.1/LICENSE.html", "jsr-305/LICENSE", - "netty-4.1.72.Final/*", + "netty-4.1.74.Final/*", "paranamer-2.8/LICENSE.txt", "protobuf-3.14.0/LICENSE", "protobuf-3.12.0/LICENSE", diff --git a/bookkeeper-dist/all/pom.xml b/bookkeeper-dist/all/pom.xml deleted file mode 100644 index 146dafa3271..00000000000 --- a/bookkeeper-dist/all/pom.xml +++ /dev/null @@ -1,159 +0,0 @@ - - - 4.0.0 - - bookkeeper-dist - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - - bookkeeper-dist-all - jar - Apache BookKeeper :: Dist (All) - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.version} - - - - org.apache.bookkeeper.stats - codahale-metrics-provider - ${project.version} - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.version} - - - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - org.apache.bookkeeper.http - vertx-http-server - ${project.version} - - - - - org.apache.bookkeeper - bookkeeper-tools - ${project.version} - - - - - org.apache.distributedlog - distributedlog-core - ${project.version} - - - - - - org.apache.bookkeeper - stream-storage-server - ${project.version} - - - - - org.apache.bookkeeper - bookkeeper-benchmark - ${project.version} - - - - - org.apache.logging.log4j - log4j-1.2-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-slf4j-impl - - - - - org.xerial.snappy - snappy-java - - - - io.dropwizard.metrics - metrics-core - - - - - - - maven-assembly-plugin - ${maven-assembly-plugin.version} - - bookkeeper-all-${project.version} - false - - ../src/assemble/bin-all.xml - - posix - - - - package - - single - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - true - - - - - diff --git a/bookkeeper-dist/bkctl/build.gradle b/bookkeeper-dist/bkctl/build.gradle index 2495b7b0148..27d27533eeb 100644 --- a/bookkeeper-dist/bkctl/build.gradle +++ b/bookkeeper-dist/bkctl/build.gradle @@ -49,7 +49,7 @@ releaseArtifacts { def depLicences = [ "checker-qual-3.5.0/LICENSE", "google-auth-library-credentials-0.20.0/LICENSE", - "netty-4.1.72.Final/*", + "netty-4.1.74.Final/*", "bouncycastle-1.0.2.1/LICENSE.html", "protobuf-3.14.0/LICENSE", "protobuf-3.12.0/LICENSE", diff --git a/bookkeeper-dist/bkctl/pom.xml b/bookkeeper-dist/bkctl/pom.xml deleted file mode 100644 index 8ac6545851e..00000000000 --- a/bookkeeper-dist/bkctl/pom.xml +++ /dev/null @@ -1,96 +0,0 @@ - - - 4.0.0 - - bookkeeper-dist - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - - bkctl - jar - Apache BookKeeper :: Dist (Bkctl) - - - - - org.apache.bookkeeper - bookkeeper-tools - ${project.version} - - - org.rocksdb - rocksdbjni - - - - - - - org.apache.logging.log4j - log4j-1.2-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-slf4j-impl - - - - - - - maven-assembly-plugin - ${maven-assembly-plugin.version} - - bkctl-${project.version} - true - - ../src/assemble/bkctl.xml - - posix - - - - package - - single - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - true - - - - - diff --git a/bookkeeper-dist/build.gradle b/bookkeeper-dist/build.gradle index e8c086821da..aeac682345a 100644 --- a/bookkeeper-dist/build.gradle +++ b/bookkeeper-dist/build.gradle @@ -38,7 +38,6 @@ distributions { "**/README.md", "**/bin/**", "**/conf/**", - "**/pom.xml", "**/src/**", "deploy/**", "doc/**", diff --git a/bookkeeper-dist/pom.xml b/bookkeeper-dist/pom.xml deleted file mode 100644 index 5285131dbcb..00000000000 --- a/bookkeeper-dist/pom.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - 4.0.0 - bookkeeper-dist - pom - Apache BookKeeper :: Dist (Parent) - - all - server - bkctl - - - UTF-8 - UTF-8 - - - - - maven-jar-plugin - ${maven-jar-plugin.version} - - - default-jar - none - - - - - maven-assembly-plugin - ${maven-assembly-plugin.version} - - bookkeeper-${project.version} - - src/assemble/src.xml - - posix - - - - package - - single - - - - - - - diff --git a/bookkeeper-dist/server/build.gradle b/bookkeeper-dist/server/build.gradle index ba2d50da268..796cc602656 100644 --- a/bookkeeper-dist/server/build.gradle +++ b/bookkeeper-dist/server/build.gradle @@ -53,7 +53,7 @@ def depLicences = [ "checker-qual-3.5.0/LICENSE", "google-auth-library-credentials-0.20.0/LICENSE", "javax.servlet-api-4.0.0/CDDL+GPL-1.1", - "netty-4.1.72.Final/*", + "netty-4.1.74.Final/*", "bouncycastle-1.0.2.1/LICENSE.html", "protobuf-3.14.0/LICENSE", "protobuf-3.12.0/LICENSE", diff --git a/bookkeeper-dist/server/pom.xml b/bookkeeper-dist/server/pom.xml deleted file mode 100644 index 83777f5f342..00000000000 --- a/bookkeeper-dist/server/pom.xml +++ /dev/null @@ -1,146 +0,0 @@ - - - 4.0.0 - - bookkeeper-dist - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - - bookkeeper-dist-server - jar - Apache BookKeeper :: Dist (Server) - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.version} - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.version} - - - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - org.apache.bookkeeper.http - vertx-http-server - ${project.version} - - - - - org.apache.bookkeeper - stream-storage-server - ${project.version} - - - - - org.apache.bookkeeper - bookkeeper-tools - ${project.version} - - - - - org.apache.distributedlog - distributedlog-core - ${project.version} - - - - - org.apache.logging.log4j - log4j-1.2-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-slf4j-impl - - - - - org.xerial.snappy - snappy-java - - - - io.dropwizard.metrics - metrics-core - - - - - - - maven-assembly-plugin - ${maven-assembly-plugin.version} - - bookkeeper-server-${project.version} - true - - ../src/assemble/bin-server.xml - - posix - - - - package - - single - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - true - - - - - - diff --git a/bookkeeper-dist/src/assemble/bin-all.xml b/bookkeeper-dist/src/assemble/bin-all.xml index 4ec9dea8221..1bf4fb87978 100644 --- a/bookkeeper-dist/src/assemble/bin-all.xml +++ b/bookkeeper-dist/src/assemble/bin-all.xml @@ -59,7 +59,7 @@ javax.servlet-api-4.0.0/CDDL+GPL-1.1 bouncycastle-1.0.2.1/LICENSE.html jsr-305/LICENSE - netty-4.1.72.Final/* + netty-4.1.74.Final/* paranamer-2.8/LICENSE.txt protobuf-3.14.0/LICENSE protobuf-3.12.0/LICENSE diff --git a/bookkeeper-dist/src/assemble/bin-server.xml b/bookkeeper-dist/src/assemble/bin-server.xml index cfb176d3d41..c3d9955c0e9 100644 --- a/bookkeeper-dist/src/assemble/bin-server.xml +++ b/bookkeeper-dist/src/assemble/bin-server.xml @@ -52,7 +52,7 @@ checker-qual-3.5.0/LICENSE google-auth-library-credentials-0.20.0/LICENSE javax.servlet-api-4.0.0/CDDL+GPL-1.1 - netty-4.1.72.Final/* + netty-4.1.74.Final/* bouncycastle-1.0.2.1/LICENSE.html protobuf-3.14.0/LICENSE protobuf-3.12.0/LICENSE diff --git a/bookkeeper-dist/src/assemble/bkctl.xml b/bookkeeper-dist/src/assemble/bkctl.xml index 855f410d962..cb4ea8f3a3b 100644 --- a/bookkeeper-dist/src/assemble/bkctl.xml +++ b/bookkeeper-dist/src/assemble/bkctl.xml @@ -66,7 +66,7 @@ checker-qual-3.5.0/LICENSE google-auth-library-credentials-0.20.0/LICENSE - netty-4.1.72.Final/* + netty-4.1.74.Final/* bouncycastle-1.0.2.1/LICENSE.html protobuf-3.14.0/LICENSE protobuf-3.12.0/LICENSE diff --git a/bookkeeper-dist/src/assemble/src.xml b/bookkeeper-dist/src/assemble/src.xml index 146061e28a4..0301c8a9bde 100644 --- a/bookkeeper-dist/src/assemble/src.xml +++ b/bookkeeper-dist/src/assemble/src.xml @@ -30,7 +30,6 @@ **/README.md **/LICENSE **/NOTICE - **/pom.xml **/*gradle* **/src/** **/conf/** diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt index 2721b5b2886..27a4f3c857d 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt @@ -208,7 +208,7 @@ Apache Software License, Version 2. - lib/com.fasterxml.jackson.core-jackson-annotations-2.11.0.jar [1] - lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] - lib/com.fasterxml.jackson.core-jackson-databind-2.11.0.jar [3] -- lib/com.google.guava-guava-30.0-jre.jar [4] +- lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] - lib/commons-cli-commons-cli-1.2.jar [5] @@ -217,23 +217,23 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar [11] -- lib/io.netty-netty-common-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.72.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar [11] -- lib/io.netty-netty-transport-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.74.Final.jar [11] +- lib/io.netty-netty-common-4.1.74.Final.jar [11] +- lib/io.netty-netty-handler-4.1.74.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.74.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.74.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.74.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.48.Final.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.48.Final.jar [11] +- lib/io.netty-netty-transport-4.1.74.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.74.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.74.Final.jar [11] - lib/io.prometheus-simpleclient-0.8.1.jar [12] - lib/io.prometheus-simpleclient_common-0.8.1.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.8.1.jar [12] @@ -260,7 +260,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-servlet-9.4.43.v20210629.jar [22] - lib/org.eclipse.jetty-jetty-util-9.4.43.v20210629.jar [22] - lib/org.eclipse.jetty-jetty-util-ajax-9.4.43.v20210629.jar [22] -- lib/org.rocksdb-rocksdbjni-6.22.1.1.jar [23] +- lib/org.rocksdb-rocksdbjni-6.27.3.jar [23] - lib/com.beust-jcommander-1.78.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] @@ -312,14 +312,14 @@ Apache Software License, Version 2. [1] Source available at https://github.com/FasterXML/jackson-annotations/tree/jackson-annotations-2.11.0 [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.11.3 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.11.0 -[4] Source available at https://github.com/google/guava/tree/v30.0 +[4] Source available at https://github.com/google/guava/tree/v31.0 [5] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-cli.git;a=tag;h=bc8f0e [6] Source available at http://svn.apache.org/viewvc/commons/proper/codec/tags/1_6/ [7] Source available at http://svn.apache.org/viewvc/commons/proper/configuration/tags/CONFIGURATION_1_10/ [8] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-io.git;a=tag;h=603579 [9] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=tag;h=375459 [10] Source available at http://svn.apache.org/viewvc/commons/proper/logging/tags/commons-logging-1.1.1/ -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.72.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.74.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.8.1 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/3.9.8 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/3.9.8 @@ -359,234 +359,234 @@ Apache Software License, Version 2. [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.72.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.74.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.72.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.74.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jsr166y.txt (Public Domain) + * deps/netty-4.1.74.Final/LICENSE.jsr166y.txt (Public Domain) * HOMEPAGE: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.base64.txt (Public Domain) + * deps/netty-4.1.74.Final/LICENSE.base64.txt (Public Domain) * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.webbit.txt (BSD License) + * deps/netty-4.1.74.Final/LICENSE.webbit.txt (BSD License) * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.slf4j.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.slf4j.txt (MIT License) * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: - * deps/netty-4.1.72.Final/NOTICE.harmony.txt + * deps/netty-4.1.74.Final/NOTICE.harmony.txt * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.harmony.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.harmony.txt (Apache License 2.0) * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jbzip2.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.jbzip2.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.libdivsufsort.txt (MIT License) * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jctools.txt (ASL2 License) + * deps/netty-4.1.74.Final/LICENSE.jctools.txt (ASL2 License) * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jzlib.txt (BSD style License) + * deps/netty-4.1.74.Final/LICENSE.jzlib.txt (BSD style License) * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.compress-lzf.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lz4.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lzma-java.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.lzma-java.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jfastlz.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.jfastlz.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.protobuf.txt (New BSD License) + * deps/netty-4.1.74.Final/LICENSE.protobuf.txt (New BSD License) * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.bouncycastle.txt (MIT License) * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.snappy.txt (New BSD License) + * deps/netty-4.1.74.Final/LICENSE.snappy.txt (New BSD License) * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.caliper.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.caliper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-logging.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.commons-logging.txt (Apache License 2.0) * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.log4j.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.log4j.txt (Apache License 2.0) * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.aalto-xml.txt (Apache License 2.0) * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hpack.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.hpack.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.hyper-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.nghttp2-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-lang.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.commons-lang.txt (Apache License 2.0) * HOMEPAGE: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.74.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.72.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.74.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.dnsinfo.txt (Apache License 2.0) * HOMEPAGE: * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h ------------------------------------------------------------------------------------ -lib/org.rocksdb-rocksdbjni-6.22.1.1.jar is derived from leveldb, which is under the following license. +lib/org.rocksdb-rocksdbjni-6.27.3.jar is derived from leveldb, which is under the following license. Copyright (c) 2011 The LevelDB Authors. All rights reserved. @@ -667,10 +667,10 @@ This product uses the annotations from The Checker Framework, which are licensed MIT License. For details, see deps/checker-qual-3.5.0/LICENSE Bundles as - - lib/org.checkerframework-checker-qual-3.5.0.jar + - lib/org.checkerframework-checker-qual-3.12.0.jar ------------------------------------------------------------------------------------ This product bundles the Reactive Streams library, which is licensed under Public Domain (CC0). For details, see deps/reactivestreams-1.0.3/LICENSE Bundles as - - lib/org.reactivestreams-reactive-streams-1.0.3.jar \ No newline at end of file + - lib/org.reactivestreams-reactive-streams-1.0.3.jar diff --git a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt index 5d5e822d4f2..4ed5c49cc0b 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-bkctl.bin.txt @@ -208,7 +208,7 @@ Apache Software License, Version 2. - lib/com.fasterxml.jackson.core-jackson-annotations-2.11.0.jar [1] - lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] - lib/com.fasterxml.jackson.core-jackson-databind-2.11.0.jar [3] -- lib/com.google.guava-guava-30.0-jre.jar [4] +- lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] - lib/commons-cli-commons-cli-1.2.jar [5] @@ -217,21 +217,21 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar [11] -- lib/io.netty-netty-common-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.72.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar [11] -- lib/io.netty-netty-transport-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.74.Final.jar [11] +- lib/io.netty-netty-common-4.1.74.Final.jar [11] +- lib/io.netty-netty-handler-4.1.74.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.74.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.74.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.48.Final.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.48.Final.jar [11] +- lib/io.netty-netty-transport-4.1.74.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.74.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.74.Final.jar [11] - lib/org.apache.logging.log4j-log4j-1.2-api-2.17.1.jar [16] - lib/org.apache.logging.log4j-log4j-api-2.17.1.jar [16] - lib/org.apache.logging.log4j-log4j-core-2.17.1.jar [16] @@ -289,14 +289,14 @@ Apache Software License, Version 2. [1] Source available at https://github.com/FasterXML/jackson-annotations/tree/jackson-annotations-2.11.0 [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.11.3 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.11.0 -[4] Source available at https://github.com/google/guava/tree/v30.0 +[4] Source available at https://github.com/google/guava/tree/v31.0 [5] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-cli.git;a=tag;h=bc8f0e [6] Source available at http://svn.apache.org/viewvc/commons/proper/codec/tags/1_6/ [7] Source available at http://svn.apache.org/viewvc/commons/proper/configuration/tags/CONFIGURATION_1_10/ [8] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-io.git;a=tag;h=603579 [9] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=tag;h=375459 [10] Source available at http://svn.apache.org/viewvc/commons/proper/logging/tags/commons-logging-1.1.1/ -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.72.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.74.Final [16] Source available at https://github.com/apache/logging-log4j2/tree/rel/2.17.1 [17] Source available at https://github.com/java-native-access/jna/tree/3.2.7 [18] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-collections.git;a=tag;h=a3a5ad @@ -328,229 +328,229 @@ Apache Software License, Version 2. [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.72.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.74.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.72.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.74.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jsr166y.txt (Public Domain) + * deps/netty-4.1.74.Final/LICENSE.jsr166y.txt (Public Domain) * HOMEPAGE: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.base64.txt (Public Domain) + * deps/netty-4.1.74.Final/LICENSE.base64.txt (Public Domain) * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.webbit.txt (BSD License) + * deps/netty-4.1.74.Final/LICENSE.webbit.txt (BSD License) * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.slf4j.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.slf4j.txt (MIT License) * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: - * deps/netty-4.1.72.Final/NOTICE.harmony.txt + * deps/netty-4.1.74.Final/NOTICE.harmony.txt * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.harmony.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.harmony.txt (Apache License 2.0) * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jbzip2.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.jbzip2.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.libdivsufsort.txt (MIT License) * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jctools.txt (ASL2 License) + * deps/netty-4.1.74.Final/LICENSE.jctools.txt (ASL2 License) * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jzlib.txt (BSD style License) + * deps/netty-4.1.74.Final/LICENSE.jzlib.txt (BSD style License) * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.compress-lzf.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lz4.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lzma-java.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.lzma-java.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jfastlz.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.jfastlz.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.protobuf.txt (New BSD License) + * deps/netty-4.1.74.Final/LICENSE.protobuf.txt (New BSD License) * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.bouncycastle.txt (MIT License) * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.snappy.txt (New BSD License) + * deps/netty-4.1.74.Final/LICENSE.snappy.txt (New BSD License) * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.caliper.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.caliper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-logging.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.commons-logging.txt (Apache License 2.0) * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.log4j.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.log4j.txt (Apache License 2.0) * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.aalto-xml.txt (Apache License 2.0) * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hpack.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.hpack.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.hyper-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.nghttp2-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-lang.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.commons-lang.txt (Apache License 2.0) * HOMEPAGE: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.74.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.72.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.74.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.dnsinfo.txt (Apache License 2.0) * HOMEPAGE: * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h @@ -594,10 +594,10 @@ This product uses the annotations from The Checker Framework, which are licensed MIT License. For details, see deps/checker-qual-3.5.0/LICENSE Bundles as - - lib/org.checkerframework-checker-qual-3.5.0.jar + - lib/org.checkerframework-checker-qual-3.12.0.jar ------------------------------------------------------------------------------------ This product bundles the Reactive Streams library, which is licensed under Public Domain (CC0). For details, see deps/reactivestreams-1.0.3/LICENSE Bundles as - - lib/org.reactivestreams-reactive-streams-1.0.3.jar \ No newline at end of file + - lib/org.reactivestreams-reactive-streams-1.0.3.jar diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt index 23f1c9a265b..aeb9fe2eed7 100644 --- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt @@ -208,7 +208,7 @@ Apache Software License, Version 2. - lib/com.fasterxml.jackson.core-jackson-annotations-2.11.0.jar [1] - lib/com.fasterxml.jackson.core-jackson-core-2.11.3.jar [2] - lib/com.fasterxml.jackson.core-jackson-databind-2.11.0.jar [3] -- lib/com.google.guava-guava-30.0-jre.jar [4] +- lib/com.google.guava-guava-31.0.1-jre.jar [4] - lib/com.google.guava-failureaccess-1.0.1.jar [4] - lib/com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar [4] - lib/commons-cli-commons-cli-1.2.jar [5] @@ -217,23 +217,23 @@ Apache Software License, Version 2. - lib/commons-io-commons-io-2.7.jar [8] - lib/commons-lang-commons-lang-2.6.jar [9] - lib/commons-logging-commons-logging-1.1.1.jar [10] -- lib/io.netty-netty-buffer-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-dns-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar [11] -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar [11] -- lib/io.netty-netty-common-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-4.1.72.Final.jar [11] -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-4.1.72.Final.jar [11] -- lib/io.netty-netty-resolver-dns-4.1.72.Final.jar [11] -- lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar [11] -- lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar [11] -- lib/io.netty-netty-transport-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar [11] -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar [11] -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar [11] +- lib/io.netty-netty-buffer-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-dns-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-http-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-http2-4.1.74.Final.jar [11] +- lib/io.netty-netty-codec-socks-4.1.74.Final.jar [11] +- lib/io.netty-netty-common-4.1.74.Final.jar [11] +- lib/io.netty-netty-handler-4.1.74.Final.jar [11] +- lib/io.netty-netty-handler-proxy-4.1.74.Final.jar [11] +- lib/io.netty-netty-resolver-4.1.74.Final.jar [11] +- lib/io.netty-netty-resolver-dns-4.1.74.Final.jar [11] +- lib/io.netty-netty-tcnative-boringssl-static-2.0.48.Final.jar [11] +- lib/io.netty-netty-tcnative-classes-2.0.48.Final.jar [11] +- lib/io.netty-netty-transport-4.1.74.Final.jar [11] +- lib/io.netty-netty-transport-classes-epoll-4.1.74.Final.jar [11] +- lib/io.netty-netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar [11] +- lib/io.netty-netty-transport-native-unix-common-4.1.74.Final.jar [11] - lib/io.prometheus-simpleclient-0.8.1.jar [12] - lib/io.prometheus-simpleclient_common-0.8.1.jar [12] - lib/io.prometheus-simpleclient_hotspot-0.8.1.jar [12] @@ -260,7 +260,7 @@ Apache Software License, Version 2. - lib/org.eclipse.jetty-jetty-servlet-9.4.43.v20210629.jar [22] - lib/org.eclipse.jetty-jetty-util-9.4.43.v20210629.jar [22] - lib/org.eclipse.jetty-jetty-util-ajax-9.4.43.v20210629.jar [22] -- lib/org.rocksdb-rocksdbjni-6.22.1.1.jar [23] +- lib/org.rocksdb-rocksdbjni-6.27.3.jar [23] - lib/com.beust-jcommander-1.78.jar [24] - lib/com.yahoo.datasketches-memory-0.8.3.jar [25] - lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [25] @@ -310,14 +310,14 @@ Apache Software License, Version 2. [1] Source available at https://github.com/FasterXML/jackson-annotations/tree/jackson-annotations-2.11.0 [2] Source available at https://github.com/FasterXML/jackson-core/tree/jackson-core-2.11.3 [3] Source available at https://github.com/FasterXML/jackson-databind/tree/jackson-databind-2.11.0 -[4] Source available at https://github.com/google/guava/tree/v30.0 +[4] Source available at https://github.com/google/guava/tree/v31.0 [5] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-cli.git;a=tag;h=bc8f0e [6] Source available at http://svn.apache.org/viewvc/commons/proper/codec/tags/1_6/ [7] Source available at http://svn.apache.org/viewvc/commons/proper/configuration/tags/CONFIGURATION_1_10/ [8] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-io.git;a=tag;h=603579 [9] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=tag;h=375459 [10] Source available at http://svn.apache.org/viewvc/commons/proper/logging/tags/commons-logging-1.1.1/ -[11] Source available at https://github.com/netty/netty/tree/netty-4.1.72.Final +[11] Source available at https://github.com/netty/netty/tree/netty-4.1.74.Final [12] Source available at https://github.com/prometheus/client_java/tree/parent-0.8.1 [13] Source available at https://github.com/vert-x3/vertx-auth/tree/3.9.8 [14] Source available at https://github.com/vert-x3/vertx-bridge-common/tree/3.9.8 @@ -357,234 +357,234 @@ Apache Software License, Version 2. [51] Source available at https://github.com/ReactiveX/RxJava/tree/v3.0.1 ------------------------------------------------------------------------------------ -lib/io.netty-netty-codec-4.1.72.Final.jar bundles some 3rd party dependencies +lib/io.netty-netty-codec-4.1.74.Final.jar bundles some 3rd party dependencies -lib/io.netty-netty-codec-4.1.72.Final.jar contains the extensions to Java Collections Framework which has +lib/io.netty-netty-codec-4.1.74.Final.jar contains the extensions to Java Collections Framework which has been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jsr166y.txt (Public Domain) + * deps/netty-4.1.74.Final/LICENSE.jsr166y.txt (Public Domain) * HOMEPAGE: * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of Robert Harder's Public Domain +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified version of Robert Harder's Public Domain Base64 Encoder and Decoder, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.base64.txt (Public Domain) + * deps/netty-4.1.74.Final/LICENSE.base64.txt (Public Domain) * HOMEPAGE: * http://iharder.sourceforge.net/current/java/base64/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Webbit', an event based +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'Webbit', an event based WebSocket and HTTP server, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.webbit.txt (BSD License) + * deps/netty-4.1.74.Final/LICENSE.webbit.txt (BSD License) * HOMEPAGE: * https://github.com/joewalnes/webbit -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'SLF4J', a simple logging +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'SLF4J', a simple logging facade for Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.slf4j.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.slf4j.txt (MIT License) * HOMEPAGE: * http://www.slf4j.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Harmony', an open source +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: * NOTICE: - * deps/netty-4.1.72.Final/NOTICE.harmony.txt + * deps/netty-4.1.74.Final/NOTICE.harmony.txt * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.harmony.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.harmony.txt (Apache License 2.0) * HOMEPAGE: * http://archive.apache.org/dist/harmony/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jbzip2.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.jbzip2.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jbzip2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'libdivsufsort', a C API library to construct the suffix array and the Burrows-Wheeler transformed string for any input string of a constant-size alphabet written by Yuta Mori. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.libdivsufsort.txt (MIT License) * HOMEPAGE: * https://github.com/y-256/libdivsufsort -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jctools.txt (ASL2 License) + * deps/netty-4.1.74.Final/LICENSE.jctools.txt (ASL2 License) * HOMEPAGE: * https://github.com/JCTools/JCTools -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'JZlib', a re-implementation of zlib in pure Java, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jzlib.txt (BSD style License) + * deps/netty-4.1.74.Final/LICENSE.jzlib.txt (BSD style License) * HOMEPAGE: * http://www.jcraft.com/jzlib/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Compress-LZF', a Java library for encoding and decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.compress-lzf.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/ning/compress -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lz4', a LZ4 Java compression +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'lz4', a LZ4 Java compression and decompression library written by Adrien Grand. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lz4.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.lz4.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jpountz/lz4-java -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'lzma-java', a LZMA Java compression +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'lzma-java', a LZMA Java compression and decompression library, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.lzma-java.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.lzma-java.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jponge/lzma-java -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jfastlz.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.jfastlz.txt (MIT License) * HOMEPAGE: * https://code.google.com/p/jfastlz/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data interchange format, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.protobuf.txt (New BSD License) + * deps/netty-4.1.74.Final/LICENSE.protobuf.txt (New BSD License) * HOMEPAGE: * https://github.com/google/protobuf -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Bouncy Castle Crypto APIs' to generate a temporary self-signed X.509 certificate when the JVM does not provide the equivalent functionality. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.bouncycastle.txt (MIT License) * HOMEPAGE: * http://www.bouncycastle.org/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Snappy', a compression library produced +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.snappy.txt (New BSD License) + * deps/netty-4.1.74.Final/LICENSE.snappy.txt (New BSD License) * HOMEPAGE: * https://github.com/google/snappy -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.jboss-marshalling.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/jboss-remoting/jboss-marshalling -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Caliper', Google's micro- +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.caliper.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.caliper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/google/caliper -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Commons Logging', a logging +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Apache Commons Logging', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-logging.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.commons-logging.txt (Apache License 2.0) * HOMEPAGE: * http://commons.apache.org/logging/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Apache Log4J', a logging framework, which +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.log4j.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.log4j.txt (Apache License 2.0) * HOMEPAGE: * http://logging.apache.org/log4j/ -lib/io.netty-netty-codec-4.1.72.Final.jar optionally depends on 'Aalto XML', an ultra-high performance +lib/io.netty-netty-codec-4.1.74.Final.jar optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.aalto-xml.txt (Apache License 2.0) * HOMEPAGE: * http://wiki.fasterxml.com/AaltoHome -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hpack.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.hpack.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/twitter/hpack -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.hyper-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/python-hyper/hpack/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified version of 'HPACK', a Java implementation of +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt (MIT License) + * deps/netty-4.1.74.Final/LICENSE.nghttp2-hpack.txt (MIT License) * HOMEPAGE: * https://github.com/nghttp2/nghttp2/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library +lib/io.netty-netty-codec-4.1.74.Final.jar contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.commons-lang.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.commons-lang.txt (Apache License 2.0) * HOMEPAGE: * https://commons.apache.org/proper/commons-lang/ -lib/io.netty-netty-codec-4.1.72.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', +lib/io.netty-netty-codec-4.1.74.Final.jar contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.mvn-wrapper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/takari/maven-wrapper -lib/io.netty-netty-codec-4.1.72.Final.jar contains the dnsinfo.h header file, +lib/io.netty-netty-codec-4.1.74.Final.jar contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. This private header is also used by Apple's open source mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). * LICENSE: - * deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt (Apache License 2.0) + * deps/netty-4.1.74.Final/LICENSE.dnsinfo.txt (Apache License 2.0) * HOMEPAGE: * http://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h ------------------------------------------------------------------------------------ -lib/org.rocksdb-rocksdbjni-6.22.1.1.jar is derived from leveldb, which is under the following license. +lib/org.rocksdb-rocksdbjni-6.27.3.jar is derived from leveldb, which is under the following license. Copyright (c) 2011 The LevelDB Authors. All rights reserved. @@ -659,10 +659,10 @@ This product uses the annotations from The Checker Framework, which are licensed MIT License. For details, see deps/checker-qual-3.5.0/LICENSE Bundles as - - lib/org.checkerframework-checker-qual-3.5.0.jar + - lib/org.checkerframework-checker-qual-3.12.0.jar ------------------------------------------------------------------------------------ This product bundles the Reactive Streams library, which is licensed under Public Domain (CC0). For details, see deps/reactivestreams-1.0.3/LICENSE Bundles as - - lib/org.reactivestreams-reactive-streams-1.0.3.jar \ No newline at end of file + - lib/org.reactivestreams-reactive-streams-1.0.3.jar diff --git a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt index 0bff137ce70..11d955a3504 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-all.bin.txt @@ -22,23 +22,23 @@ LongAdder), which was released with the following comments: http://creativecommons.org/publicdomain/zero/1.0/ ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.72.Final.jar -- lib/io.netty-netty-codec-4.1.72.Final.jar -- lib/io.netty-netty-codec-dns-4.1.72.Final.jar -- lib/io.netty-netty-codec-http-4.1.72.Final.jar -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar -- lib/io.netty-netty-common-4.1.72.Final.jar -- lib/io.netty-netty-handler-4.1.72.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar -- lib/io.netty-netty-resolver-4.1.72.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.72.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar -- lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar -- lib/io.netty-netty-transport-4.1.72.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar +- lib/io.netty-netty-buffer-4.1.74.Final.jar +- lib/io.netty-netty-codec-4.1.74.Final.jar +- lib/io.netty-netty-codec-dns-4.1.74.Final.jar +- lib/io.netty-netty-codec-http-4.1.74.Final.jar +- lib/io.netty-netty-codec-http2-4.1.74.Final.jar +- lib/io.netty-netty-codec-socks-4.1.74.Final.jar +- lib/io.netty-netty-common-4.1.74.Final.jar +- lib/io.netty-netty-handler-4.1.74.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.74.Final.jar +- lib/io.netty-netty-resolver-4.1.74.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.74.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.48.Final.jar +- lib/io.netty-netty-tcnative-classes-2.0.48.Final.jar +- lib/io.netty-netty-transport-4.1.74.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.74.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.74.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt index 8ee2bd437e0..efec04f7392 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-bkctl.bin.txt @@ -5,21 +5,21 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.72.Final.jar -- lib/io.netty-netty-codec-4.1.72.Final.jar -- lib/io.netty-netty-codec-http-4.1.72.Final.jar -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar -- lib/io.netty-netty-common-4.1.72.Final.jar -- lib/io.netty-netty-handler-4.1.72.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar -- lib/io.netty-netty-resolver-4.1.72.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar -- lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar -- lib/io.netty-netty-transport-4.1.72.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar +- lib/io.netty-netty-buffer-4.1.74.Final.jar +- lib/io.netty-netty-codec-4.1.74.Final.jar +- lib/io.netty-netty-codec-http-4.1.74.Final.jar +- lib/io.netty-netty-codec-http2-4.1.74.Final.jar +- lib/io.netty-netty-codec-socks-4.1.74.Final.jar +- lib/io.netty-netty-common-4.1.74.Final.jar +- lib/io.netty-netty-handler-4.1.74.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.74.Final.jar +- lib/io.netty-netty-resolver-4.1.74.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.48.Final.jar +- lib/io.netty-netty-tcnative-classes-2.0.48.Final.jar +- lib/io.netty-netty-transport-4.1.74.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.74.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.74.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt index a991cd6ba09..1e812083806 100644 --- a/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt +++ b/bookkeeper-dist/src/main/resources/NOTICE-server.bin.txt @@ -5,23 +5,23 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ------------------------------------------------------------------------------------ -- lib/io.netty-netty-buffer-4.1.72.Final.jar -- lib/io.netty-netty-codec-4.1.72.Final.jar -- lib/io.netty-netty-codec-dns-4.1.72.Final.jar -- lib/io.netty-netty-codec-http-4.1.72.Final.jar -- lib/io.netty-netty-codec-http2-4.1.72.Final.jar -- lib/io.netty-netty-codec-socks-4.1.72.Final.jar -- lib/io.netty-netty-common-4.1.72.Final.jar -- lib/io.netty-netty-handler-4.1.72.Final.jar -- lib/io.netty-netty-handler-proxy-4.1.72.Final.jar -- lib/io.netty-netty-resolver-4.1.72.Final.jar -- lib/io.netty-netty-resolver-dns-4.1.72.Final.jar -- lib/io.netty-netty-tcnative-boringssl-static-2.0.46.Final.jar -- lib/io.netty-netty-tcnative-classes-2.0.46.Final.jar -- lib/io.netty-netty-transport-4.1.72.Final.jar -- lib/io.netty-netty-transport-classes-epoll-4.1.72.Final.jar -- lib/io.netty-netty-transport-native-epoll-4.1.72.Final-linux-x86_64.jar -- lib/io.netty-netty-transport-native-unix-common-4.1.72.Final.jar +- lib/io.netty-netty-buffer-4.1.74.Final.jar +- lib/io.netty-netty-codec-4.1.74.Final.jar +- lib/io.netty-netty-codec-dns-4.1.74.Final.jar +- lib/io.netty-netty-codec-http-4.1.74.Final.jar +- lib/io.netty-netty-codec-http2-4.1.74.Final.jar +- lib/io.netty-netty-codec-socks-4.1.74.Final.jar +- lib/io.netty-netty-common-4.1.74.Final.jar +- lib/io.netty-netty-handler-4.1.74.Final.jar +- lib/io.netty-netty-handler-proxy-4.1.74.Final.jar +- lib/io.netty-netty-resolver-4.1.74.Final.jar +- lib/io.netty-netty-resolver-dns-4.1.74.Final.jar +- lib/io.netty-netty-tcnative-boringssl-static-2.0.48.Final.jar +- lib/io.netty-netty-tcnative-classes-2.0.48.Final.jar +- lib/io.netty-netty-transport-4.1.74.Final.jar +- lib/io.netty-netty-transport-classes-epoll-4.1.74.Final.jar +- lib/io.netty-netty-transport-native-epoll-4.1.74.Final-linux-x86_64.jar +- lib/io.netty-netty-transport-native-unix-common-4.1.74.Final.jar The Netty Project diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.aalto-xml.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.aalto-xml.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.aalto-xml.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.base64.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.base64.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.base64.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.base64.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.bouncycastle.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.bouncycastle.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.bouncycastle.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.caliper.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.caliper.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.caliper.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.caliper.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.commons-lang.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.commons-lang.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.commons-lang.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.commons-lang.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.commons-logging.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.commons-logging.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.commons-logging.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.commons-logging.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.compress-lzf.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.compress-lzf.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.compress-lzf.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.dnsinfo.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.dnsinfo.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.dnsinfo.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.harmony.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.harmony.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.harmony.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.harmony.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.hpack.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.hpack.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.hpack.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.hpack.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.hyper-hpack.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.hyper-hpack.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.hyper-hpack.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.jboss-marshalling.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jboss-marshalling.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.jboss-marshalling.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jbzip2.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.jbzip2.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jbzip2.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.jbzip2.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jctools.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.jctools.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jctools.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.jctools.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jfastlz.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.jfastlz.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jfastlz.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.jfastlz.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jsr166y.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.jsr166y.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jsr166y.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.jsr166y.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jzlib.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.jzlib.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.jzlib.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.jzlib.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.libdivsufsort.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.libdivsufsort.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.libdivsufsort.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.log4j.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.log4j.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.log4j.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.log4j.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.lz4.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.lz4.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.lz4.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.lz4.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.lzma-java.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.lzma-java.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.lzma-java.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.lzma-java.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.mvn-wrapper.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.mvn-wrapper.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.mvn-wrapper.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.nghttp2-hpack.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.nghttp2-hpack.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.nghttp2-hpack.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.protobuf.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.protobuf.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.protobuf.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.protobuf.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.slf4j.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.slf4j.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.slf4j.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.slf4j.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.snappy.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.snappy.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.snappy.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.snappy.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.webbit.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.webbit.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/LICENSE.webbit.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/LICENSE.webbit.txt diff --git a/bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/NOTICE.harmony.txt b/bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/NOTICE.harmony.txt similarity index 100% rename from bookkeeper-dist/src/main/resources/deps/netty-4.1.72.Final/NOTICE.harmony.txt rename to bookkeeper-dist/src/main/resources/deps/netty-4.1.74.Final/NOTICE.harmony.txt diff --git a/bookkeeper-http/http-server/pom.xml b/bookkeeper-http/http-server/pom.xml deleted file mode 100644 index 044a5c02850..00000000000 --- a/bookkeeper-http/http-server/pom.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - 4.0.0 - org.apache.bookkeeper.http - http-server - Apache BookKeeper :: Http :: Http Server - http://maven.apache.org - - - commons-configuration - commons-configuration - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-annotations - - - diff --git a/bookkeeper-http/pom.xml b/bookkeeper-http/pom.xml deleted file mode 100644 index bb9307322bf..00000000000 --- a/bookkeeper-http/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.http - bookkeeper-http - pom - Apache BookKeeper :: Http - - http-server - vertx-http-server - servlet-http-server - - diff --git a/bookkeeper-http/servlet-http-server/pom.xml b/bookkeeper-http/servlet-http-server/pom.xml deleted file mode 100644 index 2efa234a518..00000000000 --- a/bookkeeper-http/servlet-http-server/pom.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - 4.0.0 - org.apache.bookkeeper.http - servlet-http-server - Apache BookKeeper :: Bookkeeper Http :: Servlet Http Server - http://maven.apache.org - - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - javax.servlet - javax.servlet-api - - - commons-io - commons-io - - - org.eclipse.jetty - jetty-server - test - - - org.eclipse.jetty - jetty-webapp - test - - - - diff --git a/bookkeeper-http/vertx-http-server/pom.xml b/bookkeeper-http/vertx-http-server/pom.xml deleted file mode 100644 index c8914241361..00000000000 --- a/bookkeeper-http/vertx-http-server/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - 4.0.0 - org.apache.bookkeeper.http - vertx-http-server - Apache BookKeeper :: Bookkeeper Http :: Vertx Http Server - http://maven.apache.org - - - io.vertx - vertx-core - - - io.vertx - vertx-web - - - com.google.guava - guava - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - diff --git a/bookkeeper-proto/pom.xml b/bookkeeper-proto/pom.xml deleted file mode 100644 index 0894fd064ec..00000000000 --- a/bookkeeper-proto/pom.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-proto - Apache BookKeeper :: Protocols - - - com.google.protobuf - protobuf-java - - - - - - org.apache.rat - apache-rat-plugin - - - - **/DataFormats.java - **/BookkeeperProtocol.java - **/DbLedgerStorageDataFormats.java - **/.checkstyle - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - true - - - - - compile - - - - - - - diff --git a/bookkeeper-server/build.gradle b/bookkeeper-server/build.gradle index 5b0cd26ba5d..53187915aba 100644 --- a/bookkeeper-server/build.gradle +++ b/bookkeeper-server/build.gradle @@ -28,9 +28,9 @@ dependencies { implementation project(':bookkeeper-http:http-server') implementation project(':bookkeeper-proto') implementation project(':bookkeeper-stats') + implementation project(':bookkeeper-tools-framework') implementation project(':circe-checksum') implementation project(':cpu-affinity') - implementation project(':tools:framework') compileOnly depLibs.lombok compileOnly depLibs.spotbugsAnnotations @@ -80,11 +80,6 @@ dependencies { testImplementation depLibs.log4jCore } -task writeClasspath { - buildDir.mkdirs() - new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" -} - test { retry { maxFailures = 200 @@ -101,3 +96,6 @@ test.doFirst { jvmArgs("-Djunit.timeout.test=600000", "-Djunit.max.retry=3", "-Djava.net.preferIPv4Stack=true", "-Dio.netty.leakDetection.level=paranoid") } +jar { + dependsOn tasks.named("writeClasspath") +} diff --git a/bookkeeper-server/pom.xml b/bookkeeper-server/pom.xml deleted file mode 100644 index 469aab161d9..00000000000 --- a/bookkeeper-server/pom.xml +++ /dev/null @@ -1,336 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-server - Apache BookKeeper :: Server - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.bookkeeper - bookkeeper-common-allocator - ${project.parent.version} - - - org.apache.bookkeeper - bookkeeper-proto - ${project.parent.version} - - - org.apache.bookkeeper - bookkeeper-tools-framework - ${project.parent.version} - - - org.rocksdb - rocksdbjni - - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - test - - - org.apache.zookeeper - zookeeper - - - io.netty - netty-handler - - - io.netty - netty-transport-native-epoll - linux-x86_64 - - - io.netty - netty-tcnative-boringssl-static - - - org.apache.bookkeeper.http - http-server - ${project.version} - - - org.apache.bookkeeper - circe-checksum - ${project.version} - - - commons-cli - commons-cli - - - commons-codec - commons-codec - - - commons-io - commons-io - - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-collections4 - - - org.bouncycastle - bc-fips - - - com.beust - jcommander - - - net.java.dev.jna - jna - - - org.apache.httpcomponents - httpclient - - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - test-jar - test - - - org.apache.kerby - kerby-config - ${kerby.version} - test - - - org.slf4j - * - - - - - org.apache.kerby - kerb-simplekdc - ${kerby.version} - test - - - org.slf4j - * - - - - - org.apache.zookeeper - zookeeper - test-jar - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - test - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.parent.version} - test - - - org.apache.bookkeeper.http - vertx-http-server - ${project.parent.version} - test - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - org.apache.rat - apache-rat-plugin - - - - **/target/**/* - **/.classpath - **/.gitignore - **/.project - **/.checkstyle - **/.settings/* - src/test/resources/server-key.pem - src/test/resources/server-key.p12 - src/test/resources/server-key.jks - src/test/resources/server-cert.pem - src/test/resources/client-key.pem - src/test/resources/client-key.p12 - src/test/resources/client-key.jks - src/test/resources/client-cert.pem - src/test/resources/keyStoreClientPassword.txt - src/test/resources/keyStoreServerPassword.txt - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - false - - - listener - org.apache.bookkeeper.common.testing.util.TimedOutTestsListener - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven-javadoc-plugin.version} - - - none - org.apache.bookkeeper.client:org.apache.bookkeeper.conf:org.apache.bookkeeper.feature - - - Bookkeeper - org.apache.bookkeeper* - - - - - - attach-javadocs - - jar - - - - - - - - - vertx-http-server - - - org.apache.bookkeeper.http - vertx-http-server - ${project.parent.version} - - - - - tls-certs - - - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - - Generate Self-Signed Certificates - generate-test-resources - - exec - - - ${basedir}/src/test/resources - ${basedir}/src/test/resources/generateKeysAndCerts.sh - - - - - - - - - - skipBookKeeperServerTests - - - skipBookKeeperServerTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - - - diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java index f0a002f4583..ff0d15d4e5c 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java @@ -25,6 +25,7 @@ import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_INDEX_SCOPE; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_LEDGER_SCOPE; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; @@ -1082,7 +1083,8 @@ public ByteBuf readEntry(long ledgerId, long entryId) LOG.trace("Reading {}@{}", entryId, ledgerId); } ByteBuf entry = handle.readEntry(entryId); - bookieStats.getReadBytes().add(entry.readableBytes()); + entrySize = entry.readableBytes(); + bookieStats.getReadBytes().add(entrySize); success = true; return entry; } finally { @@ -1192,10 +1194,12 @@ public static boolean format(ServerConfiguration conf, // Clean up metadata directories if they are separate from the // ledger dirs - File metadataDir = new File(conf.getGcEntryLogMetadataCachePath()); - if (!cleanDir(metadataDir)) { - LOG.error("Formatting ledger metadata directory {} failed", metadataDir); - return false; + if (!Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath())) { + File metadataDir = new File(conf.getGcEntryLogMetadataCachePath()); + if (!cleanDir(metadataDir)) { + LOG.error("Formatting ledger metadata directory {} failed", metadataDir); + return false; + } } LOG.info("Bookie format completed successfully"); return true; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index bf00566f1b3..ba0286429f2 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -21,7 +21,9 @@ package org.apache.bookkeeper.bookie; +import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.IOException; @@ -45,6 +47,7 @@ import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.util.MathUtils; import org.apache.bookkeeper.util.SafeRunnable; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -120,6 +123,7 @@ public class GarbageCollectorThread extends SafeRunnable { final GarbageCleaner garbageCleaner; final ServerConfiguration conf; + final LedgerDirsManager ledgerDirsManager; /** * Create a garbage collector thread. @@ -129,8 +133,10 @@ public class GarbageCollectorThread extends SafeRunnable { * @throws IOException */ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerManager, - final CompactableLedgerStorage ledgerStorage, StatsLogger statsLogger) throws IOException { - this(conf, ledgerManager, ledgerStorage, statsLogger, + final LedgerDirsManager ledgerDirsManager, + final CompactableLedgerStorage ledgerStorage, + StatsLogger statsLogger) throws IOException { + this(conf, ledgerManager, ledgerDirsManager, ledgerStorage, statsLogger, Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("GarbageCollectorThread"))); } @@ -143,6 +149,7 @@ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerMana */ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerManager, + final LedgerDirsManager ledgerDirsManager, final CompactableLedgerStorage ledgerStorage, StatsLogger statsLogger, ScheduledExecutorService gcExecutor) @@ -150,6 +157,7 @@ public GarbageCollectorThread(ServerConfiguration conf, this.gcExecutor = gcExecutor; this.conf = conf; + this.ledgerDirsManager = ledgerDirsManager; this.entryLogger = ledgerStorage.getEntryLogger(); this.entryLogMetaMap = createEntryLogMetadataMap(); this.ledgerStorage = ledgerStorage; @@ -260,11 +268,13 @@ public void removeEntryLog(long logToRemove) { private EntryLogMetadataMap createEntryLogMetadataMap() throws IOException { if (conf.isGcEntryLogMetadataCacheEnabled()) { - String baseDir = this.conf.getGcEntryLogMetadataCachePath(); + String baseDir = Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath()) + ? this.ledgerDirsManager.getAllLedgerDirs().get(0).getPath() : conf.getGcEntryLogMetadataCachePath(); try { return new PersistentEntryLogMetadataMap(baseDir, conf); } catch (IOException e) { - LOG.error("Failed to initialize persistent-metadata-map , clean up {}", baseDir, e); + LOG.error("Failed to initialize persistent-metadata-map , clean up {}", + baseDir + "/" + METADATA_CACHE, e); throw e; } } else { @@ -390,35 +400,43 @@ public void runWithFlags(boolean force, boolean suspendMajor, boolean suspendMin // enter major compaction LOG.info("Enter major compaction, suspendMajor {}", suspendMajor); majorCompacting.set(true); - doCompactEntryLogs(majorCompactionThreshold, majorCompactionMaxTimeMillis); - lastMajorCompactionTime = System.currentTimeMillis(); - // and also move minor compaction time - lastMinorCompactionTime = lastMajorCompactionTime; - gcStats.getMajorCompactionCounter().inc(); - majorCompacting.set(false); + try { + doCompactEntryLogs(majorCompactionThreshold, majorCompactionMaxTimeMillis); + } finally { + lastMajorCompactionTime = System.currentTimeMillis(); + // and also move minor compaction time + lastMinorCompactionTime = lastMajorCompactionTime; + gcStats.getMajorCompactionCounter().inc(); + majorCompacting.set(false); + } } else if (((isForceMinorCompactionAllow && force) || (enableMinorCompaction && (force || curTime - lastMinorCompactionTime > minorCompactionInterval))) && (!suspendMinor)) { // enter minor compaction LOG.info("Enter minor compaction, suspendMinor {}", suspendMinor); minorCompacting.set(true); - doCompactEntryLogs(minorCompactionThreshold, minorCompactionMaxTimeMillis); - lastMinorCompactionTime = System.currentTimeMillis(); - gcStats.getMinorCompactionCounter().inc(); - minorCompacting.set(false); + try { + doCompactEntryLogs(minorCompactionThreshold, minorCompactionMaxTimeMillis); + } finally { + lastMinorCompactionTime = System.currentTimeMillis(); + gcStats.getMinorCompactionCounter().inc(); + minorCompacting.set(false); + } } - + gcStats.getGcThreadRuntime().registerSuccessfulEvent( + MathUtils.nowInNano() - threadStart, TimeUnit.NANOSECONDS); + } catch (EntryLogMetadataMapException e) { + LOG.error("Error in entryLog-metadatamap, Failed to complete GC/Compaction due to entry-log {}", + e.getMessage(), e); + gcStats.getGcThreadRuntime().registerFailedEvent( + MathUtils.nowInNano() - threadStart, TimeUnit.NANOSECONDS); + } finally { if (force && forceGarbageCollection.compareAndSet(true, false)) { LOG.info("{} Set forceGarbageCollection to false after force GC to make it forceGC-able again.", Thread.currentThread().getName()); } - } catch (EntryLogMetadataMapException e) { - LOG.error("Error in entryLog-metadatamap, Failed to complete GC/Compaction due to entry-log {}", - e.getMessage(), e); } - gcStats.getGcThreadRuntime().registerSuccessfulEvent( - MathUtils.nowInNano() - threadStart, TimeUnit.NANOSECONDS); } /** @@ -438,9 +456,7 @@ private void doGcEntryLogs() throws EntryLogMetadataMapException { // Loop through all of the entry logs and remove the non-active ledgers. entryLogMetaMap.forEach((entryLogId, meta) -> { try { - removeIfLedgerNotExists(meta); - // update entryMetadta to persistent-map - entryLogMetaMap.put(meta.getEntryLogId(), meta); + boolean modified = removeIfLedgerNotExists(meta); if (meta.isEmpty()) { // This means the entry log is not associated with any active // ledgers anymore. @@ -448,6 +464,9 @@ private void doGcEntryLogs() throws EntryLogMetadataMapException { LOG.info("Deleting entryLogId {} as it has no active ledgers!", entryLogId); removeEntryLog(entryLogId); gcStats.getReclaimedSpaceViaDeletes().add(meta.getTotalSize()); + } else if (modified) { + // update entryLogMetaMap only when the meta modified. + entryLogMetaMap.put(meta.getEntryLogId(), meta); } } catch (EntryLogMetadataMapException e) { // Ignore and continue because ledger will not be cleaned up @@ -462,16 +481,23 @@ private void doGcEntryLogs() throws EntryLogMetadataMapException { this.numActiveEntryLogs = entryLogMetaMap.size(); } - private void removeIfLedgerNotExists(EntryLogMetadata meta) throws EntryLogMetadataMapException { + private boolean removeIfLedgerNotExists(EntryLogMetadata meta) throws EntryLogMetadataMapException { + MutableBoolean modified = new MutableBoolean(false); meta.removeLedgerIf((entryLogLedger) -> { // Remove the entry log ledger from the set if it isn't active. try { - return !ledgerStorage.ledgerExists(entryLogLedger); + boolean exist = ledgerStorage.ledgerExists(entryLogLedger); + if (!exist) { + modified.setTrue(); + } + return !exist; } catch (IOException e) { LOG.error("Error reading from ledger storage", e); return false; } }); + + return modified.getValue(); } /** diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java index dcfeed0bf3c..0960c6658ca 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java @@ -193,7 +193,8 @@ public void initializeWithEntryLogger(ServerConfiguration conf, this.entryLogger.addListener(this); ledgerCache = new LedgerCacheImpl(conf, activeLedgers, null == indexDirsManager ? ledgerDirsManager : indexDirsManager, statsLogger); - gcThread = new GarbageCollectorThread(conf, ledgerManager, this, statsLogger.scope("gc")); + gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager, + this, statsLogger.scope("gc")); pageSize = conf.getPageSize(); ledgerDirsManager.addLedgerDirsListener(getLedgerDirsListener()); // Expose Stats diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java index 2c054a1d93b..4d7059d0cdd 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Journal.java @@ -379,9 +379,9 @@ public int process(boolean shouldForceWrite) throws IOException { return 0; } + long startTime = MathUtils.nowInNano(); try { if (shouldForceWrite) { - long startTime = MathUtils.nowInNano(); this.logFile.forceWrite(false); journalStats.getJournalSyncStats() .registerSuccessfulEvent(MathUtils.elapsedNanos(startTime), TimeUnit.NANOSECONDS); @@ -397,6 +397,10 @@ public int process(boolean shouldForceWrite) throws IOException { } return forceWriteWaiters.size(); + } catch (IOException e) { + journalStats.getJournalSyncStats() + .registerFailedEvent(MathUtils.elapsedNanos(startTime), TimeUnit.NANOSECONDS); + throw e; } finally { closeFileIfNecessary(); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java index 93bf47c4e03..b3c724b231e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ScanAndCompareGarbageCollector.java @@ -243,6 +243,10 @@ private Set removeOverReplicatedledgers(Set bkActiveledgers, final G continue; } } catch (Throwable t) { + if (!(t.getCause() instanceof BKException.BKNoSuchLedgerExistsOnMetadataServerException)) { + LOG.warn("Failed to get metadata for ledger {}. {}: {}", + ledgerId, t.getClass().getName(), t.getMessage()); + } latch.countDown(); continue; } @@ -268,6 +272,10 @@ private Set removeOverReplicatedledgers(Set bkActiveledgers, final G overReplicatedLedgers.add(ledgerId); garbageCleaner.clean(ledgerId); } + } else if (!(exception + instanceof BKException.BKNoSuchLedgerExistsOnMetadataServerException)) { + LOG.warn("Failed to get metadata for ledger {}. {}: {}", + ledgerId, exception.getClass().getName(), exception.getMessage()); } } finally { semaphore.release(); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java index 2ba63cc222c..0a563a1c9ce 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java @@ -379,7 +379,7 @@ public static void readLedgerIndexEntries(long ledgerId, ServerConfiguration ser EntryLocationIndex entryLocationIndex = new EntryLocationIndex(serverConf, (basePath, subPath, dbConfigType, conf1) -> - new KeyValueStorageRocksDB(basePath, subPath, DbConfigType.Small, conf1, true), + new KeyValueStorageRocksDB(basePath, subPath, DbConfigType.Default, conf1, true), ledgerBasePath, NullStatsLogger.INSTANCE); try { long lastEntryId = entryLocationIndex.getLastEntryInLedger(ledgerId); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/EntryLocationIndex.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/EntryLocationIndex.java index 6b7a20d7485..79784651e90 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/EntryLocationIndex.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/EntryLocationIndex.java @@ -53,7 +53,7 @@ public class EntryLocationIndex implements Closeable { public EntryLocationIndex(ServerConfiguration conf, KeyValueStorageFactory storageFactory, String basePath, StatsLogger stats) throws IOException { - locationsDb = storageFactory.newKeyValueStorage(basePath, "locations", DbConfigType.Huge, conf); + locationsDb = storageFactory.newKeyValueStorage(basePath, "locations", DbConfigType.EntryLocation, conf); this.stats = new EntryLocationIndexStats( stats, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageFactory.java index f3023c1ebae..08f68feff78 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageFactory.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageFactory.java @@ -33,8 +33,9 @@ public interface KeyValueStorageFactory { * Enum used to specify different config profiles in the underlying storage. */ enum DbConfigType { - Small, // Used for ledgers db, doesn't need particular configuration - Huge // Used for location index, lots of writes and much bigger dataset + Default, // Used for default,command until or test case + LedgerMetadata, // Used for ledgers db, doesn't need particular configuration + EntryLocation // Used for location index, lots of writes and much bigger dataset } KeyValueStorage newKeyValueStorage(String defaultBasePath, String subPath, DbConfigType dbConfigType, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java index bda82725586..e6eb1978c9a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java @@ -83,6 +83,7 @@ public class KeyValueStorageRocksDB implements KeyValueStorage { private static final String ROCKSDB_NUM_LEVELS = "dbStorage_rocksDB_numLevels"; private static final String ROCKSDB_NUM_FILES_IN_LEVEL0 = "dbStorage_rocksDB_numFilesInLevel0"; private static final String ROCKSDB_MAX_SIZE_IN_LEVEL1_MB = "dbStorage_rocksDB_maxSizeInLevel1MB"; + private static final String ROCKSDB_FORMAT_VERSION = "dbStorage_rocksDB_format_version"; public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbConfigType, ServerConfiguration conf) throws IOException { @@ -122,6 +123,7 @@ public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbCo int blockSize = conf.getInt(ROCKSDB_BLOCK_SIZE, 64 * 1024); int bloomFilterBitsPerKey = conf.getInt(ROCKSDB_BLOOM_FILTERS_BITS_PER_KEY, 10); boolean lz4CompressionEnabled = conf.getBoolean(ROCKSDB_LZ4_COMPRESSION_ENABLED, true); + int formatVersion = conf.getInt(ROCKSDB_FORMAT_VERSION, 2); if (lz4CompressionEnabled) { options.setCompressionType(CompressionType.LZ4_COMPRESSION); @@ -144,7 +146,7 @@ public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbCo BlockBasedTableConfig tableOptions = new BlockBasedTableConfig(); tableOptions.setBlockSize(blockSize); tableOptions.setBlockCache(cache); - tableOptions.setFormatVersion(2); + tableOptions.setFormatVersion(formatVersion); tableOptions.setChecksumType(ChecksumType.kxxHash); if (bloomFilterBitsPerKey > 0) { tableOptions.setFilterPolicy(new BloomFilter(bloomFilterBitsPerKey, false)); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java index eb7548ca9f2..c48410048c3 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java @@ -70,7 +70,7 @@ public class LedgerMetadataIndex implements Closeable { public LedgerMetadataIndex(ServerConfiguration conf, KeyValueStorageFactory storageFactory, String basePath, StatsLogger stats) throws IOException { - ledgersDb = storageFactory.newKeyValueStorage(basePath, "ledgers", DbConfigType.Small, conf); + ledgersDb = storageFactory.newKeyValueStorage(basePath, "ledgers", DbConfigType.LedgerMetadata, conf); ledgers = new ConcurrentLongHashMap<>(); ledgersCount = new AtomicInteger(); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexCheckOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexCheckOp.java index a48de342943..9a2a5f5cafd 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexCheckOp.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexCheckOp.java @@ -59,7 +59,7 @@ public boolean initiate() throws IOException { try { KeyValueStorage index = new KeyValueStorageRocksDB(basePath, LedgersSubPath, - DbConfigType.Small, conf, true); + DbConfigType.Default, conf, true); // Read all ledgers from db KeyValueStorage.CloseableIterator> iterator = index.iterator(); int ctr = 0; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexRebuildOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexRebuildOp.java index 15410ef1193..7e10de8fe6c 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexRebuildOp.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexRebuildOp.java @@ -91,7 +91,7 @@ public boolean initiate() { + "Starting to build a new ledgers index", ledgers.size()); try (KeyValueStorage newIndex = KeyValueStorageRocksDB.factory.newKeyValueStorage( - basePath, tempLedgersSubPath, DbConfigType.Small, conf)) { + basePath, tempLedgersSubPath, DbConfigType.Default, conf)) { LOG.info("Created ledgers index at temp location {}", tempPath); for (Long ledgerId : ledgers) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java index 7eae6da8a44..55c5c90c31e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java @@ -77,7 +77,7 @@ public void initiate() throws IOException { LOG.info("Found {} active ledgers in ledger manager", activeLedgers.size()); KeyValueStorage newIndex = KeyValueStorageRocksDB.factory.newKeyValueStorage(basePath, "locations", - DbConfigType.Huge, conf); + DbConfigType.Default, conf); int totalEntryLogs = entryLogs.size(); int completedEntryLogs = 0; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java index 812ab845385..99f60691f55 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java @@ -20,6 +20,7 @@ */ package org.apache.bookkeeper.bookie.storage.ldb; +import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; import io.netty.util.concurrent.FastThreadLocal; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -31,6 +32,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; +import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.bookie.BookieException.EntryLogMetadataMapException; import org.apache.bookkeeper.bookie.EntryLogMetadata; import org.apache.bookkeeper.bookie.EntryLogMetadata.EntryLogMetadataRecyclable; @@ -38,15 +40,13 @@ import org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorage.CloseableIterator; import org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageFactory.DbConfigType; import org.apache.bookkeeper.conf.ServerConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Persistent entryLogMetadata-map that stores entry-loggers metadata into * rocksDB. */ +@Slf4j public class PersistentEntryLogMetadataMap implements EntryLogMetadataMap { - private static final Logger LOG = LoggerFactory.getLogger(PersistentEntryLogMetadataMap.class); // persistent Rocksdb to store metadata-map private final KeyValueStorage metadataMapDB; private AtomicBoolean isClosed = new AtomicBoolean(false); @@ -77,19 +77,20 @@ protected DataInputStream initialValue() { }; public PersistentEntryLogMetadataMap(String metadataPath, ServerConfiguration conf) throws IOException { - LOG.info("Loading persistent entrylog metadata-map from {}", metadataPath); + log.info("Loading persistent entrylog metadata-map from {}/{}", metadataPath, METADATA_CACHE); File dir = new File(metadataPath); if (!dir.mkdirs() && !dir.exists()) { String err = "Unable to create directory " + dir; - LOG.error(err); + log.error(err); throw new IOException(err); } - metadataMapDB = KeyValueStorageRocksDB.factory.newKeyValueStorage(metadataPath, "metadata-cache", + metadataMapDB = KeyValueStorageRocksDB.factory.newKeyValueStorage(metadataPath, METADATA_CACHE, DbConfigType.Small, conf); } @Override public boolean containsKey(long entryLogId) throws EntryLogMetadataMapException { + throwIfClosed(); LongWrapper key = LongWrapper.get(entryLogId); try { boolean isExist; @@ -106,6 +107,7 @@ public boolean containsKey(long entryLogId) throws EntryLogMetadataMapException @Override public void put(long entryLogId, EntryLogMetadata entryLogMeta) throws EntryLogMetadataMapException { + throwIfClosed(); LongWrapper key = LongWrapper.get(entryLogId); try { baos.get().reset(); @@ -113,7 +115,7 @@ public void put(long entryLogId, EntryLogMetadata entryLogMeta) throws EntryLogM entryLogMeta.serialize(dataos.get()); metadataMapDB.put(key.array, baos.get().toByteArray()); } catch (IllegalStateException | IOException e) { - LOG.error("Failed to serialize entrylog-metadata, entryLogId {}", entryLogId); + log.error("Failed to serialize entrylog-metadata, entryLogId {}", entryLogId); throw new EntryLogMetadataMapException(e); } } finally { @@ -128,6 +130,7 @@ public void put(long entryLogId, EntryLogMetadata entryLogMeta) throws EntryLogM */ @Override public void forEach(BiConsumer action) throws EntryLogMetadataMapException { + throwIfClosed(); CloseableIterator> iterator = metadataMapDB.iterator(); try { while (iterator.hasNext()) { @@ -157,19 +160,20 @@ public void forEach(BiConsumer action) throws EntryLogMe } } } catch (IOException e) { - LOG.error("Failed to iterate over entry-log metadata map {}", e.getMessage(), e); + log.error("Failed to iterate over entry-log metadata map {}", e.getMessage(), e); throw new EntryLogMetadataMapException(e); } finally { try { iterator.close(); } catch (IOException e) { - LOG.error("Failed to close entry-log metadata-map rocksDB iterator {}", e.getMessage(), e); + log.error("Failed to close entry-log metadata-map rocksDB iterator {}", e.getMessage(), e); } } } @Override public void remove(long entryLogId) throws EntryLogMetadataMapException { + throwIfClosed(); LongWrapper key = LongWrapper.get(entryLogId); try { try { @@ -184,6 +188,7 @@ public void remove(long entryLogId) throws EntryLogMetadataMapException { @Override public int size() throws EntryLogMetadataMapException { + throwIfClosed(); try { return (int) metadataMapDB.count(); } catch (IOException e) { @@ -193,8 +198,18 @@ public int size() throws EntryLogMetadataMapException { @Override public void close() throws IOException { - isClosed.set(true); - metadataMapDB.close(); + if (isClosed.compareAndSet(false, true)) { + metadataMapDB.close(); + } else { + log.warn("Attempted to close already closed PersistentEntryLogMetadataMap"); + } } + public void throwIfClosed() throws EntryLogMetadataMapException { + if (isClosed.get()) { + final String msg = "Attempted to use PersistentEntryLogMetadataMap after it was closed"; + log.error(msg); + throw new EntryLogMetadataMapException(new IOException(msg)); + } + } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java index 3ec4e111649..f4038b4c85c 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java @@ -149,7 +149,7 @@ public ByteBuf get(long ledgerId, long entryId) { int entryOffset = (int) res.first; int entryLen = (int) res.second; - ByteBuf entry = allocator.directBuffer(entryLen, entryLen); + ByteBuf entry = allocator.buffer(entryLen, entryLen); entry.writeBytes(cacheSegments.get(segmentIdx), entryOffset, entryLen); return entry; } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java index 4d2e4adafc4..ce8af36b4e0 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java @@ -183,7 +183,7 @@ public SingleDirectoryDbLedgerStorage(ServerConfiguration conf, LedgerManager le TransientLedgerInfo.LEDGER_INFO_CACHING_TIME_MINUTES, TimeUnit.MINUTES); entryLogger = new EntryLogger(conf, ledgerDirsManager, null, statsLogger, allocator); - gcThread = new GarbageCollectorThread(conf, ledgerManager, this, statsLogger); + gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager, this, statsLogger); dbLedgerStorageStats = new DbLedgerStorageStats( ledgerDirStatsLogger, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java index 50297d121c8..46c5a10786b 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RackawareEnsemblePlacementPolicyImpl.java @@ -269,7 +269,7 @@ public RackawareEnsemblePlacementPolicyImpl initialize(ClientConfiguration conf, } } catch (RuntimeException re) { if (!conf.getEnforceMinNumRacksPerWriteQuorum()) { - LOG.error("Failed to initialize DNS Resolver {}, used default subnet resolver ", + LOG.warn("Failed to initialize DNS Resolver {}, used default subnet resolver ", dnsResolverName, re); dnsResolver = new DefaultResolver(this::getDefaultRack); dnsResolver.setBookieAddressResolver(bookieAddressResolver); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index a587145fcaf..144c415219d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -17,7 +17,6 @@ */ package org.apache.bookkeeper.conf; -import static org.apache.bookkeeper.util.BookKeeperConstants.ENTRYLOG_INDEX_CACHE; import static org.apache.bookkeeper.util.BookKeeperConstants.MAX_LOG_SIZE_LIMIT; import com.google.common.annotations.Beta; @@ -319,6 +318,15 @@ public class ServerConfiguration extends AbstractConfiguration cb) + throws ReplicationException.UnavailableException; + /** * Receive notification asynchronously when the lostBookieRecoveryDelay value is Changed. * diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/NullMetadataBookieDriver.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/NullMetadataBookieDriver.java index 3a4247f6c61..4a1e6e419fa 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/NullMetadataBookieDriver.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/NullMetadataBookieDriver.java @@ -394,5 +394,7 @@ public String getReplicationWorkerIdRereplicatingLedger(long ledgerId) throws ReplicationException.UnavailableException { throw new ReplicationException.UnavailableException("null"); } + @Override + public void notifyUnderReplicationLedgerChanged(GenericCallback cb) {} } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java index b340e0721cb..9dcc81c1622 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/ZkLedgerUnderreplicationManager.java @@ -59,6 +59,7 @@ import org.apache.bookkeeper.util.BookKeeperConstants; import org.apache.bookkeeper.util.SubTreeCache; import org.apache.bookkeeper.util.ZkUtils; +import org.apache.zookeeper.AddWatchMode; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.Code; @@ -866,6 +867,28 @@ public int getLostBookieRecoveryDelay() throws UnavailableException { } } + @Override + public void notifyUnderReplicationLedgerChanged(GenericCallback cb) throws UnavailableException { + LOG.debug("notifyUnderReplicationLedgerChanged()"); + Watcher w = new Watcher() { + @Override + public void process(WatchedEvent e) { + if (e.getType() == Event.EventType.NodeDeleted && idExtractionPattern.matcher(e.getPath()).find()) { + cb.operationComplete(0, null); + } + } + }; + try { + zkc.addWatch(urLedgerPath, w, AddWatchMode.PERSISTENT_RECURSIVE); + } catch (KeeperException ke) { + LOG.error("Error while checking the state of underReplicated ledgers", ke); + throw new ReplicationException.UnavailableException("Error contacting zookeeper", ke); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new ReplicationException.UnavailableException("Interrupted while contacting zookeeper", ie); + } + } + @Override public void notifyLostBookieRecoveryDelayChanged(GenericCallback cb) throws UnavailableException { LOG.debug("notifyLostBookieRecoveryDelayChanged()"); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/Value.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/Value.java index 43354c344e5..9a3b43e12d6 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/Value.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/metastore/Value.java @@ -92,7 +92,7 @@ public Value project(Set fields) { @Override public int hashCode() { - HashFunction hf = Hashing.murmur3_32(); + HashFunction hf = Hashing.murmur3_32_fixed(); Hasher hc = hf.newHasher(); for (String key : fields.keySet()) { hc.putString(key, Charset.defaultCharset()); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopologyImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopologyImpl.java index 123f13ca8a3..19b2c0497f7 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopologyImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/net/NetworkTopologyImpl.java @@ -17,6 +17,7 @@ */ package org.apache.bookkeeper.net; +import com.google.common.base.Strings; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -119,9 +120,10 @@ boolean isRack() { * @return true if this node is an ancestor of n */ boolean isAncestor(Node n) { - return getPath(this).equals(NodeBase.PATH_SEPARATOR_STR) + return !Strings.isNullOrEmpty(n.getNetworkLocation()) + && (getPath(this).equals(NodeBase.PATH_SEPARATOR_STR) || (n.getNetworkLocation() + NodeBase.PATH_SEPARATOR_STR).startsWith(getPath(this) - + NodeBase.PATH_SEPARATOR_STR); + + NodeBase.PATH_SEPARATOR_STR)); } /** diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java index 8c69812ff94..c3b69dd4beb 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java @@ -34,8 +34,10 @@ import static org.apache.bookkeeper.replication.ReplicationStats.NUM_LEDGERS_HAVING_NO_REPLICA_OF_AN_ENTRY; import static org.apache.bookkeeper.replication.ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY; import static org.apache.bookkeeper.replication.ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY; +import static org.apache.bookkeeper.replication.ReplicationStats.NUM_REPLICATED_LEDGERS; import static org.apache.bookkeeper.replication.ReplicationStats.NUM_UNDERREPLICATED_LEDGERS_ELAPSED_RECOVERY_GRACE_PERIOD; import static org.apache.bookkeeper.replication.ReplicationStats.NUM_UNDER_REPLICATED_LEDGERS; +import static org.apache.bookkeeper.replication.ReplicationStats.NUM_UNDER_REPLICATED_LEDGERS_GUAGE; import static org.apache.bookkeeper.replication.ReplicationStats.PLACEMENT_POLICY_CHECK_TIME; import static org.apache.bookkeeper.replication.ReplicationStats.REPLICAS_CHECK_TIME; import static org.apache.bookkeeper.replication.ReplicationStats.UNDER_REPLICATED_LEDGERS_TOTAL_SIZE; @@ -45,6 +47,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; import com.google.common.collect.HashMultiset; +import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Multiset; import com.google.common.collect.Sets; @@ -89,7 +92,6 @@ import org.apache.bookkeeper.common.concurrent.FutureUtils; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; -import org.apache.bookkeeper.meta.AbstractZkLedgerManagerFactory; import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.meta.LedgerManager.LedgerRange; import org.apache.bookkeeper.meta.LedgerManager.LedgerRangeIterator; @@ -163,6 +165,7 @@ public class Auditor implements AutoCloseable { private final AtomicInteger numLedgersFoundHavingLessThanAQReplicasOfAnEntry; private final AtomicInteger numLedgersHavingLessThanWQReplicasOfAnEntryGuageValue; private final AtomicInteger numLedgersFoundHavingLessThanWQReplicasOfAnEntry; + private final AtomicInteger underReplicatedLedgersGuageValue; private final long underreplicatedLedgerRecoveryGracePeriod; private final int zkOpTimeoutMs; private final Semaphore openLedgerNoRecoverySemaphore; @@ -235,6 +238,11 @@ public class Auditor implements AutoCloseable { help = "the number of delayed-bookie-audits cancelled" ) private final Counter numDelayedBookieAuditsCancelled; + @StatsDoc( + name = NUM_REPLICATED_LEDGERS, + help = "the number of replicated ledgers" + ) + private final Counter numReplicatedLedgers; @StatsDoc( name = NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY, help = "Gauge for number of ledgers not adhering to placement policy found in placement policy check" @@ -267,6 +275,11 @@ public class Auditor implements AutoCloseable { + ", this doesn't include ledgers counted towards numLedgersHavingLessThanAQReplicasOfAnEntry" ) private final Gauge numLedgersHavingLessThanWQReplicasOfAnEntry; + @StatsDoc( + name = NUM_UNDER_REPLICATED_LEDGERS_GUAGE, + help = "Gauge for num of underreplicated ledgers" + ) + private final Gauge numUnderReplicatedLedgers; static BookKeeper createBookKeeperClient(ServerConfiguration conf) throws InterruptedException, IOException { return createBookKeeperClient(conf, NullStatsLogger.INSTANCE); @@ -364,6 +377,7 @@ public Auditor(final String bookieIdentifier, this.openLedgerNoRecoverySemaphoreWaitTimeoutMSec = conf.getAuditorAcquireConcurrentOpenLedgerOperationsTimeoutMSec(); + this.underReplicatedLedgersGuageValue = new AtomicInteger(0); numUnderReplicatedLedger = this.statsLogger.getOpStatsLogger(ReplicationStats.NUM_UNDER_REPLICATED_LEDGERS); underReplicatedLedgerTotalSize = this.statsLogger.getOpStatsLogger(UNDER_REPLICATED_LEDGERS_TOTAL_SIZE); uRLPublishTimeForLostBookies = this.statsLogger @@ -380,6 +394,7 @@ public Auditor(final String bookieIdentifier, numBookieAuditsDelayed = this.statsLogger.getCounter(ReplicationStats.NUM_BOOKIE_AUDITS_DELAYED); numDelayedBookieAuditsCancelled = this.statsLogger .getCounter(ReplicationStats.NUM_DELAYED_BOOKIE_AUDITS_DELAYES_CANCELLED); + numReplicatedLedgers = this.statsLogger.getCounter(NUM_REPLICATED_LEDGERS); numLedgersNotAdheringToPlacementPolicy = new Gauge() { @Override public Integer getDefaultValue() { @@ -460,6 +475,18 @@ public Integer getSample() { }; this.statsLogger.registerGauge(ReplicationStats.NUM_LEDGERS_HAVING_LESS_THAN_WQ_REPLICAS_OF_AN_ENTRY, numLedgersHavingLessThanWQReplicasOfAnEntry); + numUnderReplicatedLedgers = new Gauge() { + @Override + public Integer getDefaultValue() { + return 0; + } + + @Override + public Integer getSample() { + return underReplicatedLedgersGuageValue.get(); + } + }; + this.statsLogger.registerGauge(NUM_UNDER_REPLICATED_LEDGERS_GUAGE, numUnderReplicatedLedgers); this.bkc = bkc; this.ownBkc = ownBkc; @@ -480,10 +507,7 @@ public Thread newThread(Runnable r) { private void initialize(ServerConfiguration conf, BookKeeper bkc) throws UnavailableException { try { - LedgerManagerFactory ledgerManagerFactory = AbstractZkLedgerManagerFactory - .newLedgerManagerFactory( - conf, - bkc.getMetadataClientDriver().getLayoutManager()); + LedgerManagerFactory ledgerManagerFactory = bkc.getLedgerManagerFactory(); ledgerManager = ledgerManagerFactory.newLedgerManager(); this.bookieLedgerIndexer = new BookieLedgerIndexer(ledgerManager); @@ -503,7 +527,7 @@ private void initialize(ServerConfiguration conf, BookKeeper bkc) } catch (CompatibilityException ce) { throw new UnavailableException( "CompatibilityException while initializing Auditor", ce); - } catch (IOException | KeeperException ioe) { + } catch (KeeperException ioe) { throw new UnavailableException( "Exception while initializing Auditor", ioe); } catch (InterruptedException ie) { @@ -707,6 +731,15 @@ public void start() { submitShutdownTask(); } + try { + this.ledgerUnderreplicationManager.notifyUnderReplicationLedgerChanged( + new UnderReplicatedLedgersChangedCb()); + } catch (UnavailableException ue) { + LOG.error("Exception while registering for under-replicated ledgers change notification, so exiting", + ue); + submitShutdownTask(); + } + scheduleBookieCheckTask(); scheduleCheckAllLedgersTask(); schedulePlacementPolicyCheckTask(); @@ -1014,6 +1047,16 @@ public void run() { }), initialDelay, interval, TimeUnit.SECONDS); } + private class UnderReplicatedLedgersChangedCb implements GenericCallback { + @Override + public void operationComplete(int rc, Void result) { + Iterator underreplicatedLedgersInfo = ledgerUnderreplicationManager + .listLedgersToRereplicate(null); + underReplicatedLedgersGuageValue.set(Iterators.size(underreplicatedLedgersInfo)); + numReplicatedLedgers.inc(); + } + } + private class LostBookieRecoveryDelayChangedCb implements GenericCallback { @Override public void operationComplete(int rc, Void result) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationStats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationStats.java index d04fcf04279..74b76b23b22 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationStats.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/ReplicationStats.java @@ -64,4 +64,6 @@ public interface ReplicationStats { String REPLICATE_EXCEPTION = "exceptions"; String NUM_DEFER_LEDGER_LOCK_RELEASE_OF_FAILED_LEDGER = "NUM_DEFER_LEDGER_LOCK_RELEASE_OF_FAILED_LEDGER"; String NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION = "NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION"; + String NUM_UNDER_REPLICATED_LEDGERS_GUAGE = "NUM_UNDER_REPLICATED_LEDGERS_GUAGE"; + String NUM_REPLICATED_LEDGERS = "NUM_REPLICATED_LEDGERS"; } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java index 3c7d7fe7cfd..ddc8613acd2 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/Main.java @@ -116,6 +116,7 @@ public class Main { BK_OPTS.addOption("z", "zkserver", true, "Zookeeper Server"); BK_OPTS.addOption("m", "zkledgerpath", true, "Zookeeper ledgers root path"); BK_OPTS.addOption("p", "bookieport", true, "bookie port exported"); + BK_OPTS.addOption("hp", "httpport", true, "bookie http port exported"); BK_OPTS.addOption("j", "journal", true, "bookie journal directory"); Option indexDirs = new Option ("i", "indexdirs", true, "bookie index directories"); indexDirs.setArgs(10); @@ -212,6 +213,13 @@ private static ServerConfiguration parseArgs(String[] args) conf.setBookiePort(Integer.parseInt(sPort)); } + if (cmdLine.hasOption("httpport")) { + String sPort = cmdLine.getOptionValue("httpport"); + log.info("Get cmdline http port: {}", sPort); + Integer iPort = Integer.parseInt(sPort); + conf.setHttpServerPort(iPort.intValue()); + } + if (cmdLine.hasOption('j')) { String sJournalDir = cmdLine.getOptionValue('j'); log.info("Get cmdline journal dir: {}", sJournalDir); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java index c047977c383..be55ca18f3f 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/BookieAuthZFactory.java @@ -95,6 +95,7 @@ public void onProtocolUpgrade() { for (String allowedRole : allowedRoles) { if (certRole[0].equals(allowedRole)) { authorized = true; + break; } } if (authorized) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java index 8e10a151d55..d928aac2fc5 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java @@ -31,7 +31,7 @@ public class BookKeeperConstants { public static final String BOOKIE_STATUS_FILENAME = "BOOKIE_STATUS"; public static final String PASSWD = "passwd"; public static final String CURRENT_DIR = "current"; - public static final String ENTRYLOG_INDEX_CACHE = "entrylogIndexCache"; + public static final String METADATA_CACHE = "metadata-cache"; public static final String READONLY = "readonly"; // ////////////////////////// diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByBytesTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByBytesWithMetadataCacheTest.java similarity index 85% rename from bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByBytesTest.java rename to bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByBytesWithMetadataCacheTest.java index 29303d742d5..8f22f625175 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByBytesTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByBytesWithMetadataCacheTest.java @@ -23,8 +23,8 @@ /** * Test compaction by bytes. */ -public class CompactionByBytesTest extends CompactionTest { - public CompactionByBytesTest() { - super(true); +public class CompactionByBytesWithMetadataCacheTest extends CompactionTest { + public CompactionByBytesWithMetadataCacheTest() { + super(true, true); } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByBytesWithoutMetadataCacheTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByBytesWithoutMetadataCacheTest.java new file mode 100644 index 00000000000..7b1418dab94 --- /dev/null +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByBytesWithoutMetadataCacheTest.java @@ -0,0 +1,30 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.bookkeeper.bookie; + +/** + * Test compaction by bytes. + */ +public class CompactionByBytesWithoutMetadataCacheTest extends CompactionTest { + public CompactionByBytesWithoutMetadataCacheTest() { + super(true, false); + } +} diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByEntriesTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByEntriesWithMetadataCacheTest.java similarity index 84% rename from bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByEntriesTest.java rename to bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByEntriesWithMetadataCacheTest.java index df871a3d497..86a1f3bd980 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByEntriesTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByEntriesWithMetadataCacheTest.java @@ -23,8 +23,8 @@ /** * Test compactions by entries. */ -public class CompactionByEntriesTest extends CompactionTest { - public CompactionByEntriesTest() { - super(false); +public class CompactionByEntriesWithMetadataCacheTest extends CompactionTest { + public CompactionByEntriesWithMetadataCacheTest() { + super(false, true); } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByEntriesWithoutMetadataCacheTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByEntriesWithoutMetadataCacheTest.java new file mode 100644 index 00000000000..2a02eb26b05 --- /dev/null +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionByEntriesWithoutMetadataCacheTest.java @@ -0,0 +1,30 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.bookkeeper.bookie; + +/** + * Test compactions by entries. + */ +public class CompactionByEntriesWithoutMetadataCacheTest extends CompactionTest { + public CompactionByEntriesWithoutMetadataCacheTest() { + super(false, false); + } +} diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java index 1c2ee73f11d..2b35d7f0b53 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java @@ -104,11 +104,13 @@ public abstract class CompactionTest extends BookKeeperClusterTestCase { private final long minorCompactionInterval; private final long majorCompactionInterval; private final String msg; + private final boolean useMetadataCache; - public CompactionTest(boolean isByBytes) { + public CompactionTest(boolean isByBytes, boolean useMetadataCache) { super(NUM_BOOKIES); this.isThrottleByBytes = isByBytes; + this.useMetadataCache = useMetadataCache; this.digestType = DigestType.CRC32; this.passwdBytes = "".getBytes(); numEntries = 100; @@ -142,7 +144,7 @@ public void setUp() throws Exception { baseConf.setLedgerStorageClass(InterleavedLedgerStorage.class.getName()); baseConf.setIsThrottleByBytes(this.isThrottleByBytes); baseConf.setIsForceGCAllowWhenNoSpace(false); - baseConf.setGcEntryLogMetadataCacheEnabled(true); + baseConf.setGcEntryLogMetadataCacheEnabled(useMetadataCache); super.setUp(); } @@ -221,7 +223,9 @@ public void testDisableCompaction() throws Exception { assertFalse(getGCThread().enableMajorCompaction); assertFalse(getGCThread().enableMinorCompaction); getGCThread().triggerGC().get(); - assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); + if (useMetadataCache) { + assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); + } // after garbage collection, compaction should not be executed assertEquals(lastMinorCompactionTime, getGCThread().lastMinorCompactionTime); @@ -350,7 +354,9 @@ public void testMinorCompaction() throws Exception { getGCThread().enableForceGC(); getGCThread().triggerGC().get(); - assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); + if (useMetadataCache) { + assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); + } assertTrue( "ACTIVE_ENTRY_LOG_COUNT should have been updated", getStatsProvider(0) @@ -402,7 +408,7 @@ public void testMinorCompaction() throws Exception { } @Test - public void testMinorCompactionWithMaxTimeMillis() throws Exception { + public void testMinorCompactionWithMaxTimeMillisOk() throws Exception { // prepare data LedgerHandle[] lhs = prepareData(6, false); @@ -419,8 +425,10 @@ public void testMinorCompactionWithMaxTimeMillis() throws Exception { c.setMajorCompactionInterval(240000); // Setup limit on compaction duration. - c.setMinorCompactionMaxTimeMillis(15); - c.setMajorCompactionMaxTimeMillis(15); + // The limit is enough to compact. + c.setMinorCompactionMaxTimeMillis(5000); + c.setMajorCompactionMaxTimeMillis(5000); + return c; }); @@ -477,7 +485,70 @@ public void testMinorCompactionWithMaxTimeMillis() throws Exception { } + @Test + public void testMinorCompactionWithMaxTimeMillisTooShort() throws Exception { + // prepare data + LedgerHandle[] lhs = prepareData(6, false); + + for (LedgerHandle lh : lhs) { + lh.close(); + } + + // disable major compaction + // restart bookies + restartBookies(c-> { + c.setMajorCompactionThreshold(0.0f); + c.setGcWaitTime(60000); + c.setMinorCompactionInterval(120000); + c.setMajorCompactionInterval(240000); + + // Setup limit on compaction duration. + // The limit is not enough to finish the compaction + c.setMinorCompactionMaxTimeMillis(1); + c.setMajorCompactionMaxTimeMillis(1); + + return c; + }); + + getGCThread().enableForceGC(); + getGCThread().triggerGC().get(); + assertTrue( + "ACTIVE_ENTRY_LOG_COUNT should have been updated", + getStatsProvider(0) + .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_COUNT) + .getSample().intValue() > 0); + assertTrue( + "ACTIVE_ENTRY_LOG_SPACE_BYTES should have been updated", + getStatsProvider(0) + .getGauge("bookie.gc." + ACTIVE_ENTRY_LOG_SPACE_BYTES) + .getSample().intValue() > 0); + + long lastMinorCompactionTime = getGCThread().lastMinorCompactionTime; + long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime; + assertFalse(getGCThread().enableMajorCompaction); + assertTrue(getGCThread().enableMinorCompaction); + + // remove ledger2 and ledger3 + bkc.deleteLedger(lhs[1].getId()); + bkc.deleteLedger(lhs[2].getId()); + + LOG.info("Finished deleting the ledgers contains most entries."); + getGCThread().enableForceGC(); + getGCThread().triggerGC().get(); + + // after garbage collection, major compaction should not be executed + assertEquals(lastMajorCompactionTime, getGCThread().lastMajorCompactionTime); + assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime); + + // entry logs ([0,1,2].log) should be compacted. + for (File ledgerDirectory : tmpDirs.getDirs()) { + // Compaction of at least one of the files should not finish up + assertTrue("Not found entry log file ([0,1,2].log that should not have been compacted in ledgerDirectory: " + + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2)); + } + verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed()); + } @Test public void testForceMinorCompaction() throws Exception { @@ -660,8 +731,9 @@ public void testMinorCompactionWithNoWritableLedgerDirs() throws Exception { LOG.info("Finished deleting the ledgers contains most entries."); getGCThread().triggerGC().get(); - assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); - + if (useMetadataCache) { + assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); + } // after garbage collection, major compaction should not be executed assertEquals(lastMajorCompactionTime, getGCThread().lastMajorCompactionTime); assertEquals(lastMinorCompactionTime, getGCThread().lastMinorCompactionTime); @@ -710,8 +782,9 @@ public void testMinorCompactionWithNoWritableLedgerDirsButIsForceGCAllowWhenNoSp long lastMajorCompactionTime = getGCThread().lastMajorCompactionTime; assertFalse(getGCThread().enableMajorCompaction); assertTrue(getGCThread().enableMinorCompaction); - assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); - + if (useMetadataCache) { + assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); + } for (int i = 0; i < bookieCount(); i++) { BookieImpl bookie = ((BookieImpl) serverByIndex(i).getBookie()); bookie.getLedgerStorage().flush(); @@ -794,8 +867,9 @@ public void testMajorCompaction() throws Exception { LOG.info("Finished deleting the ledgers contains most entries."); getGCThread().enableForceGC(); getGCThread().triggerGC().get(); - assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); - + if (useMetadataCache) { + assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); + } // after garbage collection, minor compaction should not be executed assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime); assertTrue(getGCThread().lastMajorCompactionTime > lastMajorCompactionTime); @@ -845,8 +919,9 @@ public void testForceMajorCompaction() throws Exception { LOG.info("Finished deleting the ledgers contains most entries."); getGCThread().enableForceGC(); getGCThread().triggerGC().get(); - assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); - + if (useMetadataCache) { + assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); + } // after garbage collection, minor compaction should not be executed assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime); assertTrue(getGCThread().lastMajorCompactionTime > lastMajorCompactionTime); @@ -1020,8 +1095,9 @@ public void testCompactionWhenLedgerDirsAreFull() throws Exception { LOG.info("Finished deleting the ledgers contains most entries."); getGCThread().enableForceGC(); getGCThread().triggerGC().get(); - assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); - + if (useMetadataCache) { + assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); + } // after garbage collection, minor compaction should not be executed assertTrue(getGCThread().lastMinorCompactionTime > lastMinorCompactionTime); assertTrue(getGCThread().lastMajorCompactionTime > lastMajorCompactionTime); @@ -1106,8 +1182,9 @@ public void testCompactionSmallEntryLogs() throws Exception { getGCThread().enableForceGC(); getGCThread().triggerGC().get(); - assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); - + if (useMetadataCache) { + assertTrue(getGCThread().getEntryLogMetaMap() instanceof PersistentEntryLogMetadataMap); + } // entry logs (0.log) should not be compacted // entry logs ([1,2,3].log) should be compacted. for (File ledgerDirectory : bookieLedgerDirs()) { diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageTest.java index 1c8ae855e6b..ed1e30825b4 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageTest.java @@ -77,7 +77,7 @@ public void simple() throws Exception { File tmpDir = Files.createTempDirectory("junitTemporaryFolder").toFile(); Files.createDirectory(Paths.get(tmpDir.toString(), "subDir")); - KeyValueStorage db = storageFactory.newKeyValueStorage(tmpDir.toString(), "subDir", DbConfigType.Small, + KeyValueStorage db = storageFactory.newKeyValueStorage(tmpDir.toString(), "subDir", DbConfigType.Default, configuration); assertEquals(null, db.getFloor(toArray(3))); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/net/NetworkTopologyImplTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/net/NetworkTopologyImplTest.java index a6cc8c330a3..6122e51f0cd 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/net/NetworkTopologyImplTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/net/NetworkTopologyImplTest.java @@ -17,7 +17,9 @@ */ package org.apache.bookkeeper.net; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.util.Set; import org.junit.Test; @@ -97,4 +99,31 @@ public void getLeavesShouldReturnLeavesThatAreNotInExcludedScope() { assertTrue(leavesExcludingRack2Scope.contains(bookieRack0ScopeNode)); assertTrue(leavesExcludingRack2Scope.contains(bookieRack2ScopeNode)); } + + @Test + public void testInvalidRackName() { + NetworkTopologyImpl networkTopology = new NetworkTopologyImpl(); + String rack0Scope = ""; + BookieId bookieIdScopeRack0 = BookieId.parse("bookieIdScopeRack0"); + BookieNode bookieRack0ScopeNode = new BookieNode(bookieIdScopeRack0, rack0Scope); + + String rack1Scope = "/"; + BookieId bookieIdScopeRack1 = BookieId.parse("bookieIdScopeRack1"); + BookieNode bookieRack1ScopeNode = new BookieNode(bookieIdScopeRack1, rack1Scope); + + try { + networkTopology.add(bookieRack0ScopeNode); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("bookieIdScopeRack0, which is located at , is not a decendent of /", e.getMessage()); + } + + try { + networkTopology.add(bookieRack1ScopeNode); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("bookieIdScopeRack1, which is located at , is not a decendent of /", e.getMessage()); + } + + } } \ No newline at end of file diff --git a/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml b/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml deleted file mode 100644 index 7ba68aa96e2..00000000000 --- a/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - org.apache.bookkeeper.stats - codahale-metrics-provider - Apache BookKeeper :: Stats Providers :: Codahale Metrics - http://maven.apache.org - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.parent.version} - - - io.dropwizard.metrics - metrics-core - - - io.dropwizard.metrics - metrics-jvm - - - io.dropwizard.metrics - metrics-graphite - - - com.google.guava - guava - - - diff --git a/bookkeeper-stats-providers/pom.xml b/bookkeeper-stats-providers/pom.xml deleted file mode 100644 index faed3ad46de..00000000000 --- a/bookkeeper-stats-providers/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - 4.0.0 - bookkeeper-stats-providers - pom - Apache BookKeeper :: Stats Providers - - codahale-metrics-provider - prometheus-metrics-provider - - - diff --git a/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml b/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml deleted file mode 100644 index 3dfe19c0b2a..00000000000 --- a/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - ../.. - - org.apache.bookkeeper.stats - prometheus-metrics-provider - Apache BookKeeper :: Stats Providers :: Prometheus - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.parent.version} - - - - io.prometheus - simpleclient - - - - io.prometheus - simpleclient_hotspot - - - - io.prometheus - simpleclient_servlet - - - - io.netty - netty-common - - - - org.eclipse.jetty - jetty-servlet - - - - com.google.guava - guava - - - - com.yahoo.datasketches - sketches-core - - - - diff --git a/bookkeeper-stats/pom.xml b/bookkeeper-stats/pom.xml deleted file mode 100644 index b89cb4bcc97..00000000000 --- a/bookkeeper-stats/pom.xml +++ /dev/null @@ -1,63 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - org.apache.bookkeeper.stats - bookkeeper-stats-api - Apache BookKeeper :: Stats API - http://maven.apache.org - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven-javadoc-plugin.version} - - - none - org.apache.bookkeeper.stats - - - Bookkeeper Stats API - org.apache.bookkeeper.stats - - - - - - attach-javadocs - - jar - - - - - - - - - commons-configuration - commons-configuration - - - diff --git a/build.gradle b/build.gradle index 881981592f4..fc7b6654b68 100644 --- a/build.gradle +++ b/build.gradle @@ -29,6 +29,7 @@ plugins { id 'checkstyle' id 'org.nosphere.apache.rat' id 'com.github.spotbugs' + id 'org.owasp.dependencycheck' } subprojects { @@ -43,6 +44,18 @@ releaseArtifacts { fromProject("bookkeeper-dist-all") fromProject("bookkeeper-dist-bkctl") fromProject("bookkeeper-dist-server") + + tasks.withType(Jar) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + tasks.withType(Tar) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + tasks.withType(Zip) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } } releaseParams { // ReleaseExtension @@ -58,6 +71,13 @@ releaseParams { // ReleaseExtension } } +def skipDepCheck = [] +allprojects { + if (it.path.startsWith(':tests')) { + skipDepCheck << it.path + } +} + allprojects { apply from: "$rootDir/dependencies.gradle" if (it.path != ':circe-checksum:src:main:circe' @@ -70,6 +90,42 @@ allprojects { apply plugin: 'org.nosphere.apache.rat' apply plugin: "checkstyle" apply plugin: 'com.github.spotbugs' + + if (!it.path.startsWith(':tests')) { + apply plugin: 'org.owasp.dependencycheck' + + dependencyCheck { + // see https://jeremylong.github.io/DependencyCheck/dependency-check-gradle/configuration.html + // for configuration properties + suppressionFile = "$rootDir/src/owasp-dependency-check-suppressions.xml" + skipProjects = skipDepCheck + skipConfigurations = ["checkstyle", "spotbugs"] + failBuildOnCVSS = 7 + analyzers { + msbuildEnabled = false + rubygemsEnabled = false + pyDistributionEnabled = false + pyPackageEnabled = false + nuspecEnabled = false + nugetconfEnabled = false + assemblyEnabled = false + cmakeEnabled = false + composerEnabled = false + cpanEnabled = false + nodeEnabled = false + cocoapodsEnabled = false + swiftEnabled = false + swiftPackageResolvedEnabled = false + bundleAuditEnabled = false + golangDepEnabled = false + golangModEnabled = false + + nodeAudit.enabled = false + retirejs.enabled = false + } + } + } + checkstyle { toolVersion "${checkStyleVersion}" configFile file("$rootDir/buildtools/src/main/resources/bookkeeper/checkstyle.xml") @@ -87,14 +143,29 @@ allprojects { toolVersion = '3.1.8' excludeFilter = file("$rootDir/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml") reportLevel = 'high' + spotbugsTest.enabled = false + } + + + + tasks.withType(Jar) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + tasks.withType(Tar) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + tasks.withType(Zip) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE } task testJar(type: Jar, dependsOn: testClasses) { - classifier = 'tests' + archiveClassifier = 'tests' from sourceSets.test.output } - // TODO- Seperate rat plugins and other plugins configuration in seperate files + // TODO- Separate rat plugins and other plugins configuration in separate files rat { excludes = [ '**/.*/**', @@ -139,7 +210,9 @@ allprojects { } configurations { - testArtifacts.extendsFrom testRuntime + testArtifacts { + extendsFrom testRuntimeOnly + } } artifacts { @@ -148,6 +221,12 @@ allprojects { plugins.withType(DistributionPlugin) { distTar { + dependsOn(jar) + if (it.path.contains('bookkeeper-dist-all')) { + dependsOn(':bookkeeper-benchmark:jar') + dependsOn(':bookkeeper-common:jar') + dependsOn(':bookkeeper-common-allocator:jar') + } archiveClassifier = "bin" compression = Compression.GZIP archiveExtension = 'tar.gz' @@ -231,9 +310,24 @@ allprojects { systemProperty "currentVersion", project.rootProject.version testLogging { outputs.upToDateWhen {false} - showStandardStreams = true + showStandardStreams = !Boolean.getBoolean("testHideStandardOut") + } + } + dependencies { + implementation(enforcedPlatform(depLibs.nettyBom)) + testImplementation depLibs.log4jSlf4jImpl + } + tasks.register('writeClasspath') { + doLast { + buildDir.mkdirs() + new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" } } + + dependencies { + implementation(enforcedPlatform(depLibs.guavaBom)) + } + } repositories { diff --git a/buildtools/pom.xml b/buildtools/pom.xml deleted file mode 100644 index 1dced8306c6..00000000000 --- a/buildtools/pom.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - buildtools - Apache BookKeeper :: Build Tools - 4.15.0-SNAPSHOT - diff --git a/circe-checksum/pom.xml b/circe-checksum/pom.xml deleted file mode 100644 index 75fac546587..00000000000 --- a/circe-checksum/pom.xml +++ /dev/null @@ -1,236 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - .. - - - circe-checksum - nar - Apache BookKeeper :: Circe Checksum Library - Circe Checksum Library - - - dynamic - -msse4.2 -mpclmul - - - - - - com.google.guava - guava - - - - io.netty - netty-buffer - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - - - src/main/assembly/assembly.xml - - false - posix - - - - make-assembly - package - - single - - - - - - org.jacoco - jacoco-maven-plugin - ${jacoco-maven-plugin.version} - - - - com/scurrilous/circe/checksum/NarSystem* - - - - - - - - - - jdk-without-javah - - [10,) - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - - - default-nar-javah - none - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - -h - ${project.build.directory}/nar/javah-include - - - - - - - - mac - - - Mac OS X - - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - ${nar.runtime} - circe-checksum - - - jni - com.scurrilous.circe.checksum - - - - ${nar.cpp.optionSet} - false - false - full - - - - - - - - Linux - - - Linux - - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - ${nar.runtime} - circe-checksum - - - jni - com.scurrilous.circe.checksum - - - - ${nar.cpp.optionSet} - false - false - full - - - rt - - - - - - - - - diff --git a/conf/bk_rocksdb/default_rocksdb.conf b/conf/bk_rocksdb/default_rocksdb.conf new file mode 100644 index 00000000000..cbdb82c083a --- /dev/null +++ b/conf/bk_rocksdb/default_rocksdb.conf @@ -0,0 +1,11 @@ +[DBOptions] + # set by jni: options.setCreateIfMissing + create_if_missing=true + # set by jni: options.setInfoLogLevel + info_log_level=INFO_LEVEL + # set by jni: options.setKeepLogFileNum + keep_log_file_num=30 + +[CFOptions "default"] + # set by jni: options.setLogFileTimeToRoll + log_file_time_to_roll=86400 \ No newline at end of file diff --git a/conf/bk_rocksdb/entry_location_rocksdb.conf b/conf/bk_rocksdb/entry_location_rocksdb.conf new file mode 100644 index 00000000000..5e5bf5f4dbc --- /dev/null +++ b/conf/bk_rocksdb/entry_location_rocksdb.conf @@ -0,0 +1,51 @@ +[DBOptions] + # set by jni: options.setCreateIfMissing + create_if_missing=true + # set by jni: options.setInfoLogLevel + info_log_level=INFO_LEVEL + # set by jni: options.setKeepLogFileNum + keep_log_file_num=30 + # set by jni: options.setLogFileTimeToRoll + log_file_time_to_roll=86400 + # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism + max_background_jobs=32 + # set by jni: options.setMaxSubcompactions + max_subcompactions=1 + # set by jni: options.setMaxTotalWalSize + max_total_wal_size=536870912 + # set by jni: options.setMaxOpenFiles + max_open_files=-1 + # set by jni: options.setDeleteObsoleteFilesPeriodMicros + delete_obsolete_files_period_micros=3600000000 + +[CFOptions "default"] + # set by jni: options.setCompressionType + compression=kLZ4Compression + # set by jni: options.setWriteBufferSize + write_buffer_size=67108864 + # set by jni: options.setMaxWriteBufferNumber + max_write_buffer_number=4 + # set by jni: options.setNumLevels + num_levels=7 + # set by jni: options.setLevelZeroFileNumCompactionTrigger + level0_file_num_compaction_trigger=4 + # set by jni: options.setMaxBytesForLevelBase + max_bytes_for_level_base=268435456 + # set by jni: options.setTargetFileSizeBase + target_file_size_base=67108864 + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setBlockSize + block_size=65536 + # set by jni: tableOptions.setBlockCache + block_cache=206150041 + # set by jni: tableOptions.setFormatVersion + format_version=2 + # set by jni: tableOptions.setChecksumType + checksum=kxxHash + # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder] + filter_policy=rocksdb.BloomFilter:10:false + # set by jni: tableOptions.setCacheIndexAndFilterBlocks + cache_index_and_filter_blocks=true + # set by jni: options.setLevelCompactionDynamicLevelBytes + level_compaction_dynamic_level_bytes=true \ No newline at end of file diff --git a/conf/bk_rocksdb/ledger_metadata_rocksdb.conf b/conf/bk_rocksdb/ledger_metadata_rocksdb.conf new file mode 100644 index 00000000000..cbdb82c083a --- /dev/null +++ b/conf/bk_rocksdb/ledger_metadata_rocksdb.conf @@ -0,0 +1,11 @@ +[DBOptions] + # set by jni: options.setCreateIfMissing + create_if_missing=true + # set by jni: options.setInfoLogLevel + info_log_level=INFO_LEVEL + # set by jni: options.setKeepLogFileNum + keep_log_file_num=30 + +[CFOptions "default"] + # set by jni: options.setLogFileTimeToRoll + log_file_time_to_roll=86400 \ No newline at end of file diff --git a/conf/bk_server.conf b/conf/bk_server.conf index 801976ebbb3..f83a46e852c 100755 --- a/conf/bk_server.conf +++ b/conf/bk_server.conf @@ -740,6 +740,7 @@ gcEntryLogMetadataCacheEnabled=false # dbStorage_rocksDB_numFilesInLevel0=4 # dbStorage_rocksDB_maxSizeInLevel1MB=256 # dbStorage_rocksDB_logPath= +# dbStorage_rocksDB_format_version=2 ############################################## Metadata Services ############################################## diff --git a/cpu-affinity/pom.xml b/cpu-affinity/pom.xml deleted file mode 100644 index c1f8eba9d92..00000000000 --- a/cpu-affinity/pom.xml +++ /dev/null @@ -1,226 +0,0 @@ - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - .. - - - cpu-affinity - nar - Apache BookKeeper :: CPU Affinity Library - CPU Affinity Library - - - dynamic - - - - - com.google.guava - guava - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - - - src/main/assembly/assembly.xml - - false - posix - - - - make-assembly - package - - single - - - - - - org.apache.rat - apache-rat-plugin - - - - **/src/test/resources/proc_cpuinfo.txt - - - - - - - - - - jdk-without-javah - - [10,) - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - - - default-nar-javah - none - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - 1.8 - 1.8 - - - - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - -h - ${project.build.directory}/nar/javah-include - - - - - - - - mac - - - Mac OS X - - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - ${nar.runtime} - cpu-affinity - - - jni - org.apache.bookkeeper.utils.affinity - - - - ${nar.cpp.optionSet} - false - false - full - - - - - - - - - Linux - - - Linux - - - - - - com.github.maven-nar - nar-maven-plugin - ${nar-maven-plugin.version} - true - - ${nar.runtime} - cpu-affinity - - - jni - org.apache.bookkeeper.utils.affinity - - - - ${nar.cpp.optionSet} - false - false - full - - - rt - - - - - - - - - diff --git a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java index 67c8679ab12..218e9f1af5e 100644 --- a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java +++ b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java @@ -20,8 +20,6 @@ */ package org.apache.bookkeeper.common.util.affinity.impl; -import static com.google.common.base.Preconditions.checkArgument; - import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.File; @@ -50,7 +48,7 @@ public class NativeUtils { value = "OBL_UNSATISFIED_OBLIGATION", justification = "work around for java 9: https://github.com/spotbugs/spotbugs/issues/493") public static void loadLibraryFromJar(String path) throws Exception { - checkArgument(path.startsWith("/"), "absolute path must start with /"); + com.google.common.base.Preconditions.checkArgument(path.startsWith("/"), "absolute path must start with /"); String[] parts = path.split("/"); String filename = (parts.length > 0) ? parts[parts.length - 1] : null; diff --git a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/ProcessorsInfo.java b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/ProcessorsInfo.java index 8a812a84d1a..a1b3963d2c7 100644 --- a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/ProcessorsInfo.java +++ b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/ProcessorsInfo.java @@ -20,8 +20,6 @@ */ package org.apache.bookkeeper.common.util.affinity.impl; -import static com.google.common.base.Preconditions.checkArgument; - import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -80,8 +78,8 @@ static ProcessorsInfo parseCpuInfo(String cpuInfoString) { } } - checkArgument(cpuId >= 0); - checkArgument(coreId >= 0); + com.google.common.base.Preconditions.checkArgument(cpuId >= 0); + com.google.common.base.Preconditions.checkArgument(coreId >= 0); pi.cpus.put(cpuId, coreId); } diff --git a/dependencies.gradle b/dependencies.gradle index 075b3bb7091..c8aeb48cdb6 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -45,7 +45,7 @@ depVersions = [ gradleTooling: "4.0.1", grpc: "1.42.1", groovy: "2.5.8", - guava: "30.0-jre", + guava: "31.0.1-jre", hamcrest: "1.3", hdrhistogram: "2.1.10", httpclient: "4.5.13", @@ -66,16 +66,16 @@ depVersions = [ lombok: "1.18.20", lz4: "1.3.0", mockito: "3.0.0", - netty: "4.1.72.Final", - nettyTcnativeBoringSsl: "2.0.46.Final", + netty: "4.1.74.Final", + nettyTcnativeBoringSsl: "2.0.48.Final", powermock: "2.0.2", prometheus: "0.8.1", - protobuf: "3.14.0", + protobuf: "3.16.1", reflections: "0.9.11", - rocksDb: "6.22.1.1", + rocksDb: "6.27.3", rxjava: "3.0.1", slf4j: "1.7.32", - snakeyaml: "1.19", + snakeyaml: "1.30", spotbugsAnnotations: "3.1.8", protocGenGrpcJava: "1.12.0", shrinkwrap:"3.1.4", @@ -139,6 +139,7 @@ depLibs = [ exclude group: 'com.google.guava', module: 'guava' }, groovy: "org.codehaus.groovy:groovy-all:${depVersions.groovy}", + guavaBom: "com.google.guava:guava-bom:${depVersions.guava}", guava: dependencies.create("com.google.guava:guava:${depVersions.guava}"){ exclude group: 'com.google.code.findbugs', module: 'jsr305' }, @@ -176,6 +177,7 @@ depLibs = [ metricsJvm: "io.dropwizard.metrics:metrics-jvm:${depVersions.dropwizard}", metricsGraphite: "io.dropwizard.metrics:metrics-graphite:${depVersions.dropwizard}", mockito: "org.mockito:mockito-core:${depVersions.mockito}", + nettyBom: "io.netty:netty-bom:${depVersions.netty}", nettyBuffer: "io.netty:netty-buffer:${depVersions.netty}", nettyCommon: "io.netty:netty-common:${depVersions.netty}", nettyHandler: "io.netty:netty-handler:${depVersions.netty}", diff --git a/dev/check-all-licenses b/dev/check-all-licenses index 1f548c3cef1..03eeb51c65b 100755 --- a/dev/check-all-licenses +++ b/dev/check-all-licenses @@ -27,7 +27,7 @@ set -e -x HERE=$(dirname $0) BOOKKEEPER_DIST=$HERE/../bookkeeper-dist -$HERE/check-binary-license $BOOKKEEPER_DIST/server/target/bookkeeper-server-*-bin.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/all/target/bookkeeper-all-*-bin.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/bkctl/target/bkctl-*-bin.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/server/build/distributions/bookkeeper-server-*.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/all/build/distributions/bookkeeper-all-*.tar.gz +$HERE/check-binary-license $BOOKKEEPER_DIST/bkctl/build/distributions/bkctl-*.tar.gz diff --git a/dev/check-all-licenses-gradle b/dev/check-all-licenses-gradle deleted file mode 100755 index 03eeb51c65b..00000000000 --- a/dev/check-all-licenses-gradle +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Script to check licenses on a binary tarball. -# It extracts the list of bundled jars, the NOTICE, and the LICENSE -# files. It checked that every non-bk jar bundled is mentioned in the -# LICENSE file. It checked that all jar files mentioned in NOTICE and -# LICENSE are actually bundled. - -# all error fatal -set -e -x - -HERE=$(dirname $0) -BOOKKEEPER_DIST=$HERE/../bookkeeper-dist -$HERE/check-binary-license $BOOKKEEPER_DIST/server/build/distributions/bookkeeper-server-*.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/all/build/distributions/bookkeeper-all-*.tar.gz -$HERE/check-binary-license $BOOKKEEPER_DIST/bkctl/build/distributions/bkctl-*.tar.gz - diff --git a/dev/ci-tool b/dev/ci-tool new file mode 100755 index 00000000000..604f4fb5f0c --- /dev/null +++ b/dev/ci-tool @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# shell function library for Bookkeeper CI builds + +# lists all available functions in this tool +function ci_list_functions() { + declare -F | awk '{print $NF}' | sort | egrep '^ci_' | sed 's/^ci_//' +} + +# prints thread dumps for all running JVMs +# used in CI when a job gets cancelled because of a job timeout +function ci_print_thread_dumps() { + for java_pid in $(jps -q -J-XX:+PerfDisableSharedMem); do + echo "----------------------- pid $java_pid -----------------------" + cat /proc/$java_pid/cmdline | xargs -0 echo + jcmd $java_pid Thread.print -l + jcmd $java_pid GC.heap_info + done + return 0 +} + +if [ -z "$1" ]; then + echo "usage: $0 [ci_tool_function_name]" + echo "Available ci tool functions:" + ci_list_functions + exit 1 +fi +ci_function_name="ci_$1" +shift + +if [[ "$(LC_ALL=C type -t $ci_function_name)" == "function" ]]; then + eval "$ci_function_name" "$@" +else + echo "Invalid ci tool function" + echo "Available ci tool functions:" + ci_list_functions + exit 1 +fi \ No newline at end of file diff --git a/dev/common.sh b/dev/common.sh index d4f3d3f2efa..49825406a55 100644 --- a/dev/common.sh +++ b/dev/common.sh @@ -21,11 +21,7 @@ # function get_bk_version() { - bk_version=$(mvn -q \ - -Dexec.executable="echo" \ - -Dexec.args='${project.version}' \ - --non-recursive \ - org.codehaus.mojo:exec-maven-plugin:1.3.1:exec) + bk_version=$(${BK_HOME}/gradlew --no-daemon properties -q | grep "version:" | awk '{print $2}') echo ${bk_version} } diff --git a/dev/docker/ci.sh b/dev/docker/ci.sh deleted file mode 100755 index 75588a3d3e2..00000000000 --- a/dev/docker/ci.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -x -u - -SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) - -export IMAGE_NAME="bookkeeper/dev" - -pushd ${SCRIPT_DIR} - -docker build --rm=true -t ${IMAGE_NAME} . - -popd - -if [ "$(uname -s)" == "Linux" ]; then - USER_NAME=${SUDO_USER:=$USER} - USER_ID=$(id -u "${USER_NAME}") - GROUP_ID=$(id -g "${USER_NAME}") - LOCAL_HOME="/home/${USER_NAME}" -else # boot2docker uid and gid - USER_NAME=$USER - USER_ID=1000 - GROUP_ID=50 - LOCAL_HOME="/Users/${USER_NAME}" -fi - -docker build -t "${IMAGE_NAME}-${USER_NAME}" - < \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +APP_BASE_NAME=${0##*/} # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +121,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -106,80 +140,95 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/metadata-drivers/etcd/pom.xml b/metadata-drivers/etcd/pom.xml deleted file mode 100644 index 20bfe459995..00000000000 --- a/metadata-drivers/etcd/pom.xml +++ /dev/null @@ -1,147 +0,0 @@ - - - - - org.apache.bookkeeper.metadata.drivers - metadata-drivers-parent - 4.15.0-SNAPSHOT - .. - - 4.0.0 - org.apache.bookkeeper.metadata.drivers - metadata-stores-etcd - Apache BookKeeper :: Metadata Drivers:: Etcd - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - - io.etcd - jetcd-core - ${etcd.version} - - - io.grpc - * - - - - - - io.grpc - grpc-all - ${grpc.version} - - - org.bouncycastle - bcpkix-jdk15on - - - io.grpc - grpc-okhttp - - - - - - org.arquillian.cube - arquillian-cube-docker - ${arquillian-cube.version} - - - com.github.docker-java - * - - - test - - - org.jboss.arquillian.junit - arquillian-junit-standalone - ${arquillian-junit.version} - - - com.github.docker-java - * - - - test - - - org.testcontainers - testcontainers - test - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test-jar - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - integrationTests - - - integrationTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - false - - - - - - - diff --git a/metadata-drivers/pom.xml b/metadata-drivers/pom.xml deleted file mode 100644 index e44e116edeb..00000000000 --- a/metadata-drivers/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.metadata.drivers - metadata-drivers-parent - pom - Apache BookKeeper :: Metadata Drivers :: Parent - - etcd - - diff --git a/microbenchmarks/build.gradle b/microbenchmarks/build.gradle index 526c170753e..a58eea03b75 100644 --- a/microbenchmarks/build.gradle +++ b/microbenchmarks/build.gradle @@ -41,7 +41,6 @@ jar { manifest { attributes 'Main-Class': 'org.openjdk.jmh.Main' } - println configurations.runtimeClasspath.collect().size() from { configurations.compileClasspath.collect {it.isDirectory() ? it : zipTree(it) } } @@ -49,4 +48,4 @@ jar { exclude 'META-INF/*.SF' exclude 'META-INF/*.DSA' archiveBaseName = 'microbenchmarks' -} \ No newline at end of file +} diff --git a/microbenchmarks/pom.xml b/microbenchmarks/pom.xml deleted file mode 100644 index dc0093954db..00000000000 --- a/microbenchmarks/pom.xml +++ /dev/null @@ -1,129 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - microbenchmarks - Apache BookKeeper :: microbenchmarks - http://maven.apache.org - - benchmarks - - - - org.openjdk.jmh - jmh-core - - - org.openjdk.jmh - jmh-generator-annprocess - provided - - - org.slf4j - slf4j-api - - - org.apache.logging.log4j - log4j-1.2-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.bookkeeper - bookkeeper-server - ${project.parent.version} - compile - jar - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.version} - - - org.apache.bookkeeper.stats - codahale-metrics-provider - ${project.version} - - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${javac.target} - ${javac.target} - ${javac.target} - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - ${uberjar.name} - - - org.openjdk.jmh.Main - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - true - - - - - diff --git a/pom.xml b/pom.xml deleted file mode 100644 index ac7658821b0..00000000000 --- a/pom.xml +++ /dev/null @@ -1,1164 +0,0 @@ - - - - - org.apache - apache - 19 - - 4.0.0 - org.apache.bookkeeper - 4.15.0-SNAPSHOT - bookkeeper - pom - Apache BookKeeper :: Parent - http://bookkeeeper.apache.org - 2011 - - - Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - scm:git:https://github.com/apache/bookkeeper.git - scm:git:https://github.com/apache/bookkeeper.git - https://github.com/apache/bookkeeper - branch-4.13 - - - JIRA - https://issues.apache.org/jira/browse/BOOKKEEPER - - - Jenkins - https://builds.apache.org/job/bookkeeper-master - - - buildtools - circe-checksum - bookkeeper-common - bookkeeper-common-allocator - stats - - bookkeeper-stats - bookkeeper-proto - bookkeeper-server - bookkeeper-benchmark - bookkeeper-stats-providers - bookkeeper-http - stream - tools - cpu-affinity - metadata-drivers - bookkeeper-dist - shaded - microbenchmarks - tests - - - - BookKeeper User - user-subscribe@bookkeeper.apache.org - user-unsubscribe@bookkeeper.apache.org - user@bookkeeper.apache.org - http://www.mail-archive.com/user@bookkeeper.apache.org - - - BookKeeper Dev - dev-subscribe@bookkeeper.apache.org - dev-unsubscribe@bookkeeper.apache.org - dev@bookkeeper.apache.org - http://www.mail-archive.com/dev@bookkeeper.apache.org - - - BookKeeper Commits - commits-subscribe@bookkeeper.apache.org - commits-unsubscribe@bookkeeper.apache.org - commits@bookkeeper.apache.org - http://www.mail-archive.com/commits@bookkeeper.apache.org - - - - - The Apache BookKeeper Team - dev@bookkeeper.apache.org - http://bookkeeper.apache.org - Apache Software Foundation - http://www.apache.org - - - - UTF-8 - UTF-8 - 1.8 - true - 2 - - - 1.18.2 - 1.6.0.Final - 3.0.1 - 1.2 - 4.1 - 1.6 - 1.10 - 1.19 - 2.6 - 3.6 - 2.7 - 1.0.2.1 - 5.1.0 - 3.2.5 - 0.5.11 - 2.7.0 - 3.0.2 - 2.4.0 - 1.42.1 - 30.0-jre - 1.1.1 - 2.10.0 - 1.3 - 2.1.10 - 2.11.0 - 1.78 - 9.4.43.v20210629 - 1.19 - 2.8.2 - 3.2.7 - 4.12 - 0.14.2 - 1.18.20 - 2.17.1 - 1.3.0 - 3.0.0 - 4.1.72.Final - 2.0.46.Final - 9.1.3 - 2.0.2 - 0.8.1 - 0.8.3 - 4.5.13 - 3.14.0 - 3.14.0 - ${grpc.version} - 0.9.11 - 6.22.1.1 - 3.0.1 - 1.7.32 - 1.19 - 3.1.8 - 1.3.2 - 1.15.1 - 3.9.8 - 3.6.2 - 1.1.7 - 2.1.2 - - 0.12 - 2.7 - 4.3.0 - 1.4.13 - 1.6.0 - 1.6 - 0.8.0 - 1.8 - 3.1.0 - 3.0.0 - 2.5 - 3.8.1 - 3.0.2 - 2.7 - 2.5.1 - 2.4 - 3.1.1 - 3.2.0 - 2.2.1 - 3.0.0-M5 - 3.5.2 - 1.4.1.Final - 0.6.1 - 6.19 - 3.1.8 - 1 - 4.0.0 - - - - - - - - com.github.spotbugs - spotbugs-annotations - ${spotbugs-annotations.version} - - - javax.annotation - javax.annotation-api - ${javax-annotations-api.version} - - - com.google.code.findbugs - jsr305 - ${google.code.version} - - - com.google.errorprone - error_prone_annotations - ${google.errorprone.version} - - - org.projectlombok - lombok - ${lombok.version} - - - org.inferred - freebuilder - ${freebuilder.version} - true - - - - - org.slf4j - slf4j-api - ${slf4j.version} - - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - - - - - commons-cli - commons-cli - ${commons-cli.version} - - - commons-codec - commons-codec - ${commons-codec.version} - - - commons-configuration - commons-configuration - ${commons-configuration.version} - - - commons-io - commons-io - ${commons-io.version} - - - commons-lang - commons-lang - ${commons-lang.version} - - - com.google.guava - guava - ${guava.version} - - - org.apache.commons - commons-collections4 - ${commons-collections4.version} - - - org.apache.commons - commons-compress - ${commons-compress.version} - - - org.apache.commons - commons-lang3 - ${commons-lang3.version} - - - - org.bouncycastle - bc-fips - ${bouncycastle.version} - - - - org.reflections - reflections - ${reflections.version} - - - - - net.jpountz.lz4 - lz4 - ${lz4.version} - - - - - net.java.dev.jna - jna - ${jna.version} - - - - - org.yaml - snakeyaml - ${snakeyaml.version} - - - - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - javax.servlet - javax.servlet-api - ${servlet-api.version} - - - com.fasterxml.jackson.module - jackson-module-paranamer - ${jackson.version} - - - com.fasterxml.jackson.module - jackson-module-scala_2.11 - ${jackson.version} - - - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - - - org.apache.thrift - libthrift - ${libthrift.version} - - - org.apache.tomcat.embed - tomcat-embed-core - - - javax.annotation - javax.annotation-api - - - - - - - io.netty - netty-common - ${netty.version} - - - io.netty - netty-buffer - ${netty.version} - - - io.netty - netty-transport - ${netty.version} - - - io.netty - netty-handler - ${netty.version} - - - io.netty - netty-transport-native-epoll - ${netty.version} - - - io.netty - netty-transport-native-epoll - ${netty.version} - linux-x86_64 - - - io.netty - netty-codec-dns - ${netty.version} - - - io.netty - netty-codec-http - ${netty.version} - - - io.netty - netty-codec-http2 - ${netty.version} - - - io.netty - netty-codec-socks - ${netty.version} - - - io.netty - netty-handler-proxy - ${netty.version} - - - io.netty - netty-resolver - ${netty.version} - - - io.netty - netty-resolver-dns - ${netty.version} - - - io.netty - netty-tcnative-boringssl-static - ${netty-boringssl.version} - - - - - io.grpc - grpc-bom - ${grpc.version} - pom - import - - - - - org.rocksdb - rocksdbjni - ${rocksdb.version} - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - net.java.dev.javacc - javacc - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - test-jar - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - - - org.xerial.snappy - snappy-java - ${snappy.version} - - - - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.zookeeper - zookeeper - - - - - - - io.vertx - vertx-core - ${vertx.version} - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - - - io.vertx - vertx-web - ${vertx.version} - - - org.eclipse.jetty - jetty-server - ${jetty.version} - - - org.eclipse.jetty - jetty-webapp - ${jetty.version} - - - org.eclipse.jetty - jetty-servlet - ${jetty.version} - - - - - org.jctools - jctools-core - ${jctools.version} - - - - - io.dropwizard.metrics - metrics-core - ${dropwizard.version} - - - - - io.dropwizard.metrics - metrics-jvm - ${dropwizard.version} - - - io.dropwizard.metrics - metrics-graphite - ${dropwizard.version} - - - - io.prometheus - simpleclient - ${prometheus.version} - - - io.prometheus - simpleclient_hotspot - ${prometheus.version} - - - io.prometheus - simpleclient_servlet - ${prometheus.version} - - - - com.yahoo.datasketches - sketches-core - ${datasketches.version} - - - - - org.apache.httpcomponents - httpclient - ${httpclient.version} - - - - - com.beust - jcommander - ${jcommander.version} - - - - - org.hdrhistogram - HdrHistogram - ${hdrhistogram.version} - - - - - junit - junit - ${junit.version} - - - org.hamcrest - hamcrest-all - ${hamcrest.version} - - - org.jmock - jmock - ${jmock.version} - - - org.mockito - mockito-core - ${mockito.version} - - - org.powermock - powermock-api-mockito2 - ${powermock.version} - - - org.powermock - powermock-module-junit4 - ${powermock.version} - - - org.apache.hadoop - hadoop-minikdc - ${hadoop.minikdc.version} - - - org.arquillian.cube - arquillian-cube-docker - ${arquillian-cube.version} - - - com.github.docker-java - * - - - - - org.jboss.arquillian.junit - arquillian-junit-standalone - ${arquillian-junit.version} - - - com.github.docker-java - * - - - - - org.codehaus.groovy - groovy-all - ${groovy.version} - pom - - - org.jboss.shrinkwrap.resolver - shrinkwrap-resolver-impl-maven - ${shrinkwrap.version} - - - org.jboss.shrinkwrap.resolver - shrinkwrap-resolver-api - ${shrinkwrap.version} - - - org.testcontainers - testcontainers - ${testcontainers.version} - - - - - org.openjdk.jmh - jmh-core - ${jmh.version} - - - org.openjdk.jmh - jmh-generator-annprocess - ${jmh.version} - - - - - - - - - org.projectlombok - lombok - provided - - - com.github.spotbugs - spotbugs-annotations - provided - - - - - org.slf4j - slf4j-api - - - commons-configuration - commons-configuration - - - - - junit - junit - test - - - org.hamcrest - hamcrest-all - test - - - org.apache.logging.log4j - log4j-core - test - - - org.apache.logging.log4j - log4j-slf4j-impl - test - - - org.mockito - mockito-core - test - - - org.powermock - powermock-api-mockito2 - test - - - org.powermock - powermock-module-junit4 - test - - - - - - - kr.motd.maven - os-maven-plugin - ${os-maven-plugin.version} - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - ${maven-checkstyle-plugin.version} - - - com.puppycrawl.tools - checkstyle - ${puppycrawl.checkstyle.version} - - - - buildtools/src/main/resources/bookkeeper/checkstyle.xml - buildtools/src/main/resources/bookkeeper/suppressions.xml - UTF-8 - true - true - false - true - - - - checkstyle - validate - - check - - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${session.executionRootDirectory}/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml - - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${javac.target} - ${javac.target} - - -Werror - -Xlint:deprecation - -Xlint:unchecked - - -Xpkginfo:always - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid - ${redirectTestOutputToFile} - ${forkCount.variable} - false - false - 1800 - ${testRetryCount} - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven-javadoc-plugin.version} - - -notimestamp - - none - org.apache.bookkeeper.client:org.apache.bookkeeper.client.api:org.apache.bookkeeper.common.annotation:org.apache.bookkeeper.conf:org.apache.bookkeeper.feature:org.apache.bookkeeper.stats - - - Bookkeeper Client - org.apache.bookkeeper.client:org.apache.bookkeeper.common.annotation:org.apache.bookkeeper.conf:org.apache.bookkeeper.feature - - - Bookkeeper Client (New Fluent API - Experimental) - org.apache.bookkeeper.client.api - - - Bookkeeper Stats API - - org.apache.bookkeeper.stats - - - Bookkeeper Stats Providers - org.apache.bookkeeper.stats.codahale:org.apache.bookkeeper.stats.prometheus - - - BookKeeper Java API (version ${project.version}) - site/_site/overview/index.html - package - false - - - - aggregate - - aggregate - - site - - - - - org.apache.maven.plugins - maven-source-plugin - ${maven-source-plugin.version} - - - attach-sources - - jar - - - - - - org.apache.rat - apache-rat-plugin - ${apache-rat-plugin.version} - - - - **/.idea/** - - - .git/**/* - .github/**/* - **/.gitignore - - - **/.svn/**/* - - - **/target/**/* - - - **/README.md - **/README.rst - **/apidocs/* - **/src/main/resources/deps/** - **/META-INF/** - - - **/.classpath - **/.project - **/.checkstyle - **/.settings/* - **/*.iml - **/*.iws - **/*.ipr - - - .repository/** - - - site/** - site2/** - - - **/org/apache/distributedlog/thrift/* - - - **/*.log - - - data/** - - - dev/.vagrant/** - - - **/proto/**.py - - **/python/.coverage - **/python/.Python - **/python/bin/** - **/python/include/** - **/python/lib/** - **/**.pyc - **/.nox/** - **/.pytest_cache/** - **/__pycache__/** - **/bookkeeper.egg-info/** - **/pip-selfcheck.json - - - **/test_conf_2.conf - - true - - - - - - - - code-coverage - - - - org.eluder.coveralls - coveralls-maven-plugin - ${coveralls-maven-plugin.version} - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - @{argLine} -Xmx2G -Djava.net.preferIPv4Stack=true - ${redirectTestOutputToFile} - ${forkCount.variable} - false - 1800 - - true - - - - org.jacoco - jacoco-maven-plugin - 0.8.0 - - - - prepare-agent - - - - - - - - - - - dev - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - -Xmx2G -Djava.net.preferIPv4Stack=true - false - ${forkCount.variable} - false - 1800 - false - 0 - - - - - - - dev-debug - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - -Xmx2G -Djava.net.preferIPv4Stack=true -Dbookkeeper.root.logger=DEBUG,CONSOLE - false - ${forkCount.variable} - false - 1800 - false - - - - - - - - jdk11-no-spotbugs - - [11,) - - - - - com.github.spotbugs - spotbugs-maven-plugin - - true - - - - - - - - apache-release - - - - maven-assembly-plugin - - - source-release-assembly - - - true - - - - - - - - - diff --git a/settings.gradle b/settings.gradle index 3c72c6e5f80..6043b1739f6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -25,11 +25,14 @@ pluginManagement { id "com.github.spotbugs" version "${spotbugsPlugin}" id "com.adarshr.test-logger" version "${testLogger}" id "org.gradle.test-retry" version "${testRetry}" + id "org.owasp.dependencycheck" version "${owaspPlugin}" } } rootProject.name = 'bookkeeper' + + include(':bookkeeper-benchmark', ':bookkeeper-tools', ':bookkeeper-tools-framework', @@ -88,7 +91,6 @@ include(':bookkeeper-benchmark', 'tests:backward-compat:recovery-no-password', 'tests:backward-compat:upgrade', 'tests:backward-compat:upgrade-direct', - 'tests:backward-compat:yahoo-custom-version', 'tests:integration-tests-base', 'tests:integration-tests-topologies', 'tests:integration-tests-utils', @@ -98,8 +100,7 @@ include(':bookkeeper-benchmark', 'tests:scripts', 'tests:shaded:bookkeeper-server-shaded-test', 'tests:shaded:bookkeeper-server-tests-shaded-test', - 'tests:shaded:distributedlog-core-shaded-test', - 'tools:framework') + 'tests:shaded:distributedlog-core-shaded-test') project(':bookkeeper-tools').projectDir = file('tools/all') project(':bookkeeper-tools-framework').projectDir = file('tools/framework') diff --git a/shaded/bookkeeper-server-shaded/pom.xml b/shaded/bookkeeper-server-shaded/pom.xml deleted file mode 100644 index 0644f91bcf9..00000000000 --- a/shaded/bookkeeper-server-shaded/pom.xml +++ /dev/null @@ -1,128 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - shaded-parent - 4.15.0-SNAPSHOT - .. - - bookkeeper-server-shaded - Apache BookKeeper :: Shaded :: bookkeeper-server-shaded - - UTF-8 - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - true - true - false - - - com.google.guava:guava - com.google.protobuf:protobuf-java - org.apache.bookkeeper:bookkeeper-common - org.apache.bookkeeper:bookkeeper-common-allocator - org.apache.bookkeeper:cpu-affinity - org.apache.bookkeeper:bookkeeper-tools-framework - org.apache.bookkeeper:bookkeeper-proto - org.apache.bookkeeper:bookkeeper-server - org.apache.bookkeeper:circe-checksum - org.apache.bookkeeper.stats:bookkeeper-stats-api - - - - - com.google - org.apache.bookkeeper.shaded.com.google - - - - - - - - org.codehaus.mojo - license-maven-plugin - ${license-maven-plugin.version} - - false - ${project.basedir} - - - - update-pom-license - - update-file-header - - package - - apache_v2 - - dependency-reduced-pom.xml - - - - - - - maven-clean-plugin - ${maven-clean-plugin.version} - - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - diff --git a/shaded/bookkeeper-server-tests-shaded/pom.xml b/shaded/bookkeeper-server-tests-shaded/pom.xml deleted file mode 100644 index a5cd9f6a177..00000000000 --- a/shaded/bookkeeper-server-tests-shaded/pom.xml +++ /dev/null @@ -1,147 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - shaded-parent - 4.15.0-SNAPSHOT - .. - - bookkeeper-server-tests-shaded - Apache BookKeeper :: Shaded :: bookkeeper-server-tests-shaded - - UTF-8 - - - - org.apache.bookkeeper - bookkeeper-server - test-jar - ${project.version} - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - org.apache.bookkeeper - bookkeeper-common - - - org.apache.bookkeeper - bookkeeper-tools-framework - - - org.apache.bookkeeper - bookkeeper-proto - - - org.apache.bookkeeper - bookkeeper-server - - - org.apache.bookkeeper - circe-checksum - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - true - true - false - - - com.google.guava:guava - com.google.protobuf:protobuf-java - org.apache.bookkeeper:bookkeeper-server:test-jar:tests - - - - - - com.google - org.apache.bookkeeper.shaded.com.google - - - - - - - - org.codehaus.mojo - license-maven-plugin - ${license-maven-plugin.version} - - false - ${project.basedir} - - - - update-pom-license - - update-file-header - - package - - apache_v2 - - dependency-reduced-pom.xml - - - - - - - maven-clean-plugin - ${maven-clean-plugin.version} - - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - diff --git a/shaded/distributedlog-core-shaded/pom.xml b/shaded/distributedlog-core-shaded/pom.xml deleted file mode 100644 index c1cf1fef955..00000000000 --- a/shaded/distributedlog-core-shaded/pom.xml +++ /dev/null @@ -1,251 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - shaded-parent - 4.15.0-SNAPSHOT - .. - - org.apache.distributedlog - distributedlog-core-shaded - Apache BookKeeper :: Shaded :: distributedlog-core-shaded - - UTF-8 - - - - org.apache.distributedlog - distributedlog-core - ${project.version} - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - io.netty - netty-common - - - io.netty - netty-buffer - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - true - true - false - - - commons-codec:commons-codec - commons-cli:commons-cli - commons-io:commons-io - commons-lang:commons-lang - commons-logging:commons-logging - com.fasterxml.jackson.core:jackson-core - com.fasterxml.jackson.core:jackson-databind - com.fasterxml.jackson.core:jackson-annotations - com.google.guava:guava - com.google.protobuf:protobuf-java - net.java.dev.jna:jna - net.jpountz.lz4:lz4 - org.apache.bookkeeper:bookkeeper-common - org.apache.bookkeeper:bookkeeper-common-allocator - org.apache.bookkeeper:cpu-affinity - org.apache.bookkeeper:bookkeeper-tools-framework - org.apache.bookkeeper:bookkeeper-proto - org.apache.bookkeeper:bookkeeper-server - org.apache.bookkeeper:circe-checksum - org.apache.bookkeeper.http:http-server - org.apache.bookkeeper.stats:bookkeeper-stats-api - org.apache.commons:commons-collections4 - org.apache.commons:commons-lang3 - org.apache.distributedlog:distributedlog-common - org.apache.distributedlog:distributedlog-core - org.apache.distributedlog:distributedlog-protocol - org.apache.httpcomponents:httpclient - org.apache.httpcomponents:httpcore - org.apache.thrift:libthrift - org.apache.zookeeper:zookeeper - org.apache.zookeeper:zookeeper-jute - org.rocksdb:rocksdbjni - - - - - - org.apache.commons.cli - dlshade.org.apache.commons.cli - - - org.apache.commons.codec - dlshade.org.apache.commons.codec - - - org.apache.commons.collections4 - dlshade.org.apache.commons.collections4 - - - org.apache.commons.lang - dlshade.org.apache.commons.lang - - - org.apache.commons.lang3 - dlshade.org.apache.commons.lang3 - - - org.apache.commons.logging - dlshade.org.apache.commons.logging - - - org.apache.commons.io - dlshade.org.apache.commons.io - - - - org.apache.httpcomponents - dlshade.org.apache.httpcomponents - - - org.apache.http - dlshade.org.apache.http - - - - org.apache.thrift - dlshade.org.apache.thrift - - - - org.apache.zookeeper - dlshade.org.apache.zookeeper - - - org.apache.jute - dlshade.org.apache.jute - - - - com.fasterxml.jackson - dlshade.com.fasterxml.jackson - - - - com.sun.jna - dlshade.com.sun.jna - - - - com.google - dlshade.com.google - - - - org.jboss.netty - dlshade.org.jboss.netty - - - - net.jpountz - dlshade.net.jpountz - - - - org.rocksdb - dlshade.org.rocksdb - - - - com.scurrilous.circe - dlshade.com.scurrilous.circe - - - org.apache.bookkeeper - dlshade.org.apache.bookkeeper - - - - org.apache.distributedlog - org.apache.distributedlog - - - - - - - - org.codehaus.mojo - license-maven-plugin - ${license-maven-plugin.version} - - false - ${project.basedir} - - - - update-pom-license - - update-file-header - - package - - apache_v2 - - dependency-reduced-pom.xml - - - - - - - maven-clean-plugin - ${maven-clean-plugin.version} - - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - diff --git a/shaded/pom.xml b/shaded/pom.xml deleted file mode 100644 index 254de71d961..00000000000 --- a/shaded/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - org.apache.bookkeeper - shaded-parent - Apache BookKeeper :: Shaded :: Parent - - bookkeeper-server-shaded - bookkeeper-server-tests-shaded - distributedlog-core-shaded - - diff --git a/site/_data/cli/bookkeeper.yaml b/site/_data/cli/bookkeeper.yaml index cc4b2eca5d5..fd780f151f3 100644 --- a/site/_data/cli/bookkeeper.yaml +++ b/site/_data/cli/bookkeeper.yaml @@ -7,7 +7,7 @@ commands: description: Starts up an ensemble of N bookies in a single JVM process. Typically used for local experimentation and development. argument: N - name: autorecovery - description: Runs the autorecovery service daemon. + description: Runs the autorecovery service. - name: upgrade description: Upgrades the bookie's filesystem. options: diff --git a/site/bps/BP-46-run-without-journal.md b/site/bps/BP-46-run-without-journal.md new file mode 100644 index 00000000000..83030356db2 --- /dev/null +++ b/site/bps/BP-46-run-without-journal.md @@ -0,0 +1,205 @@ +--- +title: "BP-46: Running without the journal" +issue: https://github.com/apache/bookkeeper/2705 +state: "Under Discussion" +release: "N/A" +--- + +### Motivation + +The journal allows for fast add operations that provide strong data safety guarantees. An add operation is only acked to a client once written to the journal and an fsync performed. This however means that every entry must be written twice: once to the journal and once to an entry log file. + +This double write increases the cost of ownership as more disks must be provisioned to service requests and makes disk provisioning more complex (separating journal from entry log writes onto separate disks). Running without the journal would halve the disk IO required (ignoring indexes) thereby reducing costs and simplifying provisioning. + +However, running without the journal would introduce data consistency problems as the BookKeeper Replication Protocol requires that all writes are persistent for correctness. Running without the journal introduces the possibility of lost writes. In order to continue to offer strong data safety and support running without the journal, changes to the protocol are required. + +### A note on Response Codes + +The following categories are relevant: + +- Positive: OK +- Explicit Negative: NoSuchEntry/NoSuchLedger +- Unknown: Any other non-success response that is not an explicit negative. + +For correctness explicit negatives must be treated differently than other errors. + +### A note on Quorums + +In order to explain the protocol changes, it is useful to first consider how quorums are used for safety. We have the following relevant quorums: + +- Single bookie (S) +- Ack quorum (AQ) +- Write quorum (WQ) +- Quorum Coverage (QC) where QC = (WQ - AQ) + 1 +- Ensemble Coverage (EC) where EC = (E - AQ) + 1 +- Whole Ensemble + +Quorum Coverage (QC) and Ensemble Coverage (EC) are both defined by the following, only the cohorts differ: + +- A given property is satisfied by at least one bookie from every possible ack quorum within the cohort. +- There exists no ack quorum of bookies that do not satisfy the property within the cohort. + +For QC, the cohort is the writeset of a given entry, and therefore QC is only used when we need guarantees regarding a single entry. For EC, the cohort is the ensemble of bookies of the current fragment. EC is required when we need a guarantee across an entire fragment. + +For example: + +- For fencing, we need to ensure that no AQ of bookies is unfenced before starting the read/write phase of recovery. This is true once EC successful fencing responses have been received. +- For a recovery read, a read is only negative once we know that no AQ of bookies could exist that might have the entry. Doing otherwise could truncate committed entries from a ledger. A read is negative once NoSuchEntry responses reach QC. + +Different protocol actions require different quorums: + +- Add entry: AQ success responses +- Read entry: + - Positive when positive response from a single bookie + - Negative when explicit negative from all bookies + - Unknown: when at least one unknown and no positive from all bookies +- Fencing phase, LAC read (sent to ensemble of current fragment): + - Complete when EC positive responses + - Unknown (cannot make progress) when AQ unknown responses (fencing LAC reads cannot cause an explicit negative as fencing creates the ledger on the bookie if it doesn’t exist) +- Recovery read (sent to writeset of entry): + - Entry recoverable: AQ positive read responses + - Entry Unrecoverable: QC negative read responses + - Unknown (cannot make progress): + - QC unknown responses or + - All responses received, but not enough for either a positive or negative + + +### Impact of Undetected Data Loss on Consistency + +The ledger recovery process assumes that ledger entries are never arbitrarily lost. In the event of the loss of an entry, the recovery process can: +- allow the original client to keep writing entries to a ledger that has just been fenced and closed, thus losing those entries +- allow the recovery client to truncate the ledger too soon, closing it with a last entry id lower than that of previously acknowledged entries - thus losing data. + +The following scenarios assume existing behaviour but simply skipping the writing of entries and fencing ops to the journal. + +### Scenario 1 - Lost Fenced Status Allows Writes After Ledger Close + +1. 3 bookies, B1, B2 & B3 +2. 2 clients, C1 & C2 +3. 1 ledger, L1, with e3:w3:a2 configuration. +4. C1 writes entry E1 to L1. The write hits all three bookies. +5. C1 hangs for an indeterminate length of time. +6. C2 sees that C1 is unresponsive, and assumes it has failed. C2 tries to recover the ledger L1. +7. L1 sends a fencing message to all bookies in the ensemble. +8. The fencing message succeeds in arriving at B1 & B2 and is acknowledged by both. The message to B3 is lost. +9. C2 sees that at least one bookie in each possible ack quorum has acknowledged the fencing message (EC threshold reached), so continues with the read/write phase of recovery, finding that E1 is the last entry of the ledger, and committing the endpoint of the ledger in the ZK. +10. B2 crashes and boots again with all unflushed operations lost. +11. C1 wakes up and writes entry E2 to all bookies. B2 & B3 acknowledge positively, so C1 considers E2 as persisted. B1 rejects the message as the ledger is fenced, but since ack quorum is 2, B2 & B3 are enough to consider the entry written. + +### Scenario 2 - Recovery Truncates Previously Acknowledged Entries + +1. C1 adds E0 to B1, B2, B3 +2. B1 and B3 confirms. C1 confirms the write to its client. +3. C2 starts recovery +4. B2 fails to respond. C1 tries to change ensemble but gets a metadata version conflict. +5. B1 crashes and restarts, has lost E0 (undetected) +6. C2 fences the ledger on B1, B2, B3 +7. C2 sends Read E0 to B1, B2, B3 +8. B1 responds with NoSuchEntry +9. B2 responds with NoSuchEntry +10. QC negative response threshold reached. C2 closes the ledger as empty. Losing E0. + +The problem is that without the journal (and syncing to entry log files before acknowledgement) a bookie can: +- lose the fenced status of a previously existing ledger +- respond with an explicit negative even though it had previously seen an entry. + +Undetected data loss could occur when running without the journal. Bookie crashes and loses most recent entries and fence statuses that had not yet been written and synced to disk. + +### A note on cookies + +Cookies play an essential part in the bookkeeper replication protocol, but their purpose is often unclear. + +When a bookie boots for the first time, it generates a cookie. The cookie encapsulates the identity of the bookie and should be considered immutable. This identity contains the advertised address of the bookie, the disks used for the journal, index, and ledger storage, and a unique ID. The bookie writes the cookie to ZK and each of the disks in use. On all subsequent boots, if the cookie is missing from any of these places, the bookie fails to boot. + +The absence of a disk's cookie implies that the rest of the disk's data is also missing. Cookie validation is performed on boot-up and prevents the boot from succeeding if the validation fails, thus preventing the bookie starting with undetected data loss. + +This proposal improves the cookie mechanism by automating the resolution of a cookie validation error which currently requires human intervention to resolve. This automated feature will be configurable (enabled or disabled) and additionally a CLI command will be made available so an admin can manually run the operation (for when this feature is disabled - likely to be the default). + +### Proposed Changes + +The proposed changes involve: +- A new config that controls whether add operations go into the journal +- Detecting possible data loss on boot +- Prevent explicit negative responses when data loss may have occurred, instead reply with unknown code, until data is repaired. +- Repair data loss +- Auto fix cookies (with new config to enable or disable the feature) +- CLI command for admin to run fix cookie logic in the case that auto fix is disabled + +In these proposed changes, when running "without" the journal, the journal still exists, but add entry operations skip the addition to the journal. The boot-up sequence still replays the journal. + +Add operations can be configured to be written to the journal or not based on the config `journalWriteData`. When set to `false`, add operations are not added to the journal. + +### Detecting Data Loss On Boot + +The new mechanism for data loss detection is checking for an unclean shutdown (aka a crash or abrupt termination of the bookie). When an unclean shutdown is detected further measures are taken to prevent data inconsistency. + +The unclean shutdown detection will consist of setting a bit in the index on start-up and clearing it on shutdown. On subsequent start-up, the value will be checked and if it remains set, it knows that the prior shutdown was not clean. + +Cookie validation will continue to be used to detect booting with one or more missing or empty disks (that once existed and contained a cookie). + +### Protection Mechanism + +Once possible data loss has been detected the following protection mechanism is carried out during the boot: + +- Fencing: Ledger metadata for all ledgers of the cluster are obtained and all those ledgers are fenced on this bookie. This prevents data loss scenario 1. +- Limbo: All open ledgers are placed in the limbo status. Limbo ledgers can serve read requests, but never respond with an explicit negative, all explicit negatives are converted to unknowns (with the use of a new code EUNKNOWN). +- Recovery: All open ledgers are opened and recovered. +- Repair: Each ledger is scanned and any missing entries are sourced from peers. +- Limbo ledgers that have been repaired have their limbo status cleared. + +### The Full Boot-Up Sequence + +This mechanism of limbo ledgers and self-repair needs to work hand-in hand with the cookie validation check. Combining everything together: + +On boot: +1. Check for unclean shutdown and validate cookies +2. Fetch the metadata for all ledgers in the cluster from ZK where the bookie is a member of its ensemble. +3. Phase one: + - If the cookie check fails or unclean shutdown is detected: + - For each non-closed ledger, mark the ledger as fenced and in-limbo in the index. + - Update the cookie if it was a cookie failure +4. Phase two + - For each ledger + 1. If the ledger is in-limbo, open and recover the ledger. + 2. Check that all entries assigned to this bookie exist in the index. + 3. For any entries that are missing, copy from another bookie. + 4. Clear limbo status if set + +When booting a bookie with empty disks, only phase one needs to be complete before the bookie makes itself available for client requests. + +In phase one, if the cookie check fails, we mark all non-closed ledgers as “fenced”. This prevents any future writes to these ledgers on this bookie. This solves the problem of an empty bookie disk allowing writes to closed ledgers (Scenario 1). + +Given that the algorithm solves both the issues that cookies are designed to solve, we can now allow the bookie to update its cookie without operator intervention. + +### Formal Verification of Proposed Changes + +The use of the limbo status and fencing of all ledgers on boot-up when detecting an unclean shutdown has been modelled in TLA+. It does not model the whole boot-up sequence but a simplified version with only fencing and limbo status. + +The specification models the lifetime of a single ledger and includes a single bookie crashing, losing all data. The specification allows the testing of: + +- enabling/disabling the fencing +- enabling/disabling the limbo status. + +When running without limbo status, the model checker finds the counterexample of scenario 2. When running without fencing of all ledgers, the model checker finds the counterexample of scenario 1. When running with both enabled, the model checker finds no invariant violation. + +The specification can be found here: https://github.com/Vanlightly/bookkeeper-tlaplus + +### Public Interfaces + +- Return codes. Addition of a new return code: `EUNKNOWN` which is returned when a read hits an in-limbo ledger and that ledger not contain the requested entry id. +- Bookie ledger metadata format (LedgerData). Addition of the limbo status. + +### Compatibility, Deprecation, and Migration Plan + +- Because we only skip the journal for add operations, there is no impact on existing deployments. When a bookie is booted with the new version, and `journalWriteData` is set to false, the journal is still replayed on boot-up causing no risk of data loss in the transition. + +### Test Plan + +- There is confidence in the design due to the modelling in TLA+ but this model does not include the full boot sequence. +- The implementation will require aggressive chaos testing to ensure correctness. + +### Rejected Alternatives + +Entry Log Per Ledger (ELPL) without the journal. From our performance testing of ELPL, performance degrades significantly with a large number of active ledgers and syncing to disk multiple times a second (which is required to offer low latency writes). + +In the future this design could be extended to offer ledger level configuration of journal use. The scope of this BP is limited to cluster level. \ No newline at end of file diff --git a/site/community/contributing.md b/site/community/contributing.md index 7bf513b3c6a..b56dcb742cd 100644 --- a/site/community/contributing.md +++ b/site/community/contributing.md @@ -102,20 +102,6 @@ Depending on your preferred development environment, you may need to prepare it ##### IntelliJ -###### Enable Annotation Processing - -To configure annotation processing in IntelliJ: - -1. Open Annotation Processors Settings dialog box by going to Settings -> Build, Execution, Deployment -> Compiler -> Annotation Processors. -1. Select the following buttons: - 1. "Enable annotation processing" - 1. "Obtain processors from project classpath" - 1. "Store generated sources relative to: Module content root" -1. Set the generated source directories to be equal to the Maven directories: - 1. Set "Production sources directory:" to "target/generated-sources/annotations". - 1. Set "Test sources directory:" to "target/generated-test-sources/test-annotations". -1. Click "OK". - ###### Checkstyle IntelliJ supports checkstyle within the IDE using the Checkstyle-IDEA plugin. @@ -153,7 +139,6 @@ Start Eclipse with a fresh workspace in a separate directory from your checkout. File -> Import... - -> Existing Maven Projects -> Browse to the directory you cloned into and select "bookkeeper" -> make sure all bookkeeper projects are selected -> Finalize diff --git a/site/docs/latest/getting-started/installation.md b/site/docs/latest/getting-started/installation.md index 9f856ec1d21..dcc4a82a24e 100644 --- a/site/docs/latest/getting-started/installation.md +++ b/site/docs/latest/getting-started/installation.md @@ -12,64 +12,33 @@ You can install BookKeeper either by [downloading](#download) a [GZipped](http:/ * [Unix environment](http://www.opengroup.org/unix) * [Java Development Kit 1.8](http://www.oracle.com/technetwork/java/javase/downloads/index.html) or later -* [Maven 3.0](https://maven.apache.org/install.html) or later ## Download -You can download Apache BookKeeper releases from one of many [Apache mirrors](http://www.apache.org/dyn/closer.cgi/bookkeeper). Here's an example for the [apache.claz.org](http://apache.claz.org/bookkeeper) mirror: - -```shell -$ curl -O {{ download_url }} -$ tar xvf bookkeeper-{{ site.latest_release }}-src.tar.gz -$ cd bookkeeper-{{ site.latest_release }} -``` +You can download Apache BookKeeper releases from one of many [Apache mirrors](https://dlcdn.apache.org/bookkeeper/). ## Clone -To build BookKeeper from source, clone the repository, either from the [GitHub mirror]({{ site.github_repo }}) or from the [Apache repository](http://git.apache.org/bookkeeper.git/): +To build BookKeeper from source, clone the repository, either from the [GitHub mirror]({{ site.github_repo }}): ```shell -# From the GitHub mirror $ git clone {{ site.github_repo}} - -# From Apache directly -$ git clone git://git.apache.org/bookkeeper.git/ ``` -## Build using Maven +## Build using Gradle -Once you have the BookKeeper on your local machine, either by [downloading](#download) or [cloning](#clone) it, you can then build BookKeeper from source using Maven: +Once you have the BookKeeper on your local machine, either by [downloading](#download) or [cloning](#clone) it, you can then build BookKeeper from source using Gradle: ```shell -$ mvn package +$ ./gradlew build -x signDistTar -x test ``` -Since 4.8.0, bookkeeper introduces `table service`. If you would like to build and tryout table service, you can build it with `stream` profile. +To run all the tests: ```shell -$ mvn package -Dstream +$ ./gradlew test -x signDistTar ``` -> You can skip tests by adding the `-DskipTests` flag when running `mvn package`. - -### Useful Maven commands - -Some other useful Maven commands beyond `mvn package`: - -Command | Action -:-------|:------ -`mvn clean` | Removes build artifacts -`mvn compile` | Compiles JAR files from Java sources -`mvn compile spotbugs:spotbugs` | Compile using the Maven [SpotBugs](https://github.com/spotbugs/spotbugs-maven-plugin) plugin -`mvn install` | Install the BookKeeper JAR locally in your local Maven cache (usually in the `~/.m2` directory) -`mvn deploy` | Deploy the BookKeeper JAR to the Maven repo (if you have the proper credentials) -`mvn verify` | Performs a wide variety of verification and validation tasks -`mvn apache-rat:check` | Run Maven using the [Apache Rat](http://creadur.apache.org/rat/apache-rat-plugin/) plugin -`mvn compile javadoc:aggregate` | Build Javadocs locally -`mvn -am -pl bookkeeper-dist/server package` | Build a server distribution using the Maven [Assembly](http://maven.apache.org/plugins/maven-assembly-plugin/) plugin - -> You can enable `table service` by adding the `-Dstream` flag when running above commands. - ## Package directory The BookKeeper project contains several subfolders that you should be aware of: diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml new file mode 100644 index 00000000000..c65b18b1b2e --- /dev/null +++ b/src/owasp-dependency-check-suppressions.xml @@ -0,0 +1,71 @@ + + + + + + + + + c85851ca3ea8128d480d3f75c568a37e64e8a77b + CVE-2020-15106 + + + + c85851ca3ea8128d480d3f75c568a37e64e8a77b + CVE-2020-15112 + + + + c85851ca3ea8128d480d3f75c568a37e64e8a77b + CVE-2020-15113 + + + + + 6dac6efe035a2be9ba299fbf31be5f903401869f + CVE-2020-15106 + + + + 6dac6efe035a2be9ba299fbf31be5f903401869f + CVE-2020-15112 + + + + 6dac6efe035a2be9ba299fbf31be5f903401869f + CVE-2020-15113 + + + + diff --git a/stats/pom.xml b/stats/pom.xml deleted file mode 100644 index d2c89aa671f..00000000000 --- a/stats/pom.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - .. - - pom - org.apache.bookkeeper.stats - bookkeeper-stats-parent - Apache BookKeeper :: Stats :: Parent - - - utils - - - - diff --git a/stats/utils/pom.xml b/stats/utils/pom.xml deleted file mode 100644 index cdfebcd08cc..00000000000 --- a/stats/utils/pom.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - 4.0.0 - - bookkeeper-stats-parent - org.apache.bookkeeper.stats - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper.stats - bookkeeper-stats-utils - Apache BookKeeper :: Stats :: Utils - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.version} - - - org.reflections - reflections - - - org.yaml - snakeyaml - - - com.beust - jcommander - - - com.fasterxml.jackson.core - jackson-annotations - - - diff --git a/stream/api/pom.xml b/stream/api/pom.xml deleted file mode 100644 index 82906fdb484..00000000000 --- a/stream/api/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - 4.0.0 - - stream-storage-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - stream-storage-api - Apache BookKeeper :: Stream Storage :: API - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-common - ${project.parent.version} - - - io.netty - netty-buffer - - - org.inferred - freebuilder - - - diff --git a/stream/bin/streamstorage b/stream/bin/streamstorage index a85c6e70d32..1f41b2258ac 100755 --- a/stream/bin/streamstorage +++ b/stream/bin/streamstorage @@ -20,6 +20,7 @@ BINDIR=$(dirname "$0") SS_HOME=`cd $BINDIR/..;pwd` +BK_HOME=${BK_HOME:-"`cd ${BINDIR}/../..;pwd`"} DEFAULT_STANDALONE_CONF=$SS_HOME/conf/standalone.conf DEFAULT_LOG_CONF=$SS_HOME/conf/log4j.properties @@ -79,18 +80,14 @@ EOF } add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - # Need to generate classpath from maven pom. This is costly so generate it # and cache it. Save the file into our target dir so a mvn clean will get # clean it up and force us create a new one. - f="${SS_HOME}/server/target/classpath.txt" + f="${SS_HOME}/server/build/classpath.txt" if [ ! -f "${f}" ] then - ${MVN} -f "${SS_HOME}/server/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + echo "no classpath.txt found at ${SS_HOME}/server/build" + exit 1 fi SS_CLASSPATH=${CLASSPATH}:`cat "${f}"` } diff --git a/stream/bin/streamstorage-cli b/stream/bin/streamstorage-cli index 6a1df7810d8..365c50dcf16 100755 --- a/stream/bin/streamstorage-cli +++ b/stream/bin/streamstorage-cli @@ -57,18 +57,14 @@ elif [ -e "$BUILT_JAR" ]; then fi add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - # Need to generate classpath from maven pom. This is costly so generate it # and cache it. Save the file into our target dir so a mvn clean will get # clean it up and force us create a new one. - f="${SS_HOME}/cli/target/classpath.txt" + f="${SS_HOME}/cli/build/classpath.txt" if [ ! -f "${f}" ] then - ${MVN} -f "${SS_HOME}/cli/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + echo "no classpath.txt found at ${SS_HOME}/cli/build" + exit 1 fi SS_CLASSPATH=${CLASSPATH}:`cat "${f}"` } diff --git a/stream/bk-grpc-name-resolver/pom.xml b/stream/bk-grpc-name-resolver/pom.xml deleted file mode 100644 index 37feb436205..00000000000 --- a/stream/bk-grpc-name-resolver/pom.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - - 4.0.0 - - stream-storage-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - bk-grpc-name-resolver - Apache BookKeeper :: Stream Storage :: Common :: BK Grpc Name Resolver - - - org.apache.bookkeeper - stream-storage-common - ${project.version} - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.version} - test - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.zookeeper - zookeeper - test-jar - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test-jar - test - - - diff --git a/stream/clients/java/all/pom.xml b/stream/clients/java/all/pom.xml deleted file mode 100644 index 1aa66fd526a..00000000000 --- a/stream/clients/java/all/pom.xml +++ /dev/null @@ -1,109 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-java-client-parent - 4.15.0-SNAPSHOT - - stream-storage-java-client - Apache BookKeeper :: Stream Storage :: Clients :: Java Client - - - - org.apache.bookkeeper - stream-storage-java-kv-client - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - test-jar - test - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - - package - - shade - - - true - true - false - - - org.apache.bookkeeper:stream-storage-* - - - - - - - - org.codehaus.mojo - license-maven-plugin - ${license-maven-plugin.version} - - false - - ${project.basedir} - - - - - update-pom-license - - update-file-header - - package - - apache_v2 - - dependency-reduced-pom.xml - - - - - - - maven-clean-plugin - ${maven-clean-plugin.version} - - - - ${project.basedir} - - dependency-reduced-pom.xml - - - - - - - - diff --git a/stream/clients/java/base/pom.xml b/stream/clients/java/base/pom.xml deleted file mode 100644 index c8d8388420b..00000000000 --- a/stream/clients/java/base/pom.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-java-client-parent - 4.15.0-SNAPSHOT - - stream-storage-java-client-base - Apache BookKeeper :: Stream Storage :: Clients :: Java Client :: Base - - - - org.apache.bookkeeper - stream-storage-api - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-proto - ${project.parent.version} - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/clients/java/kv/pom.xml b/stream/clients/java/kv/pom.xml deleted file mode 100644 index d9c9e359324..00000000000 --- a/stream/clients/java/kv/pom.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-java-client-parent - 4.15.0-SNAPSHOT - - stream-storage-java-kv-client - Apache BookKeeper :: Stream Storage :: Clients :: Java Client :: KV - - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - test-jar - test - - - - - diff --git a/stream/clients/java/pom.xml b/stream/clients/java/pom.xml deleted file mode 100644 index 44a462220e9..00000000000 --- a/stream/clients/java/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - stream-storage-clients-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-java-client-parent - Apache BookKeeper :: Stream Storage :: Clients :: Java Client :: Parent - - base - kv - all - - diff --git a/stream/clients/pom.xml b/stream/clients/pom.xml deleted file mode 100644 index 3001eeb8d90..00000000000 --- a/stream/clients/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-clients-parent - Apache BookKeeper :: Stream Storage :: Clients :: Parent - - java - - diff --git a/stream/common/pom.xml b/stream/common/pom.xml deleted file mode 100644 index 6f5091ad933..00000000000 --- a/stream/common/pom.xml +++ /dev/null @@ -1,77 +0,0 @@ - - - - 4.0.0 - - stream-storage-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - stream-storage-common - Apache BookKeeper :: Stream Storage :: Common - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - io.netty - netty-buffer - - - io.grpc - grpc-all - - - io.grpc - grpc-netty-shaded - - - org.bouncycastle - bcpkix-jdk15on - - - io.grpc - grpc-okhttp - - - - - javax.annotation - javax.annotation-api - - true - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.bookkeeper.tests - stream-storage-tests-common - ${project.version} - test - - - diff --git a/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java b/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java index a36b29123b4..4f46cb05b1a 100644 --- a/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java +++ b/stream/common/src/test/java/org/apache/bookkeeper/common/router/HashRouterTest.java @@ -40,7 +40,7 @@ public void testByteBufHashRouter() { int hash32 = Murmur3.hash32( key, key.readerIndex(), key.readableBytes(), (int) AbstractHashRouter.HASH_SEED); int bytesHash32 = Murmur3.hash32(keyBytes, 0, keyBytes.length, (int) AbstractHashRouter.HASH_SEED); - int guavaHash32 = Hashing.murmur3_32((int) AbstractHashRouter.HASH_SEED) + int guavaHash32 = Hashing.murmur3_32_fixed((int) AbstractHashRouter.HASH_SEED) .newHasher() .putString("foo", UTF_8) .hash() diff --git a/stream/distributedlog/common/pom.xml b/stream/distributedlog/common/pom.xml deleted file mode 100644 index f91055fa99d..00000000000 --- a/stream/distributedlog/common/pom.xml +++ /dev/null @@ -1,109 +0,0 @@ - - - - 4.0.0 - - org.apache.distributedlog - distributedlog - 4.15.0-SNAPSHOT - - distributedlog-common - Apache BookKeeper :: DistributedLog :: Common - - - org.apache.bookkeeper.stats - bookkeeper-stats-api - ${project.parent.version} - - - org.slf4j - slf4j-log4j12 - - - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.commons - commons-lang3 - - - com.google.guava - guava - - - commons-lang - commons-lang - - - commons-codec - commons-codec - - - io.netty - netty-buffer - - - net.jpountz.lz4 - lz4 - - - org.jmock - jmock - test - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G - always - 1800 - - - - - diff --git a/stream/distributedlog/core/build.gradle b/stream/distributedlog/core/build.gradle index ee9a79fc10a..d8f88a366fc 100644 --- a/stream/distributedlog/core/build.gradle +++ b/stream/distributedlog/core/build.gradle @@ -75,6 +75,7 @@ test.doFirst { } jar { + dependsOn tasks.named("writeClasspath") archiveBaseName = 'distributedlog-core' } diff --git a/stream/distributedlog/core/pom.xml b/stream/distributedlog/core/pom.xml deleted file mode 100644 index 78c6ff62b28..00000000000 --- a/stream/distributedlog/core/pom.xml +++ /dev/null @@ -1,135 +0,0 @@ - - - - 4.0.0 - - org.apache.distributedlog - distributedlog - 4.15.0-SNAPSHOT - - distributedlog-core - Apache BookKeeper :: DistributedLog :: Core Library - - - org.apache.distributedlog - distributedlog-protocol - ${project.parent.version} - - - org.apache.zookeeper - zookeeper - - - org.apache.thrift - libthrift - - - org.apache.bookkeeper - bookkeeper-server - ${project.parent.version} - - - javax.annotation - javax.annotation-api - - true - - - org.jmock - jmock - test - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - org.apache.distributedlog - distributedlog-common - ${project.parent.version} - test-jar - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - test - - - - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - false - ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G - always - 1800 - - - listener - org.apache.bookkeeper.common.testing.util.TimedOutTestsListener - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - org.apache.maven.plugins - maven-checkstyle-plugin - - ../../buildtools/src/main/resources/distributedlog/suppressions.xml - ../../buildtools/src/main/resources/bookkeeper/checkstyle.xml - **/thrift/**/* - - - - - diff --git a/stream/distributedlog/io/dlfs/pom.xml b/stream/distributedlog/io/dlfs/pom.xml deleted file mode 100644 index 31366ed9fc0..00000000000 --- a/stream/distributedlog/io/dlfs/pom.xml +++ /dev/null @@ -1,91 +0,0 @@ - - - - 4.0.0 - - distributedlog - org.apache.distributedlog - 4.15.0-SNAPSHOT - ../.. - - org.apache.distributedlog - dlfs - Apache BookKeeper :: DistributedLog :: IO :: FileSystem - http://maven.apache.org - - UTF-8 - ${basedir}/lib - - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - com.google.protobuf - protobuf-java - - - com.google.guava - guava - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - tests - test - - - - org.xerial.snappy - snappy-java - test - - - - io.dropwizard.metrics - metrics-core - test - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - - diff --git a/stream/distributedlog/io/pom.xml b/stream/distributedlog/io/pom.xml deleted file mode 100644 index 38572cb2e0b..00000000000 --- a/stream/distributedlog/io/pom.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - org.apache.distributedlog - distributedlog - 4.15.0-SNAPSHOT - - 4.0.0 - distributedlog-io - pom - Apache BookKeeper :: DistributedLog :: IO - - dlfs - - diff --git a/stream/distributedlog/pom.xml b/stream/distributedlog/pom.xml deleted file mode 100644 index 2f77d0fc025..00000000000 --- a/stream/distributedlog/pom.xml +++ /dev/null @@ -1,91 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - ../.. - - org.apache.distributedlog - distributedlog - pom - Apache BookKeeper :: DistributedLog :: Parent - - Apache DistributedLog provides a high performance replicated log service. - - 2016 - - common - protocol - core - io - - - - - org.apache.maven.plugins - maven-javadoc-plugin - ${maven-javadoc-plugin.version} - - - -notimestamp - none - - - Core Library - org.apache.distributedlog:org.apache.distributedlog.annotations:org.apache.distributedlog.callback:org.apache.distributedlog.exceptions:org.apache.distributedlog.feature:org.apache.distributedlog.io:org.apache.distributedlog.lock:org.apache.distributedlog.logsegment:org.apache.distributedlog.metadata:org.apache.distributedlog.namespace:org.apache.distributedlog.net:org.apache.distributedlog.stats:org.apache.distributedlog.api.subscription - - - - org.apache.distributedlog.acl:org.apache.distributedlog.admin:org.apache.distributedlog.auditor:org.apache.distributedlog.basic:org.apache.distributedlog.benchmark*:org.apache.distributedlog.bk:org.apache.distributedlog.ownership:org.apache.distributedlog.proxy:org.apache.distributedlog.resolver:org.apache.distributedlog.service.*:org.apache.distributedlog.config:org.apache.distributedlog.function:org.apache.distributedlog.impl*:org.apache.distributedlog.injector:org.apache.distributedlog.kafka:org.apache.distributedlog.limiter:org.apache.distributedlog.mapreduce:org.apache.distributedlog.messaging:org.apache.distributedlog.common.rate:org.apache.distributedlog.readahead:org.apache.distributedlog.selector:org.apache.distributedlog.stats:org.apache.distributedlog.thrift*:org.apache.distributedlog.tools:org.apache.distributedlog.util:org.apache.distributedlog.zk:org.apache.bookkeeper.client:org.apache.bookkeeper.stats - - - - - aggregate - - aggregate - - site - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID - always - 1800 - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${session.executionRootDirectory}/buildtools/src/main/resources/distributedlog/findbugsExclude.xml - - - - - - diff --git a/stream/distributedlog/protocol/pom.xml b/stream/distributedlog/protocol/pom.xml deleted file mode 100644 index 727a8009c21..00000000000 --- a/stream/distributedlog/protocol/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - 4.0.0 - - org.apache.distributedlog - distributedlog - 4.15.0-SNAPSHOT - - distributedlog-protocol - Apache BookKeeper :: DistributedLog :: Protocol - - - org.apache.distributedlog - distributedlog-common - ${project.version} - - - io.netty - netty-buffer - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - - diff --git a/stream/pom.xml b/stream/pom.xml deleted file mode 100644 index d1b01cec7e0..00000000000 --- a/stream/pom.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - .. - - pom - org.apache.bookkeeper - stream-storage-parent - Apache BookKeeper :: Stream Storage :: Parent - - - distributedlog - common - tests-common - statelib - api - proto - clients - storage - server - bk-grpc-name-resolver - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - true - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - true - ${redirectTestOutputToFile} - -Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G -Dio.netty.leakDetection.level=PARANOID - always - 1800 - - - - - - - - streamTests - - - streamTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - false - - - - - - - - - diff --git a/stream/proto/pom.xml b/stream/proto/pom.xml deleted file mode 100644 index 7827096cb49..00000000000 --- a/stream/proto/pom.xml +++ /dev/null @@ -1,115 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - stream-storage-proto - Apache BookKeeper :: Stream Storage :: Proto - - - - org.apache.bookkeeper - stream-storage-common - ${project.parent.version} - - - org.apache.commons - commons-lang3 - - - com.google.protobuf - protobuf-java - - - javax.annotation - javax.annotation-api - - true - - - org.apache.bookkeeper.tests - stream-storage-tests-common - ${project.version} - test - - - - - - - kr.motd.maven - os-maven-plugin - ${os-maven-plugin.version} - - - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${javac.target} - ${javac.target} - - -Xlint:unchecked - - -Xpkginfo:always - - false - false - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - grpc-java - io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} - true - - - - - compile - compile-custom - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/server/build.gradle b/stream/server/build.gradle index 19faa189683..d8cba32767f 100644 --- a/stream/server/build.gradle +++ b/stream/server/build.gradle @@ -74,12 +74,8 @@ application { mainClassName = "org.apache.bookkeeper.stream.cluster.StandaloneStarter" } -task writeClasspath { - buildDir.mkdirs() - new File(buildDir, "classpath.txt").text = sourceSets.main.runtimeClasspath.collect { it.absolutePath }.join(':') + "\n" -} - jar { + dependsOn tasks.named("writeClasspath") archiveBaseName = 'stream-storage-server' } diff --git a/stream/server/pom.xml b/stream/server/pom.xml deleted file mode 100644 index 386b611be91..00000000000 --- a/stream/server/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - - stream-storage-server - Apache BookKeeper :: Stream Storage :: Server - - - - org.apache.bookkeeper - stream-storage-service-impl - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-java-client - ${project.parent.version} - - - com.beust - jcommander - - - org.apache.bookkeeper.http - vertx-http-server - ${project.parent.version} - runtime - - - - org.xerial.snappy - snappy-java - runtime - - - - io.dropwizard.metrics - metrics-core - runtime - - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/statelib/pom.xml b/stream/statelib/pom.xml deleted file mode 100644 index d2466182dad..00000000000 --- a/stream/statelib/pom.xml +++ /dev/null @@ -1,169 +0,0 @@ - - - - 4.0.0 - - - stream-storage-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper - statelib - Apache BookKeeper :: Stream Storage :: State Library - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-common - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-api - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-proto - ${project.parent.version} - - - com.google.guava - guava - - - org.rocksdb - rocksdbjni - - - com.google.protobuf - protobuf-java - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - tests - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - net.java.dev.javacc - javacc - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - test-jar - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - - - org.xerial.snappy - snappy-java - ${snappy.version} - - - - - io.dropwizard.metrics - metrics-core - ${dropwizard.version} - - - - - - - kr.motd.maven - os-maven-plugin - ${os-maven-plugin.version} - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - - - - - compile - - - - - - - diff --git a/stream/statelib/src/test/java/org/apache/bookkeeper/statelib/testing/executors/MockExecutorController.java b/stream/statelib/src/test/java/org/apache/bookkeeper/statelib/testing/executors/MockExecutorController.java index 0b27377921c..519b244875e 100644 --- a/stream/statelib/src/test/java/org/apache/bookkeeper/statelib/testing/executors/MockExecutorController.java +++ b/stream/statelib/src/test/java/org/apache/bookkeeper/statelib/testing/executors/MockExecutorController.java @@ -21,12 +21,14 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.not; +import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.doAnswer; import com.google.common.collect.Lists; import com.google.common.util.concurrent.SettableFuture; + import java.time.Duration; import java.util.Iterator; import java.util.List; @@ -126,17 +128,20 @@ public MockExecutorController(ExecutorService executor) { } private void runTask(Runnable runnable) { - if (null == executor) { - runnable.run(); - } else { - try { + try { + if (null == executor) { + runnable.run(); + } else { Assert.assertThat("calling this on the same thread will result in deadlock", Thread.currentThread().getName(), not(containsString(THREAD_NAME_PREFIX))); executor.submit(runnable).get(); - } catch (InterruptedException | ExecutionException e) { - log.warn("runTask failed", e); } + } catch (AssertionError ae) { + throw ae; + } catch (Throwable t) { + log.error("Got unexpected exception while submitting a Runnable", t); + fail("Got unexpected exception while submitting a Runnable " + t.getMessage()); } } diff --git a/stream/storage/api/pom.xml b/stream/storage/api/pom.xml deleted file mode 100644 index bc415e3e28b..00000000000 --- a/stream/storage/api/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-service-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-service-api - Apache BookKeeper :: Stream Storage :: Storage :: Api - - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-proto - ${project.parent.version} - - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/storage/impl/pom.xml b/stream/storage/impl/pom.xml deleted file mode 100644 index 7b2ff7d5672..00000000000 --- a/stream/storage/impl/pom.xml +++ /dev/null @@ -1,160 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-service-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-service-impl - Apache BookKeeper :: Stream Storage :: Storage :: Impl - - - - org.apache.bookkeeper - stream-storage-service-api - ${project.parent.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - - - org.apache.bookkeeper - statelib - ${project.parent.version} - - - org.apache.curator - curator-recipes - - - org.apache.distributedlog - distributedlog-core - ${project.parent.version} - tests - test - - - org.apache.bookkeeper - bookkeeper-common - ${project.parent.version} - tests - test - - - org.apache.bookkeeper.tests - stream-storage-tests-common - ${project.version} - - - org.apache.bookkeeper - stream-storage-java-client-base - ${project.parent.version} - test-jar - test - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - net.java.dev.javacc - javacc - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - test-jar - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - log4j - log4j - - - io.netty - * - - - - - - - org.xerial.snappy - snappy-java - ${snappy.version} - - - - - io.dropwizard.metrics - metrics-core - ${dropwizard.version} - - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/stream/storage/pom.xml b/stream/storage/pom.xml deleted file mode 100644 index 22e3eaf395d..00000000000 --- a/stream/storage/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - .. - - stream-storage-service-parent - Apache BookKeeper :: Stream Storage :: Storage :: Parent - - api - impl - - diff --git a/stream/tests-common/pom.xml b/stream/tests-common/pom.xml deleted file mode 100644 index c14fde018ae..00000000000 --- a/stream/tests-common/pom.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - stream-storage-parent - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper.tests - stream-storage-tests-common - Apache BookKeeper :: Stream Storage :: Common Classes for Tests - - - - io.grpc - grpc-all - - - io.grpc - grpc-netty-shaded - - - org.bouncycastle - bcpkix-jdk15on - - - io.grpc - grpc-okhttp - - - - - com.google.protobuf - protobuf-java - - - javax.annotation - javax.annotation-api - - true - - - - - - - kr.motd.maven - os-maven-plugin - ${os-maven-plugin.version} - - - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${javac.target} - ${javac.target} - - -Xlint:unchecked - - -Xpkginfo:always - - false - false - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - grpc-java - io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} - true - - - - - compile - compile-custom - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - - - test-jar - - - - - - - diff --git a/tests/backward-compat/bc-non-fips/build.gradle b/tests/backward-compat/bc-non-fips/build.gradle index b87df1bc510..e321659662f 100644 --- a/tests/backward-compat/bc-non-fips/build.gradle +++ b/tests/backward-compat/bc-non-fips/build.gradle @@ -23,9 +23,14 @@ plugins { } dependencies { - + testImplementation project(':bookkeeper-common') + testImplementation(project(':bookkeeper-server')) { + exclude group: "org.bouncycastle", module: "bc-fips" + } testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts') - testImplementation project(path: ':bookkeeper-server', configuration: 'testArtifacts') + testImplementation(project(path: ':bookkeeper-server', configuration: 'testArtifacts')) { + exclude group: "org.bouncycastle", module: "bc-fips" + } testImplementation depLibs.junit testImplementation depLibs.slf4j testImplementation depLibs.bcpkixJdk15on diff --git a/tests/backward-compat/bc-non-fips/pom.xml b/tests/backward-compat/bc-non-fips/pom.xml deleted file mode 100644 index eba5b1d7219..00000000000 --- a/tests/backward-compat/bc-non-fips/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - bc-non-fips - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test Bouncy Castle Provider load non FIPS version - - 1.68 - - - - - junit - junit - ${junit.version} - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - org.bouncycastle - * - - - test - - - - org.bouncycastle - bcpkix-jdk15on - ${bc-non-fips.version} - - - - org.bouncycastle - bcprov-ext-jdk15on - ${bc-non-fips.version} - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - - diff --git a/tests/backward-compat/current-server-old-clients/pom.xml b/tests/backward-compat/current-server-old-clients/pom.xml deleted file mode 100644 index 2379977810d..00000000000 --- a/tests/backward-compat/current-server-old-clients/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - backward-compat-current-server-old-clients - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test old clients working on current server - - diff --git a/tests/backward-compat/hierarchical-ledger-manager/pom.xml b/tests/backward-compat/hierarchical-ledger-manager/pom.xml deleted file mode 100644 index a75736839c2..00000000000 --- a/tests/backward-compat/hierarchical-ledger-manager/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - hierarchical-ledger-manager - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test compat between old version and new version of hierarchical ledger manager - - diff --git a/tests/backward-compat/hostname-bookieid/pom.xml b/tests/backward-compat/hostname-bookieid/pom.xml deleted file mode 100644 index 28d64fce9c7..00000000000 --- a/tests/backward-compat/hostname-bookieid/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - hostname-bookieid - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between 4.1.0 and current version (with hostname bookie ID) - - diff --git a/tests/backward-compat/old-cookie-new-cluster/pom.xml b/tests/backward-compat/old-cookie-new-cluster/pom.xml deleted file mode 100644 index c7705c999cd..00000000000 --- a/tests/backward-compat/old-cookie-new-cluster/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - old-cookie-new-cluster - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade 4.1.0 to current in cluster with cookies - - diff --git a/tests/backward-compat/pom.xml b/tests/backward-compat/pom.xml deleted file mode 100644 index 396840bb8f4..00000000000 --- a/tests/backward-compat/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper.tests - integration-tests-base-groovy - 4.15.0-SNAPSHOT - ../integration-tests-base-groovy - - org.apache.bookkeeper.tests - backward-compat - Apache BookKeeper :: Tests :: Backward Compatibility - - upgrade - upgrade-direct - hierarchical-ledger-manager - hostname-bookieid - recovery-no-password - old-cookie-new-cluster - current-server-old-clients - yahoo-custom-version - bc-non-fips - - diff --git a/tests/backward-compat/recovery-no-password/pom.xml b/tests/backward-compat/recovery-no-password/pom.xml deleted file mode 100644 index 8eb53fc7f07..00000000000 --- a/tests/backward-compat/recovery-no-password/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - recovery-no-password - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test recovery does not work when password no in metadata - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - diff --git a/tests/backward-compat/upgrade-direct/pom.xml b/tests/backward-compat/upgrade-direct/pom.xml deleted file mode 100644 index c03d5686c38..00000000000 --- a/tests/backward-compat/upgrade-direct/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - upgrade-direct - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between 4.1.0 and current version - - diff --git a/tests/backward-compat/upgrade/pom.xml b/tests/backward-compat/upgrade/pom.xml deleted file mode 100644 index e8377df4e68..00000000000 --- a/tests/backward-compat/upgrade/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - upgrade - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between all released versions and current version - - diff --git a/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy b/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy index d885e778ff8..e7e9a3170a3 100644 --- a/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy +++ b/tests/backward-compat/upgrade/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgrade.groovy @@ -143,11 +143,11 @@ class TestCompatUpgrade { @Test public void test_006_4130to4143() throws Exception { - testUpgrade("4.13.0", "4.14.3") + testUpgrade("4.13.0", "4.14.4") } @Test public void test_007_4143toCurrentMaster() throws Exception { - testUpgrade("4.14.3", BookKeeperClusterUtils.CURRENT_VERSION) + testUpgrade("4.14.4", BookKeeperClusterUtils.CURRENT_VERSION) } } diff --git a/tests/backward-compat/yahoo-custom-version/pom.xml b/tests/backward-compat/yahoo-custom-version/pom.xml deleted file mode 100644 index 6061316a0e7..00000000000 --- a/tests/backward-compat/yahoo-custom-version/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - backward-compat - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.backward-compat - yahoo-custom-version - jar - Apache BookKeeper :: Tests :: Backward Compatibility :: Test upgrade between yahoo custom version and current - - diff --git a/tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy b/tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy deleted file mode 100644 index f1d024a61dc..00000000000 --- a/tests/backward-compat/yahoo-custom-version/src/test/groovy/org/apache/bookkeeper/tests/backwardcompat/TestCompatUpgradeYahooCustom.groovy +++ /dev/null @@ -1,220 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -package org.apache.bookkeeper.tests.backwardcompat - -import static java.nio.charset.StandardCharsets.UTF_8 - -import com.github.dockerjava.api.DockerClient - -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ExecutionException -import java.util.concurrent.TimeUnit -import org.apache.bookkeeper.tests.integration.utils.BookKeeperClusterUtils -import org.apache.bookkeeper.tests.integration.utils.MavenClassLoader - -import org.jboss.arquillian.junit.Arquillian -import org.jboss.arquillian.test.api.ArquillianResource - -import org.junit.Assert -import org.junit.Test -import org.junit.runner.RunWith - -import org.slf4j.Logger -import org.slf4j.LoggerFactory - -@RunWith(Arquillian.class) -class TestCompatUpgradeYahooCustom { - private static final Logger LOG = LoggerFactory.getLogger(TestCompatUpgradeYahooCustom.class) - private static byte[] PASSWD = "foobar".getBytes() - - def yahooRepo = "https://raw.githubusercontent.com/yahoo/bookkeeper/mvn-repo" - def yahooArtifact = "org.apache.bookkeeper:bookkeeper-server:4.3.1.85-yahoo" - - @ArquillianResource - DockerClient docker - - int numEntries = 1 - - def yahooConfiguredBookKeeper(def classLoader, def zookeeper) { - // bookkeeper client configured in same way as pulsar configures it - def bkConf = classLoader.newInstance("org.apache.bookkeeper.conf.ClientConfiguration") - - bkConf.setThrottleValue(0) - bkConf.setAddEntryTimeout(30) - bkConf.setReadEntryTimeout(30) - bkConf.setSpeculativeReadTimeout(0) - bkConf.setNumChannelsPerBookie(16) - bkConf.setUseV2WireProtocol(true) - // we can't specify the class name, because it would try to use the thread - // context classloader to load, which doesn't have the required classes loaded. - bkConf.setLedgerManagerType("hierarchical") - - bkConf.enableBookieHealthCheck() - bkConf.setBookieHealthCheckInterval(60, TimeUnit.SECONDS) - bkConf.setBookieErrorThresholdPerInterval(5) - bkConf.setBookieQuarantineTime(1800, TimeUnit.SECONDS) - - bkConf.setZkServers(zookeeper) - return classLoader.newInstance("org.apache.bookkeeper.client.BookKeeper", bkConf) - } - - def addEntry(def classLoader, def ledger, def entryData) { - def promise = new CompletableFuture() - def buffer = classLoader.callStaticMethod("io.netty.buffer.Unpooled", "copiedBuffer", - [entryData.getBytes(UTF_8)]) - def callback = classLoader.createCallback( - "org.apache.bookkeeper.client.AsyncCallback\$AddCallback", - { rc, _ledger, entryId, ctx -> - if (rc != 0) { - promise.completeExceptionally( - classLoader.callStaticMethod("org.apache.bookkeeper.client.BKException", - "create", [rc])) - } else { - promise.complete(entryId) - } - }) - ledger.asyncAddEntry(buffer, callback, null) - return promise - } - - def assertCantWrite(def cl, def ledger) throws Exception { - try { - addEntry(cl, ledger, "Shouldn't write").get() - fail("Shouldn't be able to write to ledger") - } catch (ExecutionException e) { - // correct behaviour - // TODO add more - } - } - def createAndWrite(def cl, def bk) throws Exception { - def ledger = bk.createLedger(3, 2, cl.digestType("CRC32"), PASSWD) - LOG.info("Created ledger ${ledger.getId()}") - for (int i = 0; i < numEntries; i++) { - addEntry(cl, ledger, "foobar" + i).get() - } - return ledger - } - - def openAndVerifyEntries(def cl, def bk, def ledgerId) throws Exception { - LOG.info("Opening ledger $ledgerId") - def ledger = bk.openLedger(ledgerId, cl.digestType("CRC32"), PASSWD) - def entries = ledger.readEntries(0, ledger.getLastAddConfirmed()) - int j = 0 - while (entries.hasMoreElements()) { - def e = entries.nextElement() - Assert.assertEquals(new String(e.getEntry()), "foobar"+ j) - j++ - } - ledger.close() - } - - def List exerciseClients(List toVerify) { - String zookeeper = BookKeeperClusterUtils.zookeeperConnectString(docker) - String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION - - def ledgers = [] - def yahooCL = MavenClassLoader.forArtifact(yahooRepo, yahooArtifact) - def yahooBK = yahooConfiguredBookKeeper(yahooCL, zookeeper) - def currentCL = MavenClassLoader.forBookKeeperVersion(currentVersion) - def currentBK = yahooConfiguredBookKeeper(currentCL, zookeeper) - try { - // verify we can read ledgers from previous run - for (Long id : toVerify) { - LOG.info("Verifying $id with yahoo client") - openAndVerifyEntries(yahooCL, yahooBK, id) - LOG.info("Verifying $id with current client") - openAndVerifyEntries(currentCL, currentBK, id) - } - // yahoo client and create open and read - def ledger0 = createAndWrite(yahooCL, yahooBK) - ledgers.add(ledger0.getId()) - ledger0.close() - openAndVerifyEntries(yahooCL, yahooBK, ledger0.getId()) - - // yahoo client can fence on yahoo bookies - def ledger1 = createAndWrite(yahooCL, yahooBK) - ledgers.add(ledger1.getId()) - openAndVerifyEntries(yahooCL, yahooBK, ledger1.getId()) - assertCantWrite(yahooCL, ledger1) - - // current client can create open and read - def ledger2 = createAndWrite(currentCL, currentBK) - ledgers.add(ledger2.getId()) - ledger2.close() - openAndVerifyEntries(currentCL, currentBK, ledger2.getId()) - - // current client can fence on yahoo bookies - def ledger3 = createAndWrite(currentCL, currentBK) - ledgers.add(ledger3.getId()) - openAndVerifyEntries(currentCL, currentBK, ledger3.getId()) - assertCantWrite(currentCL, ledger3) - - // current client can fence a bookie created by yahoo client - def ledger4 = createAndWrite(yahooCL, yahooBK) - ledgers.add(ledger4.getId()) - openAndVerifyEntries(currentCL, currentBK, ledger4.getId()) - assertCantWrite(yahooCL, ledger4) - - // Since METADATA_VERSION is upgraded and it is using binary format, the older - // clients which are expecting text format would fail to read ledger metadata. - def ledger5 = createAndWrite(currentCL, currentBK) - ledgers.add(ledger5.getId()) - try { - openAndVerifyEntries(yahooCL, yahooBK, ledger5.getId()) - } catch (Exception exc) { - Assert.assertEquals(exc.getClass().getName(), - "org.apache.bookkeeper.client.BKException\$ZKException") - } - } finally { - currentBK.close() - currentCL.close() - yahooBK.close() - yahooCL.close() - } - return ledgers - } - - @Test - public void testUpgradeYahooCustom() throws Exception { - String currentVersion = BookKeeperClusterUtils.CURRENT_VERSION - String yahooVersion = "4.3-yahoo" - BookKeeperClusterUtils.metadataFormatIfNeeded(docker, yahooVersion) - - Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, yahooVersion)) - def preUpgradeLedgers = exerciseClients([]) - Assert.assertTrue(BookKeeperClusterUtils.stopAllBookies(docker)) - BookKeeperClusterUtils.runOnAllBookies( - docker, "cp", "/opt/bookkeeper/${yahooVersion}/conf/bookkeeper.conf", - "/opt/bookkeeper/${currentVersion}/conf/bk_server.conf") - BookKeeperClusterUtils.updateAllBookieConf(docker, currentVersion, - "logSizeLimit", "1073741824") - BookKeeperClusterUtils.updateAllBookieConf(docker, currentVersion, - "statsProviderClass", - "org.apache.bookkeeper.stats.NullStatsProvider") - - Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, currentVersion)) - // Since METADATA_VERSION is upgraded and it is using binary format, the older - // clients which are expecting text format would fail to read ledger metadata. - try { - exerciseClients(preUpgradeLedgers) - } catch (Exception exc) { - Assert.assertEquals(exc.getClass().getName(), - "org.apache.bookkeeper.client.BKException\$ZKException") - } - } -} diff --git a/tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml b/tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml deleted file mode 100644 index f914ff2c258..00000000000 --- a/tests/backward-compat/yahoo-custom-version/src/test/resources/arquillian.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - CUBE - cube-definitions/3-node-all-version-unstarted.yaml - - - diff --git a/tests/docker-images/all-released-versions-image/Dockerfile b/tests/docker-images/all-released-versions-image/Dockerfile index 9c0301fa205..aee3227ecd1 100644 --- a/tests/docker-images/all-released-versions-image/Dockerfile +++ b/tests/docker-images/all-released-versions-image/Dockerfile @@ -38,19 +38,16 @@ RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.10.0/bookke RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.11.1/bookkeeper-server-4.11.1-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.12.1/bookkeeper-server-4.12.1-bin.tar.gz{,.sha512,.asc} RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.13.0/bookkeeper-server-4.13.0-bin.tar.gz{,.sha512,.asc} -RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.3/bookkeeper-server-4.14.3-bin.tar.gz{,.sha512,.asc} +RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.4/bookkeeper-server-4.14.4-bin.tar.gz{,.sha512,.asc} -RUN wget -nv https://archive.apache.org/dist/incubator/pulsar/pulsar-1.21.0-incubating/apache-pulsar-1.21.0-incubating-bin.tar.gz{,.asc} RUN wget -nv https://dist.apache.org/repos/dist/release/bookkeeper/KEYS RUN wget -nv http://svn.apache.org/repos/asf/zookeeper/bookkeeper/dist/KEYS?p=1620552 -O KEYS.old -RUN wget -nv https://archive.apache.org/dist/incubator/pulsar/KEYS -O KEYS.pulsar RUN mkdir -p /etc/supervisord/conf.d && mkdir -p /var/run/supervisor && mkdir -p /var/log/bookkeeper ADD conf/supervisord.conf /etc/supervisord.conf ADD scripts/install-all-tarballs.sh /install-all-tarballs.sh ADD scripts/install-tarball.sh /install-tarball.sh -ADD scripts/install-pulsar-tarball.sh /install-pulsar-tarball.sh -RUN bash /install-all-tarballs.sh && bash /install-pulsar-tarball.sh && rm -rf /tarballs +RUN bash /install-all-tarballs.sh && rm -rf /tarballs WORKDIR / ADD scripts/update-conf-and-boot.sh /update-conf-and-boot.sh diff --git a/tests/docker-images/all-released-versions-image/pom.xml b/tests/docker-images/all-released-versions-image/pom.xml deleted file mode 100644 index 7abcbb03f75..00000000000 --- a/tests/docker-images/all-released-versions-image/pom.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - - org.apache.bookkeeper.tests - docker-images - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.tests - all-released-versions-image - Apache BookKeeper :: Tests :: Docker Images :: All Released Versions - pom - - - docker - - - integrationTests - - - - - - com.spotify - dockerfile-maven-plugin - 1.4.13 - - - default - - build - - - - add-latest-tag - - tag - - - apachebookkeeper/bookkeeper-all-released-versions - latest - - - - - apachebookkeeper/bookkeeper-all-released-versions - ${project.version} - false - - - - - - - diff --git a/tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh b/tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh deleted file mode 100644 index 68fdd0437aa..00000000000 --- a/tests/docker-images/all-released-versions-image/scripts/install-pulsar-tarball.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -set -e - -gpg --import KEYS.pulsar - -TARBALL=$(ls apache-pulsar-*-incubating-bin.tar.gz | tail -1) -gpg --verify $TARBALL.asc - -tar -zxf $TARBALL -rm $TARBALL -VERSION=4.3-yahoo - -mv apache-pulsar-*-incubating /opt/bookkeeper/$VERSION - -cat > /etc/supervisord/conf.d/bookkeeper-$VERSION.conf < - - - - org.apache.bookkeeper.tests - docker-images - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.tests - all-versions-image - Apache BookKeeper :: Tests :: Docker Images :: All Versions - pom - - - org.apache.bookkeeper.tests - all-released-versions-image - ${project.parent.version} - pom - - - - org.apache.bookkeeper - bookkeeper-dist-server - ${project.parent.version} - bin - tar.gz - provided - - - - - docker - - - integrationTests - - - - - - com.spotify - dockerfile-maven-plugin - ${dockerfile-maven-plugin.version} - - - default - - build - - - - add-latest-tag - - tag - - - apachebookkeeper/bookkeeper-all-versions - latest - - - - - apachebookkeeper/bookkeeper-all-versions - ${project.version} - false - true - - target/bookkeeper-dist-server-${project.version}-bin.tar.gz - - - - - org.apache.maven.plugins - maven-dependency-plugin - ${maven-dependency-plugin.version} - - - copy-tarball - - copy-dependencies - - generate-resources - - ${project.build.directory}/ - bookkeeper-dist-server - true - - - - - - - - - diff --git a/tests/docker-images/current-version-image/pom.xml b/tests/docker-images/current-version-image/pom.xml deleted file mode 100644 index 5e221904e3a..00000000000 --- a/tests/docker-images/current-version-image/pom.xml +++ /dev/null @@ -1,147 +0,0 @@ - - - - - org.apache.bookkeeper.tests - docker-images - 4.15.0-SNAPSHOT - - 4.0.0 - org.apache.bookkeeper.tests - current-version-image - Apache BookKeeper :: Tests :: Docker Images :: Current Version - pom - - - org.apache.bookkeeper - bookkeeper-dist-server - ${project.parent.version} - bin - tar.gz - provided - - - - - docker - - - integrationTests - - - - - - - org.codehaus.mojo - exec-maven-plugin - ${exec-maven-plugin.version} - - - build-python-client - generate-resources - - exec - - - ${project.basedir}/target - ${project.basedir}/../../../stream/clients/python/scripts/docker_build.sh - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - ${maven-antrun-plugin.version} - - - generate-resources - - run - - - - copy python wheel file - - - copying docker scripts - - - - - - - - - com.spotify - dockerfile-maven-plugin - ${dockerfile-maven-plugin.version} - - - default - - build - - - - add-latest-tag - - tag - - - apachebookkeeper/bookkeeper-current - latest - - - - - apachebookkeeper/bookkeeper-current - ${project.version} - false - true - - ${project.version} - - - - - org.apache.maven.plugins - maven-dependency-plugin - ${maven-dependency-plugin.version} - - - copy-docker-dependencies - - copy-dependencies - - generate-resources - - ${project.build.directory}/ - bookkeeper-dist-server - true - - - - - - - - - diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml deleted file mode 100644 index a337e91176a..00000000000 --- a/tests/docker-images/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - org.apache.bookkeeper.tests - docker-images - Apache BookKeeper :: Tests :: Docker Images - - all-released-versions-image - all-versions-image - current-version-image - - diff --git a/tests/integration-tests-base-groovy/pom.xml b/tests/integration-tests-base-groovy/pom.xml deleted file mode 100644 index ccaa5ea4308..00000000000 --- a/tests/integration-tests-base-groovy/pom.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - integration-tests-base - 4.15.0-SNAPSHOT - ../integration-tests-base - - - org.apache.bookkeeper.tests - integration-tests-base-groovy - pom - - Apache BookKeeper :: Tests :: Base module for Arquillian based integration tests using groovy - - - 3.6.0-03 - 3.0.2-02 - - - - - - maven-compiler-plugin - - - ${javac.target} - ${javac.target} - groovy-eclipse-compiler - - - - org.codehaus.groovy - groovy-eclipse-compiler - ${groovy-eclipse-compiler.version} - - - org.codehaus.groovy - groovy-eclipse-batch - ${groovy-eclipse-batch.version} - - - - - org.codehaus.groovy - groovy-eclipse-compiler - ${groovy-eclipse-compiler.version} - true - - - org.codehaus.gmaven - groovy-maven-plugin - 2.0 - - - org.codehaus.groovy - groovy-all - ${groovy.version} - pom - - - - - org.apache.maven.plugins - maven-surefire-plugin - - 2.8.1 - - -Xmx4G -Djava.net.preferIPv4Stack=true - 1 - false - - - System.out - true - - - - - - - - - org.codehaus.groovy - groovy-all - pom - - - - - bintray - - false - - groovy-bintray-plugins - https://dl.bintray.com/groovy/maven - - - diff --git a/tests/integration-tests-base/pom.xml b/tests/integration-tests-base/pom.xml deleted file mode 100644 index b5a0ce218d7..00000000000 --- a/tests/integration-tests-base/pom.xml +++ /dev/null @@ -1,98 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - - org.apache.bookkeeper.tests - integration-tests-base - pom - - Apache BookKeeper :: Tests :: Base module for Arquillian based integration tests - - - - - org.apache.bookkeeper.tests - integration-tests-utils - ${project.version} - - - - org.apache.bookkeeper.tests - integration-tests-topologies - ${project.version} - - - - org.jboss.arquillian.junit - arquillian-junit-standalone - test - - - - junit - junit - test - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - integrationTests - - - integrationTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - false - - - - - - - diff --git a/tests/integration-tests-topologies/pom.xml b/tests/integration-tests-topologies/pom.xml deleted file mode 100644 index b2797b09635..00000000000 --- a/tests/integration-tests-topologies/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - - org.apache.bookkeeper.tests - integration-tests-topologies - jar - - Apache BookKeeper :: Tests :: Common topologies for Docker based integration tests - - - - org.testcontainers - testcontainers - - - junit - junit - compile - - - org.apache.bookkeeper.tests - integration-tests-utils - ${project.version} - - - - diff --git a/tests/integration-tests-utils/pom.xml b/tests/integration-tests-utils/pom.xml deleted file mode 100644 index 6c43bbc1d41..00000000000 --- a/tests/integration-tests-utils/pom.xml +++ /dev/null @@ -1,93 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - - org.apache.bookkeeper.tests - integration-tests-utils - jar - - Apache BookKeeper :: Tests :: Utility module for Arquillian based integration tests - - - - org.apache.commons - commons-compress - - - - org.jboss.shrinkwrap.resolver - shrinkwrap-resolver-impl-maven - - - org.jboss.shrinkwrap.resolver - shrinkwrap-resolver-api - - - - org.apache.zookeeper - zookeeper - - - - org.arquillian.cube - arquillian-cube-docker - - - com.github.docker-java - * - - - - - - org.testcontainers - testcontainers - - - - org.codehaus.groovy - groovy-all - pom - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - 2.8.1 - - 1 - false - - - - - - diff --git a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java index 56960061def..a29331131fe 100644 --- a/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java +++ b/tests/integration-tests-utils/src/main/java/org/apache/bookkeeper/tests/integration/utils/BookKeeperClusterUtils.java @@ -43,12 +43,9 @@ public class BookKeeperClusterUtils { public static final String CURRENT_VERSION = System.getProperty("currentVersion"); public static final List OLD_CLIENT_VERSIONS = - Arrays.asList("4.8.2", "4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.3"); + Arrays.asList("4.8.2", "4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.4"); private static final List OLD_CLIENT_VERSIONS_WITH_CURRENT_LEDGER_METADATA_FORMAT = - Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.3"); - - private static final List OLD_CLIENT_VERSIONS_WITH_OLD_BK_BIN_NAME = - Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.3", "4.3-yahoo"); + Arrays.asList("4.9.2", "4.10.0", "4.11.1", "4.12.1", "4.13.0", "4.14.4"); private static final Logger LOG = LoggerFactory.getLogger(BookKeeperClusterUtils.class); @@ -101,7 +98,7 @@ public static boolean metadataFormatIfNeeded(DockerClient docker, String version @Cleanup ZooKeeper zk = BookKeeperClusterUtils.zookeeperClient(docker); if (zk.exists("/ledgers", false) == null) { - String bookkeeper = "/opt/bookkeeper/" + version + "/bin/" + computeBinFilenameByVersion(version); + String bookkeeper = "/opt/bookkeeper/" + version + "/bin/bookkeeper"; runOnAnyBookie(docker, bookkeeper, "shell", "metaformat", "-nonInteractive"); return true; } else { @@ -131,7 +128,7 @@ public static String createDlogNamespaceIfNeeded(DockerClient docker, } public static void formatAllBookies(DockerClient docker, String version) throws Exception { - String bookkeeper = "/opt/bookkeeper/" + version + "/bin/" + computeBinFilenameByVersion(version); + String bookkeeper = "/opt/bookkeeper/" + version + "/bin/bookkeeper"; BookKeeperClusterUtils.runOnAllBookies(docker, bookkeeper, "shell", "bookieformat", "-nonInteractive"); } @@ -259,11 +256,4 @@ public static boolean waitAllBookieUp(DockerClient docker) { .map((b) -> waitBookieUp(docker, b, 10, TimeUnit.SECONDS)) .reduce(true, BookKeeperClusterUtils::allTrue); } - - private static String computeBinFilenameByVersion(String version) { - if (OLD_CLIENT_VERSIONS_WITH_OLD_BK_BIN_NAME.contains(version)) { - return "bookkeeper"; - } - return "bookkeeper_gradle"; - } } diff --git a/tests/integration/cluster/build.gradle b/tests/integration/cluster/build.gradle index 4cf0dac30d0..71a01ac3881 100644 --- a/tests/integration/cluster/build.gradle +++ b/tests/integration/cluster/build.gradle @@ -23,6 +23,7 @@ plugins { dependencies { implementation depLibs.grpc + testImplementation project(':bookkeeper-common') testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts') testImplementation project(':stream:api') testImplementation project(':stream:storage:api') diff --git a/tests/integration/cluster/pom.xml b/tests/integration/cluster/pom.xml deleted file mode 100644 index 604aeab3e1f..00000000000 --- a/tests/integration/cluster/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - integration - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.integration - cluster - jar - Apache BookKeeper :: Tests :: Integration :: Cluster test - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test - - - - org.apache.bookkeeper - stream-storage-server - ${project.version} - test - - - - org.apache.bookkeeper.tests - integration-tests-topologies - ${project.version} - test - - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - test-jar - test - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - 0 - ${redirectTestOutputToFile} - - - - - diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml deleted file mode 100644 index 1d7535f59bd..00000000000 --- a/tests/integration/pom.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - org.apache.bookkeeper.tests - integration - Apache BookKeeper :: Tests :: Integration - - smoke - standalone - cluster - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - integrationTests - - - integrationTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - false - - - - - - - diff --git a/tests/integration/smoke/build.gradle b/tests/integration/smoke/build.gradle index 02ad13b25fb..b154f34a841 100644 --- a/tests/integration/smoke/build.gradle +++ b/tests/integration/smoke/build.gradle @@ -22,6 +22,7 @@ plugins { } dependencies { + testImplementation project(':bookkeeper-common') testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts') testImplementation project(':bookkeeper-server') testImplementation project(':tests:integration-tests-utils') diff --git a/tests/integration/smoke/pom.xml b/tests/integration/smoke/pom.xml deleted file mode 100644 index 35cc17511b8..00000000000 --- a/tests/integration/smoke/pom.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - integration - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.integration - smoke - jar - Apache BookKeeper :: Tests :: Integration :: Smoke test - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test - - - - org.apache.bookkeeper.tests - integration-tests-utils - ${project.version} - test - - - - org.apache.bookkeeper.tests - integration-tests-topologies - ${project.version} - test - - - - org.jboss.arquillian.junit - arquillian-junit-standalone - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - 0 - ${redirectTestOutputToFile} - - - - - diff --git a/tests/integration/standalone/pom.xml b/tests/integration/standalone/pom.xml deleted file mode 100644 index 7a9cb5a4272..00000000000 --- a/tests/integration/standalone/pom.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - integration - 4.15.0-SNAPSHOT - .. - - - org.apache.bookkeeper.tests.integration - standalone - jar - Apache BookKeeper :: Tests :: Integration :: Standalone test - - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - test - - - - org.apache.bookkeeper.tests - integration-tests-topologies - ${project.version} - test - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - ${maven-surefire-plugin.version} - - - 0 - ${redirectTestOutputToFile} - - - - - diff --git a/tests/pom.xml b/tests/pom.xml deleted file mode 100644 index ae4f4b8ab3c..00000000000 --- a/tests/pom.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper - bookkeeper - 4.15.0-SNAPSHOT - - org.apache.bookkeeper.tests - tests-parent - Apache BookKeeper :: Tests - - - 2.5.8 - - - - shaded - docker-images - integration-tests-base - integration-tests-base-groovy - integration-tests-utils - integration-tests-topologies - backward-compat - integration - scripts - - - - - org.apache.maven.plugins - maven-deploy-plugin - ${maven-deploy-plugin.version} - - true - - - - - diff --git a/tests/scripts/pom.xml b/tests/scripts/pom.xml deleted file mode 100644 index 4bbc9e80021..00000000000 --- a/tests/scripts/pom.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - - - org.apache.bookkeeper.tests - scripts - jar - Apache BookKeeper :: Tests :: Bash Scripts Test - - - - - - - - com.googlecode.maven-download-plugin - download-maven-plugin - - - install-shunit2 - integration-test - - wget - - - https://github.com/kward/shunit2/archive/v2.1.7.zip - true - ${project.basedir}/target/lib - ${skipTests} - - - - - - org.codehaus.mojo - exec-maven-plugin - - - bash-tests - integration-test - - exec - - - ${skipTests} - ${project.basedir}/src/test/bash - ${project.basedir}/src/test/bash/bk_test.sh - - - - - - - diff --git a/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh b/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh index c20165f6050..9a1f7f3917e 100644 --- a/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh +++ b/tests/scripts/src/test/bash/gradle/bk_test_bin_common.sh @@ -27,7 +27,7 @@ # testDefaultVariables() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh assertEquals "BINDIR is not set correctly" "${BK_BINDIR}" "${BINDIR}" assertEquals "BK_HOME is not set correctly" "${BK_HOMEDIR}" "${BK_HOME}" assertEquals "DEFAULT_LOG_CONF is not set correctly" "${BK_CONFDIR}/log4j.properties" "${DEFAULT_LOG_CONF}" @@ -48,7 +48,7 @@ testDefaultVariables() { } testFindModuleJarAt() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh MODULE="test-module" @@ -113,7 +113,7 @@ testFindModuleJar() { echo "" > ${BK_HOME}/conf/bkenv.sh echo "" > ${BK_HOME}/conf/bk_cli_env.sh - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh MODULE="test-module" MODULE_PATH="testmodule" @@ -159,7 +159,7 @@ testLoadEnvfiles() { echo "CLI_MAX_HEAP_MEMORY=2048M" > ${BK_HOME}/conf/bk_cli_env.sh # load the common_gradle.sh - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh assertEquals "NETTY_LEAK_DETECTION_LEVEL is not set correctly" "enabled" "${NETTY_LEAK_DETECTION_LEVEL}" assertEquals "BOOKIE_MAX_HEAP_MEMORY is not set correctly" "2048M" "${BOOKIE_MAX_HEAP_MEMORY}" @@ -172,7 +172,7 @@ testLoadEnvfiles() { } testBuildBookieJVMOpts() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh TEST_LOG_DIR=${BK_TMPDIR}/logdir TEST_GC_LOG_FILENAME="test-gc.log" @@ -187,7 +187,7 @@ testBuildBookieJVMOpts() { } testBuildCLIJVMOpts() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh TEST_LOG_DIR=${BK_TMPDIR}/logdir TEST_GC_LOG_FILENAME="test-gc.log" @@ -202,7 +202,7 @@ testBuildCLIJVMOpts() { } testBuildNettyOpts() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh ACTUAL_NETTY_OPTS=$(build_netty_opts) EXPECTED_NETTY_OPTS="-Dio.netty.leakDetectionLevel=disabled \ @@ -213,7 +213,7 @@ testBuildNettyOpts() { } testBuildBookieOpts() { - source ${BK_BINDIR}/common_gradle.sh + source ${BK_BINDIR}/common.sh ACTUAL_OPTS=$(build_bookie_opts) EXPECTED_OPTS="-Djava.net.preferIPv4Stack=true" diff --git a/tests/shaded/bookkeeper-server-shaded-test/pom.xml b/tests/shaded/bookkeeper-server-shaded-test/pom.xml deleted file mode 100644 index cc548bc2c6f..00000000000 --- a/tests/shaded/bookkeeper-server-shaded-test/pom.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests.shaded - shaded-tests-parent - 4.15.0-SNAPSHOT - .. - - bookkeeper-server-shaded-test - Apache BookKeeper :: Tests :: bookkeeper-server-shaded test - - - org.apache.bookkeeper - bookkeeper-server-shaded - ${project.version} - test - - - - com.google.protobuf - protobuf-java - - - com.google.guava - guava - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - maven-compiler-plugin - ${maven-compiler-plugin.version} - - - - diff --git a/tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml b/tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml deleted file mode 100644 index 2a8cb5490c3..00000000000 --- a/tests/shaded/bookkeeper-server-tests-shaded-test/pom.xml +++ /dev/null @@ -1,78 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests.shaded - shaded-tests-parent - 4.15.0-SNAPSHOT - .. - - bookkeeper-server-tests-shaded-test - Apache BookKeeper :: Tests :: bookkeeper-server-tests-shaded test - - - org.apache.bookkeeper - bookkeeper-server-shaded - ${project.version} - test - - - - com.google.protobuf - protobuf-java - - - com.google.guava - guava - - - - - org.apache.bookkeeper - bookkeeper-server-tests-shaded - ${project.version} - test - - - - com.google.protobuf - protobuf-java - - - com.google.guava - guava - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - - diff --git a/tests/shaded/distributedlog-core-shaded-test/pom.xml b/tests/shaded/distributedlog-core-shaded-test/pom.xml deleted file mode 100644 index 7b2206c4ccf..00000000000 --- a/tests/shaded/distributedlog-core-shaded-test/pom.xml +++ /dev/null @@ -1,82 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper.tests.shaded - shaded-tests-parent - 4.15.0-SNAPSHOT - .. - - distributedlog-core-shaded-test - Apache BookKeeper :: Tests :: distributedlog-core-shaded test - - - org.apache.distributedlog - distributedlog-core-shaded - ${project.version} - test - - - - org.apache.bookkeeper - bookkeeper-server - - - org.apache.bookkeeper.http - bookkeeper-http - - - org.apache.bookkeeper - circe-checksum - - - org.apache.distributedlog - distributedlog-core - - - org.apache.distributedlog - distributedlog-common - - - org.apache.distributedlog - distributedlog-protocol - - - org.apache.zookeeper - zookeeper - - - org.apache.thrift - libthrift - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - - - diff --git a/tests/shaded/pom.xml b/tests/shaded/pom.xml deleted file mode 100644 index f86a357115b..00000000000 --- a/tests/shaded/pom.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - pom - 4.0.0 - - org.apache.bookkeeper.tests - tests-parent - 4.15.0-SNAPSHOT - .. - - org.apache.bookkeeper.tests.shaded - shaded-tests-parent - Apache BookKeeper :: Tests :: Test Shaded Jars - - bookkeeper-server-shaded-test - bookkeeper-server-tests-shaded-test - distributedlog-core-shaded-test - - diff --git a/tools/all/build.gradle b/tools/all/build.gradle index 7593b6a2c6e..670f2c41a72 100644 --- a/tools/all/build.gradle +++ b/tools/all/build.gradle @@ -36,5 +36,6 @@ dependencies { } jar { + dependsOn tasks.named("writeClasspath") archiveBaseName = 'bookkeeper-tools' } diff --git a/tools/all/pom.xml b/tools/all/pom.xml deleted file mode 100644 index ac15e6bb12a..00000000000 --- a/tools/all/pom.xml +++ /dev/null @@ -1,89 +0,0 @@ - - - - 4.0.0 - - bookkeeper-tools-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-tools - Apache BookKeeper :: Tools - - - org.apache.bookkeeper - bookkeeper-tools-ledger - ${project.version} - - - org.apache.bookkeeper - stream-storage-cli - ${project.version} - - - - - - org.apache.maven.plugins - maven-antrun-plugin - ${maven-antrun-plugin.version} - - - append-ledger-commands - generate-resources - - run - - - - - - - - - - - - - - org.apache.maven.plugins - maven-antrun-plugin - ${maven-antrun-plugin.version} - - - append-stream-commands - generate-resources - - run - - - - - - - - - - - - - - - diff --git a/tools/framework/pom.xml b/tools/framework/pom.xml deleted file mode 100644 index 9f5e86b1ef6..00000000000 --- a/tools/framework/pom.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - 4.0.0 - - bookkeeper-tools-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-tools-framework - Apache BookKeeper :: Tools :: Framework - - - com.beust - jcommander - - - org.apache.bookkeeper - bookkeeper-common - ${project.version} - - - org.apache.bookkeeper - buildtools - ${project.parent.version} - test - - - diff --git a/tools/ledger/pom.xml b/tools/ledger/pom.xml deleted file mode 100644 index 8becea1f281..00000000000 --- a/tools/ledger/pom.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - 4.0.0 - - bookkeeper-tools-parent - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-tools-ledger - Apache BookKeeper :: Tools :: Ledger - - - org.apache.bookkeeper - bookkeeper-tools-framework - ${project.version} - - - org.apache.bookkeeper - bookkeeper-server - ${project.parent.version} - - - org.apache.logging.log4j - log4j-1.2-api - runtime - - - org.apache.logging.log4j - log4j-core - runtime - - - org.apache.logging.log4j - log4j-slf4j-impl - runtime - - - org.apache.bookkeeper - buildtools - ${project.parent.version} - test - - - org.apache.bookkeeper - bookkeeper-server - test-jar - ${project.parent.version} - test - - - diff --git a/tools/perf/build.gradle b/tools/perf/build.gradle index 1223e2000d0..80df1221ae9 100644 --- a/tools/perf/build.gradle +++ b/tools/perf/build.gradle @@ -67,5 +67,6 @@ dependencies { } jar { + dependsOn tasks.named("writeClasspath") archiveBaseName = 'bookkeeper-tools-perf' } diff --git a/tools/perf/pom.xml b/tools/perf/pom.xml deleted file mode 100644 index a0c39e39687..00000000000 --- a/tools/perf/pom.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper-tools-parent - 4.15.0-SNAPSHOT - - bookkeeper-perf - Apache BookKeeper :: Tools :: Perf - - - - org.apache.bookkeeper - bookkeeper-tools-framework - ${project.version} - - - org.apache.distributedlog - distributedlog-core - ${project.version} - - - org.apache.bookkeeper - stream-storage-java-client - ${project.version} - - - org.apache.bookkeeper.stats - prometheus-metrics-provider - ${project.version} - - - org.hdrhistogram - HdrHistogram - - - - diff --git a/tools/pom.xml b/tools/pom.xml deleted file mode 100644 index 50010001879..00000000000 --- a/tools/pom.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - 4.0.0 - - bookkeeper - org.apache.bookkeeper - 4.15.0-SNAPSHOT - - bookkeeper-tools-parent - Apache BookKeeper :: Tools :: Parent - pom - - framework - ledger - stream - perf - all - - diff --git a/tools/stream/pom.xml b/tools/stream/pom.xml deleted file mode 100644 index ba9eea73590..00000000000 --- a/tools/stream/pom.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - 4.0.0 - - org.apache.bookkeeper - bookkeeper-tools-parent - 4.15.0-SNAPSHOT - - stream-storage-cli - Apache BookKeeper :: Tools :: Stream - - - - org.apache.bookkeeper - stream-storage-java-client - ${project.version} - - - org.apache.bookkeeper - stream-storage-service-impl - ${project.version} - - - org.apache.bookkeeper - bookkeeper-tools-framework - ${project.version} - - - org.apache.bookkeeper - bookkeeper-server - ${project.version} - - - -