improve rocksDB

.github/ISSUE_TEMPLATE/flaky_test.md

@@ -0,0 +1,31 @@
+---
+name: Flaky test
+about: Report a flaky test failure
+title: 'Flaky-test: test_class.test_method'
+labels: ["area/tests", "flaky-tests"]
+assignees: ''
+---
+<!--- 
+
+Instructions for reporting a flaky test using this issue template:
+
+1. Replace test_class in title and body with the short test class name WITHOUT the package name.
+2. Replace test_method in title and body with the test method that failed. Multiple methods are flaky, remove the content that refers to the test method.
+3. Replace "relevant parts of the exception stacktrace here" with the a few lines of the stack trace that shows at least the exception message and the line of test code where the stacktrace occurred.
+4. Replace "full exception stacktrace here" with the full exception stacktrace from logs. This section will be hidden by default.
+5. Remove all unused fields / content to unclutter the reported issue. Remove this comment too.
+
+-->
+test_class.test_method is flaky. It fails sporadically.
+
+```
+[relevant parts of the exception stacktrace here]
+```
+
+<!-- optionally provide the full stacktrace ->
+<details>
+<summary>Full exception stacktrace</summary>
+<code><pre>
+full exception stacktrace here
+</pre></code>
+</details>

.github/workflows/backward-compat-tests.yml

@@ -30,7 +30,7 @@ on:
     workflow_dispatch:
 
 env:
-  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
+  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
 
 jobs:
   test:
@@ -49,10 +49,10 @@ jobs:
         with:
           java-version: 1.8
       - name: Build
-        run: ./gradlew stream:server:build -x test
+        run: ./gradlew stream:server:build -x test ${GRADLE_ARGS}
       - name: Test current server with old clients
-        run: ./gradlew :tests:backward-compat:current-server-old-clients:test || (tail -n +1 tests/backward-compat/current-server-old-clients/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/current-server-old-clients/build/container-logs/**/* && exit 1)
+        run: ./gradlew :tests:backward-compat:current-server-old-clients:test ${GRADLE_ARGS} || (tail -n +1 tests/backward-compat/current-server-old-clients/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/current-server-old-clients/build/container-logs/**/* && exit 1)
       - name: Test progressive upgrade
-        run: ./gradlew :tests:backward-compat:upgrade:test || (tail -n +1 tests/backward-compat/upgrade/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/upgrade/build/container-logs/**/* && exit 1)
+        run: ./gradlew :tests:backward-compat:upgrade:test ${GRADLE_ARGS} || (tail -n +1 tests/backward-compat/upgrade/build/reports/tests/test/classes/* && tail -n +1 tests/backward-compat/upgrade/build/container-logs/**/* && exit 1)
       - name: Other tests
-        run: ./gradlew :tests:backward-compat:test -x tests:backward-compat:upgrade:test -x :tests:backward-compat:current-server-old-clients:test
+        run: ./gradlew :tests:backward-compat:test -x tests:backward-compat:upgrade:test -x :tests:backward-compat:current-server-old-clients:test ${GRADLE_ARGS}

.github/workflows/bookie-tests.yml

@@ -29,6 +29,9 @@ on:
       - 'site/**'
     workflow_dispatch:
 
+env:
+  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+
 jobs:
   test:
 
@@ -47,4 +50,8 @@ jobs:
           java-version: 1.8
 
       - name: Run bookie test
-        run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" -Dtestlogger.theme=plain
+        run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.bookie.*" ${GRADLE_ARGS}
+
+      - name: print JVM thread dumps when cancelled
+        if: cancelled()
+        run: ./dev/ci-tool print_thread_dumps

.github/workflows/client-tests.yml

@@ -29,6 +29,9 @@ on:
       - 'site/**'
     workflow_dispatch:
 
+env:
+  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+
 jobs:
   test:
 
@@ -46,4 +49,8 @@ jobs:
         with:
           java-version: 1.8
       - name: Run client tests
-        run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" -Dtestlogger.theme=plain
+        run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.client.*" ${GRADLE_ARGS}
+
+      - name: print JVM thread dumps when cancelled
+        if: cancelled()
+        run: ./dev/ci-tool print_thread_dumps

.github/workflows/compatibility-check-java11.yml

@@ -29,6 +29,9 @@ on:
       - 'site/**'
     workflow_dispatch:
 
+env:
+  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+
 jobs:
   check:
 
@@ -47,4 +50,7 @@ jobs:
           java-version: 1.11
       - name: Build with gradle
         run: |
-          ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**"
+          ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" ${GRADLE_ARGS}
+      - name: print JVM thread dumps when cancelled
+        if: cancelled()
+        run: ./dev/ci-tool print_thread_dumps

.github/workflows/compatibility-check-java8.yml

@@ -29,6 +29,9 @@ on:
       - 'site/**'
     workflow_dispatch:
 
+env:
+  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+
 jobs:
   check:
 
@@ -47,4 +50,8 @@ jobs:
           java-version: 1.8
       - name: Build with gradle
         run: |
-          ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**"
+          ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -PexcludeTests="**/distributedlog/**, **/statelib/**, **/clients/**, **/*common/**, **/stream/**, **/stream/*bk*/**, **/*backward*/**" ${GRADLE_ARGS}
+
+      - name: print JVM thread dumps when cancelled
+        if: cancelled()
+        run: ./dev/ci-tool print_thread_dumps

.github/workflows/integration-tests.yml

@@ -30,7 +30,7 @@ on:
     workflow_dispatch:
 
 env:
-  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
+  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
 
 jobs:
   test:
@@ -50,10 +50,14 @@ jobs:
           java-version: 1.8
 
       - name: Build tar
-        run: ./gradlew stream:server:build -x test
+        run: ./gradlew stream:server:build -x test ${GRADLE_ARGS}
       - name: run cluster integration test
-        run: ./gradlew :tests:integration:cluster:test -Dtestlogger.theme=plain
+        run: ./gradlew :tests:integration:cluster:test ${GRADLE_ARGS}
       - name: run smoke test
-        run: ./gradlew tests:integration:smoke:test -Dtestlogger.theme=plain
+        run: ./gradlew tests:integration:smoke:test ${GRADLE_ARGS}
       - name: run standalone test
-        run: ./gradlew tests:integration:standalone:test -Dtestlogger.theme=plain
+        run: ./gradlew tests:integration:standalone:test ${GRADLE_ARGS}
+
+      - name: print JVM thread dumps when cancelled
+        if: cancelled()
+        run: ./dev/ci-tool print_thread_dumps

.github/workflows/owasp-dep-check.yml

@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: OWASP Dependency Check
+
+on:
+  push:
+  pull_request:
+    branches:
+      - master
+      - branch-*
+    paths-ignore:
+      - 'site/**'
+    workflow_dispatch:
+
+
+jobs:
+  check:
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+
+      - name: run "clean build dependencyCheckAggregate" to trigger dependency check
+        run: ./gradlew clean build -x signDistTar -x test dependencyCheckAggregate
+
+      - name: Upload report
+        uses: actions/upload-artifact@v2
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: dependency report
+          path: build/reports/dependency-check-report.html

.github/workflows/pr-validation.yml

@@ -29,6 +29,9 @@ on:
       - 'site/**'
     workflow_dispatch:
 
+env:
+  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+
 
 jobs:
   check:
@@ -47,6 +50,7 @@ jobs:
         with:
           java-version: 1.8
       - name: Validate pull request
-        run: ./gradlew build -x spotbugsTest -x signDistTar -x test
+        run: ./gradlew build -x signDistTar -x test ${GRADLE_ARGS}
+
       - name: Check license files
-        run: dev/check-all-licenses-gradle
+        run: dev/check-all-licenses

.github/workflows/remaining-tests.yml

@@ -29,6 +29,10 @@ on:
       - 'site/**'
     workflow_dispatch:
 
+env:
+  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+
+
 jobs:
   test:
 
@@ -46,4 +50,8 @@ jobs:
         with:
           java-version: 1.8
       - name: Run remaining tests
-        run: ./gradlew bookkeeper-server:test -PexcludeTests="*org.apache.bookkeeper.bookie.*, *org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*, *org.apache.bookkeeper.tls.*"  -Dtestlogger.theme=plain
+        run: ./gradlew bookkeeper-server:test -PexcludeTests="*org.apache.bookkeeper.bookie.*, *org.apache.bookkeeper.client.*, *org.apache.bookkeeper.replication.*, *org.apache.bookkeeper.tls.*" ${GRADLE_ARGS}
+
+      - name: print JVM thread dumps when cancelled
+        if: cancelled()
+        run: ./dev/ci-tool print_thread_dumps

.github/workflows/replication-tests.yml

@@ -30,7 +30,7 @@ on:
     workflow_dispatch:
 
 env:
-  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
+  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
 
 jobs:
   test:
@@ -49,4 +49,8 @@ jobs:
         with:
           java-version: 1.8
       - name: Run replication tests
-        run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.replication.*" -Dtestlogger.theme=plain
+        run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.replication.*" ${GRADLE_ARGS}
+
+      - name: print JVM thread dumps when cancelled
+        if: cancelled()
+        run: ./dev/ci-tool print_thread_dumps

.github/workflows/stream-tests.yml

@@ -28,6 +28,9 @@ on:
       - 'site/**'
     workflow_dispatch:
 
+env:
+  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+
 jobs:
   test:
 
@@ -45,19 +48,23 @@ jobs:
         with:
           java-version: 1.8
       - name: Run stream:distributedlog:core tests
-        run: ./gradlew stream:distributedlog:core:test -Dtestlogger.theme=plain
+        run: ./gradlew stream:distributedlog:core:test ${GRADLE_ARGS}
       - name: Run stream:distributedlog:common tests
-        run: ./gradlew stream:distributedlog:common:test -Dtestlogger.theme=plain
+        run: ./gradlew stream:distributedlog:common:test ${GRADLE_ARGS}
       - name: Run stream:distributedlog:protocol tests
-        run: ./gradlew stream:distributedlog:protocol:test -Dtestlogger.theme=plain
+        run: ./gradlew stream:distributedlog:protocol:test ${GRADLE_ARGS}
       - name: Run stream:proto tests
-        run: ./gradlew stream:proto:test -Dtestlogger.theme=plain
+        run: ./gradlew stream:proto:test ${GRADLE_ARGS}
       - name: Run stream:serve tests
-        run: ./gradlew stream:server:test -Dtestlogger.theme=plain
+        run: ./gradlew stream:server:test ${GRADLE_ARGS}
       - name: Run stream:statelib tests
-        run: ./gradlew stream:statelib:test -Dtestlogger.theme=plain
+        run: ./gradlew stream:statelib:test ${GRADLE_ARGS}
       - name: Run stream:storage:api:test tests
-        run: ./gradlew stream:storage:api:test -Dtestlogger.theme=plain
+        run: ./gradlew stream:storage:api:test ${GRADLE_ARGS}
       - name: Run stream:storage:impl tests
-        run: ./gradlew stream:storage:impl:test -Dtestlogger.theme=plain
+        run: ./gradlew stream:storage:impl:test ${GRADLE_ARGS}
+
+      - name: print JVM thread dumps when cancelled
+        if: cancelled()
+        run: ./dev/ci-tool print_thread_dumps
 

.github/workflows/tls-tests.yml

@@ -29,6 +29,9 @@ on:
       - 'site/**'
     workflow_dispatch:
 
+env:
+  GRADLE_ARGS: -Dtestlogger.theme=plain -DtestHideStandardOut=true
+
 jobs:
   test:
 
@@ -46,4 +49,8 @@ jobs:
         with:
           java-version: 1.8
       - name: Run tls tests
-        run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.tls.*" -Dtestlogger.theme=plain
+        run: ./gradlew bookkeeper-server:test --tests="org.apache.bookkeeper.tls.*" ${GRADLE_ARGS}
+
+      - name: print JVM thread dumps when cancelled
+        if: cancelled()
+        run: ./dev/ci-tool print_thread_dumps

README.md

@@ -1,6 +1,5 @@
 <img src="https://pbs.twimg.com/profile_images/545716709311520769/piLLa1iC_400x400.png" alt="logo" style="width: 32px;"/>
 
-[![Coverage Status](https://coveralls.io/repos/github/apache/bookkeeper/badge.svg?branch=master)](https://coveralls.io/github/apache/bookkeeper?branch=master)
 [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.bookkeeper/bookkeeper/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.apache.bookkeeper/bookkeeper)
 
 # Apache BookKeeper
@@ -29,7 +28,7 @@ Please visit the [Documentation](https://bookkeeper.apache.org/docs/latest/overv
 
 ### Report a Bug
 
-For filing bugs, suggesting improvements, or requesting new features, help us out by [opening a Github issue](https://github.com/apache/bookkeeper/issues) or [opening an Apache jira](https://issues.apache.org/jira/browse/BOOKKEEPER).
+For filing bugs, suggesting improvements, or requesting new features, help us out by [opening a Github issue](https://github.com/apache/bookkeeper/issues).
 
 ### Need Help?
 
@@ -45,15 +44,10 @@ We feel that a welcoming open community is important and welcome contributions.
 
 ### Contributing Code
 
-1. See [Developer Setup](https://cwiki.apache.org/confluence/display/BOOKKEEPER/Developer+Setup) to get your local environment setup.
+1. See our [installation guide](https://bookkeeper.apache.org/docs/latest/getting-started/installation/) to get your local environment setup.
 
-2. Take a look at our open issues: [JIRA Issues](https://issues.apache.org/jira/browse/BOOKKEEPER) [Github Issues](https://github.com/apache/bookkeeper/issues).
+2. Take a look at our open issues: [Github Issues](https://github.com/apache/bookkeeper/issues).
 
-3. Review our [coding style](https://cwiki.apache.org/confluence/display/BOOKKEEPER/Coding+Guide) and follow our [pull requests](https://github.com/apache/bookkeeper/pulls) to learn about our conventions.
-
-4. Make your changes according to our [contribution guide](https://cwiki.apache.org/confluence/display/BOOKKEEPER/Contributing+to+BookKeeper).
-
-### Improving Website and Documentation
-
-1. See [Building the website and documentation](https://cwiki.apache.org/confluence/display/BOOKKEEPER/Building+the+website+and+documentation) on how to build the website and documentation.
+3. Review our [coding style](https://bookkeeper.apache.org/community/coding_guide/) and follow our [pull requests](https://github.com/apache/bookkeeper/pulls) to learn more about our conventions.
 
+4. Make your changes according to our [contributing guide](https://bookkeeper.apache.org/community/contributing/)

bin/bookkeeper

@@ -62,7 +62,7 @@ where command is one of:
 [service commands]
 
     bookie              Run a bookie server
-    autorecovery        Run AutoRecovery service daemon
+    autorecovery        Run AutoRecovery service
     zookeeper           Run zookeeper server
 
 [development commands]