From 324b3b63ddfdc037167832809a1b121fb0d9b670 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 14:11:21 +0800 Subject: [PATCH 01/14] fix spotbugs check failed --- .../main/java/org/apache/bookkeeper/client/PendingAddOp.java | 2 ++ buildtools/src/main/resources/bookkeeper/findbugsExclude.xml | 5 +++++ .../org/apache/bookkeeper/bookie/GroupSortBenchmark.java | 4 +++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java index 05f740d33a3..394c961cbc9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/PendingAddOp.java @@ -23,6 +23,7 @@ import static org.apache.bookkeeper.proto.BookieProtocol.FLAG_RECOVERY_ADD; import com.google.common.collect.ImmutableMap; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.netty.buffer.ByteBuf; import io.netty.util.Recycler; import io.netty.util.Recycler.Handle; @@ -80,6 +81,7 @@ class PendingAddOp implements WriteCallback { boolean allowFailFast = false; List ensemble; + @SuppressFBWarnings("IS2_INCONSISTENT_SYNC") static PendingAddOp create(LedgerHandle lh, ClientContext clientCtx, List ensemble, ByteBuf payload, EnumSet writeFlags, diff --git a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml index 09daf196569..60a3a3c3285 100644 --- a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml +++ b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml @@ -317,6 +317,11 @@ + + + + + diff --git a/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java b/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java index 8df5691ae70..94699a26197 100644 --- a/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java +++ b/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java @@ -21,8 +21,10 @@ package org.apache.bookkeeper.bookie; +import java.security.SecureRandom; import java.util.Arrays; import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.bookie.storage.ldb.ArrayGroupSort; import org.openjdk.jmh.annotations.Benchmark; @@ -55,7 +57,7 @@ public static class TestState { private long[] items; public TestState() { - Random r = new Random(); + Random r = ThreadLocalRandom.current(); for (int i = 0; i < (N * 4); i++) { randomItems[i] = r.nextLong(); } From f9911defe826dbe9d96402f17b7dd66b2a78628d Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 14:13:56 +0800 Subject: [PATCH 02/14] format code --- buildtools/src/main/resources/bookkeeper/findbugsExclude.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml index 60a3a3c3285..dec4118b58a 100644 --- a/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml +++ b/buildtools/src/main/resources/bookkeeper/findbugsExclude.xml @@ -317,7 +317,7 @@ - + From f7ce9d0905d8bc8e93bdeab2cfbdeb2dc84155fa Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 14:20:10 +0800 Subject: [PATCH 03/14] install the package first --- .github/workflows/bk-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml index c38a2680ac1..d989ba7bc94 100644 --- a/.github/workflows/bk-ci.yml +++ b/.github/workflows/bk-ci.yml @@ -81,7 +81,9 @@ jobs: - name: Validate pull request if: steps.check_changes.outputs.docs_only != 'true' - run: mvn clean -T 1C -B -nsu apache-rat:check checkstyle:check spotbugs:check package -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + run: | + mvn -T 1C -B -nsu clean install -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO + mvn -T 1C -B -nsu apache-rat:check checkstyle:check spotbugs:check package -Ddistributedlog -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - name: Check license files if: steps.check_changes.outputs.docs_only != 'true' From 44cebc0244d750e0b4606ced491204820ea21513 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 14:43:24 +0800 Subject: [PATCH 04/14] upgrade spotbugs version to 4.7.3.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 43e1217e9ff..e8227092bb3 100644 --- a/pom.xml +++ b/pom.xml @@ -205,7 +205,7 @@ 1.4.1.Final 0.6.1 9.3 - 4.6.0.0 + 4.7.3.2 1 4.0.0 3.0.1 From b6e2b309445d324d267325dd4474ee55c20688d3 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 15:27:55 +0800 Subject: [PATCH 05/14] upgrade javadoc plugin to 3.5.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e8227092bb3..3f171606298 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ 2.7 2.5.1 3.2.2 - 3.1.1 + 3.5.0 3.2.0 3.2.1 3.0.0-M5 From f71d9c1fd1e1bbbc2731ed5e3b827c8a47f52ee0 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 15:36:51 +0800 Subject: [PATCH 06/14] upgrade javadoc plugin to 3.2.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3f171606298..aeb3d95add4 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ 2.7 2.5.1 3.2.2 - 3.5.0 + 3.2.0 3.2.0 3.2.1 3.0.0-M5 From 2b3397a52fd275c1bd955ed5eb484c3aa1d72364 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 15:37:32 +0800 Subject: [PATCH 07/14] format code --- .../java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java | 1 - 1 file changed, 1 deletion(-) diff --git a/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java b/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java index 94699a26197..b51fc758b6d 100644 --- a/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java +++ b/microbenchmarks/src/main/java/org/apache/bookkeeper/bookie/GroupSortBenchmark.java @@ -21,7 +21,6 @@ package org.apache.bookkeeper.bookie; -import java.security.SecureRandom; import java.util.Arrays; import java.util.Random; import java.util.concurrent.ThreadLocalRandom; From 9ce71cdc98c28252fc49ae6ee02440d614535f87 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 18:29:43 +0800 Subject: [PATCH 08/14] set maven version 3.8.7 --- .github/workflows/bk-ci.yml | 5 +++++ pom.xml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml index d989ba7bc94..deae0a0204f 100644 --- a/.github/workflows/bk-ci.yml +++ b/.github/workflows/bk-ci.yml @@ -79,6 +79,11 @@ jobs: distribution: 'temurin' java-version: 11 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: Validate pull request if: steps.check_changes.outputs.docs_only != 'true' run: | diff --git a/pom.xml b/pom.xml index aeb3d95add4..e8227092bb3 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ 2.7 2.5.1 3.2.2 - 3.2.0 + 3.1.1 3.2.0 3.2.1 3.0.0-M5 From 5ef84fec0fce27fac85696c0119892b87eb91229 Mon Sep 17 00:00:00 2001 From: chenhang Date: Fri, 3 Mar 2023 23:38:47 +0800 Subject: [PATCH 09/14] set all test maven version to 3.8.7 --- .github/workflows/bk-ci.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/bk-ci.yml b/.github/workflows/bk-ci.yml index deae0a0204f..a274f3a01ef 100644 --- a/.github/workflows/bk-ci.yml +++ b/.github/workflows/bk-ci.yml @@ -159,6 +159,11 @@ jobs: distribution: 'temurin' java-version: 11 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: Build run: | projects_list= @@ -230,6 +235,11 @@ jobs: distribution: 'temurin' java-version: 11 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: Build with Maven run: mvn -B -nsu clean install -Pdocker -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO @@ -272,6 +282,11 @@ jobs: distribution: 'temurin' java-version: 8 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: Build with Maven run: mvn -B -nsu clean install -DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO @@ -318,6 +333,11 @@ jobs: distribution: 'temurin' java-version: 11 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: mvn package run: mvn -B -nsu clean package -DskipTests @@ -350,6 +370,11 @@ jobs: distribution: 'temurin' java-version: 11 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: mvn package run: mvn -B -nsu clean package -DskipTests @@ -394,6 +419,11 @@ jobs: distribution: 'temurin' java-version: ${{ matrix.jdk_version }} + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: Build with Maven run: mvn clean package -B -nsu -DskipBookKeeperServerTests -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO @@ -429,6 +459,11 @@ jobs: with: java-version: 11 + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + with: + maven-version: 3.8.7 + - name: run "clean install verify" to trigger dependency check # excluding dlfs because it includes hadoop lib with # CVEs that we cannot patch up anyways From 8185356cdf3b8df035d69fa918c38c5304d897d0 Mon Sep 17 00:00:00 2001 From: chenhang Date: Sat, 4 Mar 2023 10:36:09 +0800 Subject: [PATCH 10/14] fix failed CI --- .../src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java index 5dc1a5d8cce..08bf9cfb385 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/PageCacheUtil.java @@ -95,7 +95,7 @@ public static void bestEffortRemoveFromPageCache(int fd, long offset, long len) } try { NATIVE_IO.posix_fadvise(fd, offset, len, POSIX_FADV_DONTNEED); - } catch (Exception e) { + } catch (Throwable e) { log.warn("Failed to perform posix_fadvise: {}", e.getMessage()); fadvisePossible = false; } From 0ced28d4c111cc50754ec8c087fd22b83537f7e8 Mon Sep 17 00:00:00 2001 From: chenhang Date: Sat, 4 Mar 2023 19:20:20 +0800 Subject: [PATCH 11/14] add test failure --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e8227092bb3..37870611fa4 100644 --- a/pom.xml +++ b/pom.xml @@ -1118,7 +1118,7 @@ ${maven-surefire-plugin.version} - @{argLine} -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid + -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid ${test.additional.args} ${redirectTestOutputToFile} ${forkCount.variable} false From 02bf524eaab764507f0ab61b2db2c23878f75769 Mon Sep 17 00:00:00 2001 From: chenhang Date: Sat, 4 Mar 2023 21:01:03 +0800 Subject: [PATCH 12/14] address test failure --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 37870611fa4..0b438af8e38 100644 --- a/pom.xml +++ b/pom.xml @@ -1317,6 +1317,7 @@ --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/java.lang.invoke=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.nio.channels.spi=ALL-UNNAMED From 709364bcd6f2d7ef71d8723d496371089cdcf8dd Mon Sep 17 00:00:00 2001 From: chenhang Date: Sat, 4 Mar 2023 22:19:24 +0800 Subject: [PATCH 13/14] fix failed CI --- .../org/apache/bookkeeper/bookie/LedgerCacheTest.java | 9 +++++++-- pom.xml | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java index 85c3bca8d7c..4e8f06fe7f4 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -797,13 +798,17 @@ public void testSortedLedgerFlushFailure() throws Exception { // after flush failure, the bookie is set to readOnly assertTrue("Bookie is expected to be in Read mode", bookie.isReadOnly()); // write fail + CountDownLatch latch = new CountDownLatch(1); bookie.addEntry(generateEntry(1, 3), false, new BookkeeperInternalCallbacks.WriteCallback(){ public void writeComplete(int rc, long ledgerId, long entryId, BookieId addr, Object ctx){ - LOG.info("fail write to bk"); - assertTrue(rc != OK); + LOG.info("Write to bk succeed due to the bookie readOnly mode check is in the request parse step. " + + "In the addEntry step, we won't check bookie's mode"); + assertEquals(OK, rc); + latch.countDown(); } }, null, "passwd".getBytes()); + latch.await(); bookie.shutdown(); } diff --git a/pom.xml b/pom.xml index 0b438af8e38..d8bb81f571f 100644 --- a/pom.xml +++ b/pom.xml @@ -1118,7 +1118,7 @@ ${maven-surefire-plugin.version} - -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid ${test.additional.args} + @{argLine} -Xmx2G -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetection.level=paranoid ${redirectTestOutputToFile} ${forkCount.variable} false From efc0674c7485b13041673828f3bfec2107b15bdf Mon Sep 17 00:00:00 2001 From: chenhang Date: Sat, 4 Mar 2023 22:38:47 +0800 Subject: [PATCH 14/14] format code --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index d8bb81f571f..e8227092bb3 100644 --- a/pom.xml +++ b/pom.xml @@ -1317,7 +1317,6 @@ --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED - --add-opens java.base/java.lang.invoke=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.nio.channels.spi=ALL-UNNAMED