From f1d3d81aaae355c084aa885c4eee7c828276a83d Mon Sep 17 00:00:00 2001 From: Pradeep Nagaraju Date: Thu, 28 Oct 2021 22:09:19 -0700 Subject: [PATCH 01/11] Readops and writeops for the read cache are disabled after the readcache has been shutdown --- .../bookie/storage/ldb/ReadCache.java | 25 ++++++++++++++++--- .../storage/ldb/DbLedgerStorageTest.java | 12 +++++++++ .../bookie/storage/ldb/ReadCacheTest.java | 8 +++--- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java index 986c741a010..a3b23855dc9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java @@ -27,13 +27,17 @@ import io.netty.buffer.Unpooled; import java.io.Closeable; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.bookkeeper.bookie.Bookie; import org.apache.bookkeeper.util.collections.ConcurrentLongLongPairHashMap; import org.apache.bookkeeper.util.collections.ConcurrentLongLongPairHashMap.LongPair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Read cache implementation. @@ -59,6 +63,8 @@ public class ReadCache implements Closeable { private ByteBufAllocator allocator; private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private Boolean isStorageShutdown; + private static final Logger LOG = LoggerFactory.getLogger(Bookie.class); public ReadCache(ByteBufAllocator allocator, long maxCacheSize) { this(allocator, maxCacheSize, DEFAULT_MAX_SEGMENT_SIZE); @@ -76,18 +82,24 @@ public ReadCache(ByteBufAllocator allocator, long maxCacheSize, int maxSegmentSi cacheSegments.add(Unpooled.directBuffer(segmentSize, segmentSize)); cacheIndexes.add(new ConcurrentLongLongPairHashMap(4096, 2 * Runtime.getRuntime().availableProcessors())); } + isStorageShutdown = false; } @Override public void close() { cacheSegments.forEach(ByteBuf::release); + isStorageShutdown = true; } - public void put(long ledgerId, long entryId, ByteBuf entry) { + public void put(long ledgerId, long entryId, ByteBuf entry) throws IOException { int entrySize = entry.readableBytes(); int alignedSize = align64(entrySize); lock.readLock().lock(); + if (isStorageShutdown) { + LOG.error("Read cache has shutdown, not allowing put cache operation"); + throw new IOException(); + } try { int offset = currentSegmentOffset.getAndAdd(alignedSize); @@ -107,7 +119,10 @@ public void put(long ledgerId, long entryId, ByteBuf entry) { // We could not insert in segment, we to get the write lock and roll-over to // next segment lock.writeLock().lock(); - + if (isStorageShutdown) { + LOG.error("Read cache has shutdown, not allowing put cache operation"); + throw new IOException(); + } try { int offset = currentSegmentOffset.getAndAdd(entrySize); if (offset + entrySize > segmentSize) { @@ -126,9 +141,13 @@ public void put(long ledgerId, long entryId, ByteBuf entry) { } } - public ByteBuf get(long ledgerId, long entryId) { + public ByteBuf get(long ledgerId, long entryId) throws IOException { lock.readLock().lock(); + if (isStorageShutdown) { + LOG.error("Read cache has shutdown, not allowing read cache operation"); + throw new IOException(); + } try { // We need to check all the segments, starting from the current one and looking // backward to minimize the diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java index f2a3bec153d..eb3fe1fa413 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java @@ -40,6 +40,7 @@ import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.conf.TestBKConfiguration; import org.apache.bookkeeper.proto.BookieProtocol; +import org.eclipse.jetty.util.IO; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -238,6 +239,17 @@ public void testBookieCompaction() throws Exception { assertEquals(newEntry3, res); } + @Test + public void testReadsOpAfterShutdown() throws Exception { + SingleDirectoryDbLedgerStorage singleDirStorage = ((DbLedgerStorage) storage).getLedgerStorageList().get(0); + singleDirStorage.shutdown(); + try { + ByteBuf res = singleDirStorage.getEntry(4, 3); + } catch (IOException e) { + // This will pass the test since it is expected to throw IOException + } + } + @Test public void doubleDirectory() throws Exception { int gcWaitTime = 1000; diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCacheTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCacheTest.java index 337140c2dd9..561e502aeca 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCacheTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCacheTest.java @@ -29,13 +29,15 @@ import org.junit.Test; +import java.io.IOException; + /** * Unit test for {@link ReadCache}. */ public class ReadCacheTest { @Test - public void simple() { + public void simple() throws IOException { ReadCache cache = new ReadCache(UnpooledByteBufAllocator.DEFAULT, 10 * 1024); assertEquals(0, cache.count()); @@ -72,7 +74,7 @@ public void simple() { } @Test - public void emptyCache() { + public void emptyCache() throws IOException { ReadCache cache = new ReadCache(UnpooledByteBufAllocator.DEFAULT, 10 * 1024); assertEquals(0, cache.count()); @@ -83,7 +85,7 @@ public void emptyCache() { } @Test - public void multipleSegments() { + public void multipleSegments() throws IOException{ // Test with multiple smaller segments ReadCache cache = new ReadCache(UnpooledByteBufAllocator.DEFAULT, 10 * 1024, 2 * 1024); From 189755a65780fc0b82bbc12435483e0fd2956b96 Mon Sep 17 00:00:00 2001 From: Pradeep Nagaraju Date: Thu, 28 Oct 2021 22:40:09 -0700 Subject: [PATCH 02/11] Fix imports --- .../bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java index eb3fe1fa413..2c333fd859b 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java @@ -40,7 +40,6 @@ import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.conf.TestBKConfiguration; import org.apache.bookkeeper.proto.BookieProtocol; -import org.eclipse.jetty.util.IO; import org.junit.After; import org.junit.Before; import org.junit.Test; From 6e0774dfcb2bd16aa55e6ec94a997a255c92977d Mon Sep 17 00:00:00 2001 From: Pradeep Nagaraju Date: Thu, 28 Oct 2021 22:45:56 -0700 Subject: [PATCH 03/11] fix import re-ordering --- .../apache/bookkeeper/bookie/storage/ldb/ReadCacheTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCacheTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCacheTest.java index 561e502aeca..52718075f42 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCacheTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCacheTest.java @@ -26,11 +26,10 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.buffer.UnpooledByteBufAllocator; +import java.io.IOException; import org.junit.Test; -import java.io.IOException; - /** * Unit test for {@link ReadCache}. */ From 42422fdf5e70bac23553e5d98b0afa63408c8ddb Mon Sep 17 00:00:00 2001 From: Pradeep Nagaraju Date: Fri, 29 Oct 2021 11:24:45 -0700 Subject: [PATCH 04/11] Fix locking during ReadCache close; fix other review comments --- .../bookie/storage/ldb/ReadCache.java | 27 +++++++++---------- .../storage/ldb/DbLedgerStorageTest.java | 21 ++++++++++----- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java index a3b23855dc9..0e3dfa0cac9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java @@ -33,11 +33,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.bookkeeper.bookie.Bookie; import org.apache.bookkeeper.util.collections.ConcurrentLongLongPairHashMap; import org.apache.bookkeeper.util.collections.ConcurrentLongLongPairHashMap.LongPair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + /** * Read cache implementation. @@ -63,8 +61,7 @@ public class ReadCache implements Closeable { private ByteBufAllocator allocator; private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - private Boolean isStorageShutdown; - private static final Logger LOG = LoggerFactory.getLogger(Bookie.class); + private volatile boolean isStorageShutdown; public ReadCache(ByteBufAllocator allocator, long maxCacheSize) { this(allocator, maxCacheSize, DEFAULT_MAX_SEGMENT_SIZE); @@ -87,8 +84,13 @@ public ReadCache(ByteBufAllocator allocator, long maxCacheSize, int maxSegmentSi @Override public void close() { - cacheSegments.forEach(ByteBuf::release); - isStorageShutdown = true; + lock.writeLock().lock(); + try { + isStorageShutdown = true; + cacheSegments.forEach(ByteBuf::release); + } finally { + lock.writeLock().unlock(); + } } public void put(long ledgerId, long entryId, ByteBuf entry) throws IOException { @@ -97,8 +99,7 @@ public void put(long ledgerId, long entryId, ByteBuf entry) throws IOException { lock.readLock().lock(); if (isStorageShutdown) { - LOG.error("Read cache has shutdown, not allowing put cache operation"); - throw new IOException(); + throw new IOException("Read cache has shutdown, not allowing put cache operation"); } try { @@ -116,12 +117,11 @@ public void put(long ledgerId, long entryId, ByteBuf entry) throws IOException { lock.readLock().unlock(); } - // We could not insert in segment, we to get the write lock and roll-over to + // We could not insert in segment, we need to get the write lock and roll-over to // next segment lock.writeLock().lock(); if (isStorageShutdown) { - LOG.error("Read cache has shutdown, not allowing put cache operation"); - throw new IOException(); + throw new IOException("Read cache has shutdown, not allowing put cache operation"); } try { int offset = currentSegmentOffset.getAndAdd(entrySize); @@ -145,8 +145,7 @@ public ByteBuf get(long ledgerId, long entryId) throws IOException { lock.readLock().lock(); if (isStorageShutdown) { - LOG.error("Read cache has shutdown, not allowing read cache operation"); - throw new IOException(); + throw new IOException("Read cache has shutdown, not allowing get cache operation"); } try { // We need to check all the segments, starting from the current one and looking diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java index 2c333fd859b..7c7eb2c2675 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java @@ -52,6 +52,7 @@ public class DbLedgerStorageTest { private DbLedgerStorage storage; private File tmpDir; private LedgerDirsManager ledgerDirsManager; + private boolean isStorageShutdown; @Before public void setup() throws Exception { @@ -70,11 +71,14 @@ public void setup() throws Exception { ledgerDirsManager = bookie.getLedgerDirsManager(); storage = (DbLedgerStorage) bookie.getLedgerStorage(); + isStorageShutdown = false; } @After public void teardown() throws Exception { - storage.shutdown(); + if (!isStorageShutdown) { + storage.shutdown(); + } tmpDir.delete(); } @@ -238,15 +242,18 @@ public void testBookieCompaction() throws Exception { assertEquals(newEntry3, res); } - @Test + @Test(expected = IOException.class) public void testReadsOpAfterShutdown() throws Exception { SingleDirectoryDbLedgerStorage singleDirStorage = ((DbLedgerStorage) storage).getLedgerStorageList().get(0); + ByteBuf entry1 = Unpooled.buffer(64); + entry1.writeLong(1); // ledger id + entry1.writeLong(1); // entry id + entry1.writeBytes("e".getBytes()); + singleDirStorage.addEntry(entry1); + entry1.release(); singleDirStorage.shutdown(); - try { - ByteBuf res = singleDirStorage.getEntry(4, 3); - } catch (IOException e) { - // This will pass the test since it is expected to throw IOException - } + isStorageShutdown = true; + singleDirStorage.getEntry(1, 1); } @Test From cade490a1fb12a03a8c4c25d2012d65a3c633e96 Mon Sep 17 00:00:00 2001 From: Pradeep Nagaraju Date: Mon, 1 Nov 2021 12:59:26 -0700 Subject: [PATCH 05/11] Adding a comment --- .../bookkeeper/bookie/storage/ldb/ReadCache.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java index 0e3dfa0cac9..702aa34854e 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/ReadCache.java @@ -99,6 +99,10 @@ public void put(long ledgerId, long entryId, ByteBuf entry) throws IOException { lock.readLock().lock(); if (isStorageShutdown) { + // This is a hack. Ideally, we should be returning from here as a no-op or, + // the higher layers must no-op read/ writes after shutdown() + // Since the other lower layers like locationIndex, logEntry also need to handle the shutdown scenario, + // the IOException is thrown from here so that it bypasses other lower layers. throw new IOException("Read cache has shutdown, not allowing put cache operation"); } @@ -121,6 +125,10 @@ public void put(long ledgerId, long entryId, ByteBuf entry) throws IOException { // next segment lock.writeLock().lock(); if (isStorageShutdown) { + // This is a hack. Ideally, we should be returning from here as a no-op or, + // the higher layers must no-op read/ writes after shutdown() + // Since the other lower layers like locationIndex, logEntry also need to handle the shutdown scenario, + // the IOException is thrown from here so that it bypasses other lower layers. throw new IOException("Read cache has shutdown, not allowing put cache operation"); } try { @@ -145,6 +153,10 @@ public ByteBuf get(long ledgerId, long entryId) throws IOException { lock.readLock().lock(); if (isStorageShutdown) { + // This is a hack. Ideally, we should be returning from here as a no-op or, + // the higher layers must no-op read/ writes after shutdown() + // Since the other lower layers like locationIndex, logEntry also need to handle the shutdown scenario, + // the IOException is thrown from here so that it bypasses other lower layers. throw new IOException("Read cache has shutdown, not allowing get cache operation"); } try { From 9790c43f9386787990f3143398ddcd7832b0089f Mon Sep 17 00:00:00 2001 From: Pradeep Nagaraju Date: Mon, 1 Nov 2021 14:03:19 -0700 Subject: [PATCH 06/11] fix workflow yaml --- .github/workflows/compatibility-check-java11.yml | 4 ++-- .github/workflows/compatibility-check-java8.yml | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/compatibility-check-java11.yml b/.github/workflows/compatibility-check-java11.yml index 3f1306820fb..1bbf5386f08 100644 --- a/.github/workflows/compatibility-check-java11.yml +++ b/.github/workflows/compatibility-check-java11.yml @@ -45,7 +45,7 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.11 - - name: Build with gradle + - | + name: Build with gradle run: ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain -PexcludeTests="*.apache.distributedlog.*, *.apache.bookkeeper.statelib.*, *.apache.bookkeeper.clients.*, *.apache.bookkeeper.common.*, *.apache.bookkeeper.stream.*" - diff --git a/.github/workflows/compatibility-check-java8.yml b/.github/workflows/compatibility-check-java8.yml index 8caa43c8740..8658f50fccd 100644 --- a/.github/workflows/compatibility-check-java8.yml +++ b/.github/workflows/compatibility-check-java8.yml @@ -45,7 +45,8 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Build with gradle + - | + name: Build with gradle run: ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test From c4dad48d9a83cb7118824f70e80fa6f61f591d10 Mon Sep 17 00:00:00 2001 From: Pradeep Nagaraju Date: Mon, 1 Nov 2021 14:15:12 -0700 Subject: [PATCH 07/11] fix github workflow yamls --- .github/workflows/compatibility-check-java11.yml | 12 ++++++++---- .github/workflows/compatibility-check-java8.yml | 14 +++++++------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/compatibility-check-java11.yml b/.github/workflows/compatibility-check-java11.yml index 1bbf5386f08..243e8efb1c1 100644 --- a/.github/workflows/compatibility-check-java11.yml +++ b/.github/workflows/compatibility-check-java11.yml @@ -45,7 +45,11 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.11 - - | - name: Build with gradle - run: ./gradlew test -x bookkeeper-server:test -x tests:integration:cluster:test -x tests:integration:smoke:test -x tests:integration:standalone:test -Dtestlogger.theme=plain - -PexcludeTests="*.apache.distributedlog.*, *.apache.bookkeeper.statelib.*, *.apache.bookkeeper.clients.*, *.apache.bookkeeper.common.*, *.apache.bookkeeper.stream.*" + - name: Build with gradle + run: | + ./gradlew test -x bookkeeper-server:test + -x tests:integration:cluster:test + -x tests:integration:smoke:test + -x tests:integration:standalone:test + -Dtestlogger.theme=plain -PexcludeTests="*.apache.distributedlog.*, *.apache.bookkeeper.statelib.*, *.apache.bookkeeper.clients.*, *.apache.bookkeeper.common.*, *.apache.bookkeeper.stream.*" + diff --git a/.github/workflows/compatibility-check-java8.yml b/.github/workflows/compatibility-check-java8.yml index 8658f50fccd..239a85080b1 100644 --- a/.github/workflows/compatibility-check-java8.yml +++ b/.github/workflows/compatibility-check-java8.yml @@ -45,10 +45,10 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - | - name: Build with gradle - run: ./gradlew test -x bookkeeper-server:test - -x tests:integration:cluster:test - -x tests:integration:smoke:test - -x tests:integration:standalone:test -Dtestlogger.theme=plain - -PexcludeTests="*.apache.distributedlog.*, *.apache.bookkeeper.statelib.*, *.apache.bookkeeper.clients.*, *.apache.bookkeeper.common.*, *.apache.bookkeeper.stream.*" + - name: Build with gradle + run: | + ./gradlew test -x bookkeeper-server:test + -x tests:integration:cluster:test + -x tests:integration:smoke:test + -x tests:integration:standalone:test -Dtestlogger.theme=plain + -PexcludeTests="*.apache.distributedlog.*, *.apache.bookkeeper.statelib.*, *.apache.bookkeeper.clients.*, *.apache.bookkeeper.common.*, *.apache.bookkeeper.stream.*" From 146fbacb0d3df62d846bcfb9cb5701b56b377a16 Mon Sep 17 00:00:00 2001 From: Pradeep Nagaraju Date: Tue, 2 Nov 2021 12:32:53 -0700 Subject: [PATCH 08/11] Fix gradle for bookkeeper-benchmark tests --- bookkeeper-benchmark/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bookkeeper-benchmark/build.gradle b/bookkeeper-benchmark/build.gradle index 6f428f697de..3dec69e5229 100644 --- a/bookkeeper-benchmark/build.gradle +++ b/bookkeeper-benchmark/build.gradle @@ -42,6 +42,8 @@ dependencies { implementation depLibs.nettyTransportNativeEpoll implementation depLibs.zookeeper implementation depLibs.log4j12api + implementation depLibs.log4jCore + implementation depLibs.log4jSlf4jImpl testImplementation project(':bookkeeper-server') testImplementation project(path: ':bookkeeper-common', configuration: 'testArtifacts') @@ -63,7 +65,6 @@ test { maxFailures = 10 maxRetries = 3 } - maxHeapSize = '4G' forkEvery = 1 jvmArgs("-Djunit.timeout.test=600000", From 945e97b4bd639289b20acb4385dff47acfbe321b Mon Sep 17 00:00:00 2001 From: Pradeep Nagaraju Date: Tue, 2 Nov 2021 18:03:57 -0700 Subject: [PATCH 09/11] fix stream build issues --- stream/bk-grpc-name-resolver/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stream/bk-grpc-name-resolver/build.gradle b/stream/bk-grpc-name-resolver/build.gradle index 950bb61724a..2a4315e49ac 100644 --- a/stream/bk-grpc-name-resolver/build.gradle +++ b/stream/bk-grpc-name-resolver/build.gradle @@ -29,6 +29,7 @@ dependencies { implementation project(':stream:common') implementation depLibs.grpc implementation depLibs.slf4j + implementation depLibs.log4jSlf4jImpl implementation depLibs.commonsConfiguration compileOnly depLibs.jsr305 @@ -40,6 +41,7 @@ dependencies { testImplementation project(path: ':bookkeeper-server', configuration: 'testArtifacts') testImplementation depLibs.junit testImplementation depLibs.zookeeper + testImplementation depLibs.zookeeperTest testCompileOnly depLibs.lombok testAnnotationProcessor depLibs.lombok From e18cf49dbf50ecb56422ce924008612be54ab770 Mon Sep 17 00:00:00 2001 From: Pradeep Nagaraju Date: Wed, 3 Nov 2021 16:11:53 -0700 Subject: [PATCH 10/11] Fix compatibility checks test --- shaded/distributedlog-core-shaded/build.gradle | 2 +- stream/bk-grpc-name-resolver/build.gradle | 2 +- .../tests/shaded/DistributedLogCoreShadedJarTest.java | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/shaded/distributedlog-core-shaded/build.gradle b/shaded/distributedlog-core-shaded/build.gradle index 2bafc330aaa..e5c83b21e5a 100644 --- a/shaded/distributedlog-core-shaded/build.gradle +++ b/shaded/distributedlog-core-shaded/build.gradle @@ -53,7 +53,7 @@ shadowJar { relocate 'org.rocksdb', 'dlshade.org.rocksdb' relocate 'com.scurrilous.circe', 'dlshade.com.scurrilous.circe' relocate 'org.apache.bookkeeper', 'dlshade.org.apache.bookkeeper' - relocate 'org.apache.distributedlog', 'dhshade.org.apache.distributedlog' + relocate 'org.apache.distributedlog', 'dlshade.org.apache.distributedlog' archiveBaseName.set("distributedlog-core-shaded") archiveClassifier.set("") } diff --git a/stream/bk-grpc-name-resolver/build.gradle b/stream/bk-grpc-name-resolver/build.gradle index 2a4315e49ac..488f32a3310 100644 --- a/stream/bk-grpc-name-resolver/build.gradle +++ b/stream/bk-grpc-name-resolver/build.gradle @@ -29,7 +29,6 @@ dependencies { implementation project(':stream:common') implementation depLibs.grpc implementation depLibs.slf4j - implementation depLibs.log4jSlf4jImpl implementation depLibs.commonsConfiguration compileOnly depLibs.jsr305 @@ -42,6 +41,7 @@ dependencies { testImplementation depLibs.junit testImplementation depLibs.zookeeper testImplementation depLibs.zookeeperTest + testImplementation depLibs.log4jSlf4jImpl testCompileOnly depLibs.lombok testAnnotationProcessor depLibs.lombok diff --git a/tests/shaded/distributedlog-core-shaded-test/src/test/java/org/apache/bookkeeper/tests/shaded/DistributedLogCoreShadedJarTest.java b/tests/shaded/distributedlog-core-shaded-test/src/test/java/org/apache/bookkeeper/tests/shaded/DistributedLogCoreShadedJarTest.java index 6432c303a28..6632c617e73 100644 --- a/tests/shaded/distributedlog-core-shaded-test/src/test/java/org/apache/bookkeeper/tests/shaded/DistributedLogCoreShadedJarTest.java +++ b/tests/shaded/distributedlog-core-shaded-test/src/test/java/org/apache/bookkeeper/tests/shaded/DistributedLogCoreShadedJarTest.java @@ -59,7 +59,7 @@ public void testProtobufIsShaded() throws Exception { @Test public void testProtobufShadedPath() throws Exception { - Class.forName("dlshade.com.google.protobuf.Message"); + Class.forName("org.apache.bookkeeper.shaded.com.google.protobuf.Message"); } @Test(expected = ClassNotFoundException.class) @@ -69,7 +69,7 @@ public void testGuavaIsShaded() throws Exception { @Test public void testGuavaShadedPath() throws Exception { - Class.forName("dlshade.com.google.common.cache.Cache"); + Class.forName("org.apache.bookkeeper.shaded.com.google.common.cache.Cache"); assertTrue(true); } @@ -119,19 +119,19 @@ public void testCirceChecksumShade() throws Exception { @Test public void testDistributedLogCommon() throws Exception { - Class.forName("org.apache.distributedlog.common.concurrent.AsyncSemaphore"); + Class.forName("dlshade.org.apache.distributedlog.common.concurrent.AsyncSemaphore"); assertTrue(true); } @Test public void testDistributedLogProto() throws Exception { - Class.forName("org.apache.distributedlog.DLSN"); + Class.forName("dlshade.org.apache.distributedlog.DLSN"); assertTrue(true); } @Test public void testDistributedLogCore() throws Exception { - Class.forName("org.apache.distributedlog.api.AsyncLogReader"); + Class.forName("dlshade.org.apache.distributedlog.api.AsyncLogReader"); assertTrue(true); } From 25cc5145d167e8e9b2e6aef6770848251367f662 Mon Sep 17 00:00:00 2001 From: Pradeep Nagaraju Date: Wed, 3 Nov 2021 22:48:41 -0700 Subject: [PATCH 11/11] Fix cluster integration tests with gradle build --- tests/integration/cluster/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/cluster/build.gradle b/tests/integration/cluster/build.gradle index 4eafefd1b97..f29fedfec84 100644 --- a/tests/integration/cluster/build.gradle +++ b/tests/integration/cluster/build.gradle @@ -38,6 +38,7 @@ dependencies { testCompileOnly depLibs.lombok testImplementation depLibs.log4j12api + testImplementation depLibs.log4jCore testImplementation depLibs.testcontainers testImplementation depLibs.commonsConfiguration testImplementation depLibs.googleHTTPClient