From fa57330787d9fb1b6c04a118a40e055694d40627 Mon Sep 17 00:00:00 2001 From: chenhang Date: Sun, 26 Dec 2021 16:17:28 +0800 Subject: [PATCH 1/3] support multi ledger directories for rocksdb backend entryMetadataMap --- .../apache/bookkeeper/bookie/BookieImpl.java | 11 +++++++---- .../bookie/GarbageCollectorThread.java | 18 ++++++++++++++---- .../bookie/InterleavedLedgerStorage.java | 3 ++- .../ldb/PersistentEntryLogMetadataMap.java | 5 +++-- .../ldb/SingleDirectoryDbLedgerStorage.java | 2 +- .../bookkeeper/conf/ServerConfiguration.java | 12 +++++++----- .../bookkeeper/util/BookKeeperConstants.java | 2 +- 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java index ed57d7b7e16..71fbf44bbcd 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java @@ -25,6 +25,7 @@ import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_INDEX_SCOPE; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_LEDGER_SCOPE; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; @@ -1186,10 +1187,12 @@ public static boolean format(ServerConfiguration conf, // Clean up metadata directories if they are separate from the // ledger dirs - File metadataDir = new File(conf.getGcEntryLogMetadataCachePath()); - if (!cleanDir(metadataDir)) { - LOG.error("Formatting ledger metadata directory {} failed", metadataDir); - return false; + if (!Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath())) { + File metadataDir = new File(conf.getGcEntryLogMetadataCachePath()); + if (!cleanDir(metadataDir)) { + LOG.error("Formatting ledger metadata directory {} failed", metadataDir); + return false; + } } LOG.info("Bookie format completed successfully"); return true; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index bf00566f1b3..86f63001352 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -22,9 +22,11 @@ package org.apache.bookkeeper.bookie; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.IOException; +import java.nio.file.Files; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; @@ -48,6 +50,7 @@ import org.apache.commons.lang3.mutable.MutableLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; /** * This is the garbage collector thread that runs in the background to @@ -120,6 +123,7 @@ public class GarbageCollectorThread extends SafeRunnable { final GarbageCleaner garbageCleaner; final ServerConfiguration conf; + final LedgerDirsManager ledgerDirsManager; /** * Create a garbage collector thread. @@ -129,8 +133,10 @@ public class GarbageCollectorThread extends SafeRunnable { * @throws IOException */ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerManager, - final CompactableLedgerStorage ledgerStorage, StatsLogger statsLogger) throws IOException { - this(conf, ledgerManager, ledgerStorage, statsLogger, + final LedgerDirsManager ledgerDirsManager, + final CompactableLedgerStorage ledgerStorage, + StatsLogger statsLogger) throws IOException { + this(conf, ledgerManager, ledgerDirsManager, ledgerStorage, statsLogger, Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("GarbageCollectorThread"))); } @@ -143,6 +149,7 @@ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerMana */ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerManager, + final LedgerDirsManager ledgerDirsManager, final CompactableLedgerStorage ledgerStorage, StatsLogger statsLogger, ScheduledExecutorService gcExecutor) @@ -150,6 +157,7 @@ public GarbageCollectorThread(ServerConfiguration conf, this.gcExecutor = gcExecutor; this.conf = conf; + this.ledgerDirsManager = ledgerDirsManager; this.entryLogger = ledgerStorage.getEntryLogger(); this.entryLogMetaMap = createEntryLogMetadataMap(); this.ledgerStorage = ledgerStorage; @@ -260,11 +268,13 @@ public void removeEntryLog(long logToRemove) { private EntryLogMetadataMap createEntryLogMetadataMap() throws IOException { if (conf.isGcEntryLogMetadataCacheEnabled()) { - String baseDir = this.conf.getGcEntryLogMetadataCachePath(); + String baseDir = Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath()) ? + this.ledgerDirsManager.getAllLedgerDirs().get(0).getPath() : conf.getGcEntryLogMetadataCachePath(); try { return new PersistentEntryLogMetadataMap(baseDir, conf); } catch (IOException e) { - LOG.error("Failed to initialize persistent-metadata-map , clean up {}", baseDir, e); + LOG.error("Failed to initialize persistent-metadata-map , clean up {}", + baseDir + "/" + METADATA_CACHE, e); throw e; } } else { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java index 4cc34c76534..6d204b8eb40 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java @@ -192,7 +192,8 @@ public void initializeWithEntryLogger(ServerConfiguration conf, this.entryLogger.addListener(this); ledgerCache = new LedgerCacheImpl(conf, activeLedgers, null == indexDirsManager ? ledgerDirsManager : indexDirsManager, statsLogger); - gcThread = new GarbageCollectorThread(conf, ledgerManager, this, statsLogger.scope("gc")); + gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager, + this, statsLogger.scope("gc")); pageSize = conf.getPageSize(); ledgerDirsManager.addLedgerDirsListener(getLedgerDirsListener()); // Expose Stats diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java index 812ab845385..b7d1bd446cf 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java @@ -40,6 +40,7 @@ import org.apache.bookkeeper.conf.ServerConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; /** * Persistent entryLogMetadata-map that stores entry-loggers metadata into @@ -77,14 +78,14 @@ protected DataInputStream initialValue() { }; public PersistentEntryLogMetadataMap(String metadataPath, ServerConfiguration conf) throws IOException { - LOG.info("Loading persistent entrylog metadata-map from {}", metadataPath); + LOG.info("Loading persistent entrylog metadata-map from {}", metadataPath + "/" + METADATA_CACHE); File dir = new File(metadataPath); if (!dir.mkdirs() && !dir.exists()) { String err = "Unable to create directory " + dir; LOG.error(err); throw new IOException(err); } - metadataMapDB = KeyValueStorageRocksDB.factory.newKeyValueStorage(metadataPath, "metadata-cache", + metadataMapDB = KeyValueStorageRocksDB.factory.newKeyValueStorage(metadataPath, METADATA_CACHE, DbConfigType.Small, conf); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java index 4643652c81f..c5decc12500 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java @@ -179,7 +179,7 @@ public SingleDirectoryDbLedgerStorage(ServerConfiguration conf, LedgerManager le TransientLedgerInfo.LEDGER_INFO_CACHING_TIME_MINUTES, TimeUnit.MINUTES); entryLogger = new EntryLogger(conf, ledgerDirsManager, null, statsLogger, allocator); - gcThread = new GarbageCollectorThread(conf, ledgerManager, this, statsLogger); + gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager,this, statsLogger); dbLedgerStorageStats = new DbLedgerStorageStats( ledgerDirStatsLogger, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 7af77d501f7..9453721fe81 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -17,7 +17,6 @@ */ package org.apache.bookkeeper.conf; -import static org.apache.bookkeeper.util.BookKeeperConstants.ENTRYLOG_INDEX_CACHE; import static org.apache.bookkeeper.util.BookKeeperConstants.MAX_LOG_SIZE_LIMIT; import com.google.common.annotations.Beta; @@ -511,17 +510,20 @@ public ServerConfiguration setGcEntryLogMetadataCacheEnabled( * gcPersistentEntrylogMetadataMapEnabled is true. * * @return entrylog metadata-map persistent store dir path.(default: it - * creates a sub-directory under a first available base ledger - * directory with name "entrylogIndexCache"). + * creates a sub-directory under each ledger + * directory with name "metadata-cache". If it set, it only works for one ledger directory + * configured for ledgerDirectories). */ public String getGcEntryLogMetadataCachePath() { - return getString(GC_ENTRYLOG_METADATA_CACHE_PATH, getLedgerDirNames()[0] + "/" + ENTRYLOG_INDEX_CACHE); + return getString(GC_ENTRYLOG_METADATA_CACHE_PATH, null); } /** * Set directory to persist Entrylog metadata if gcPersistentEntrylogMetadataMapEnabled is true. + * If it set, it only works for one ledger directory configured for ledgerDirectories. For multi ledgerDirectory + * configured, keep the default value is the best practice. * - * @param gcPersistentEntrylogMetadataMapPath. + * @param gcEntrylogMetadataCachePath. * @return server configuration. */ public ServerConfiguration setGcEntryLogMetadataCachePath(String gcEntrylogMetadataCachePath) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java index a08ff59613c..bff0d62ce35 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java @@ -31,7 +31,7 @@ public class BookKeeperConstants { public static final String BOOKIE_STATUS_FILENAME = "BOOKIE_STATUS"; public static final String PASSWD = "passwd"; public static final String CURRENT_DIR = "current"; - public static final String ENTRYLOG_INDEX_CACHE = "entrylogIndexCache"; + public static final String METADATA_CACHE = "metadata-cache"; public static final String READONLY = "readonly"; // ////////////////////////// From 3e2b8b03e7e7fa9344fd0aaf6c21b44ce2807ce0 Mon Sep 17 00:00:00 2001 From: chenhang Date: Sun, 26 Dec 2021 16:32:15 +0800 Subject: [PATCH 2/3] format code --- .../java/org/apache/bookkeeper/conf/ServerConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 9453721fe81..7a4cd0bc27f 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -523,7 +523,7 @@ public String getGcEntryLogMetadataCachePath() { * If it set, it only works for one ledger directory configured for ledgerDirectories. For multi ledgerDirectory * configured, keep the default value is the best practice. * - * @param gcEntrylogMetadataCachePath. + * @param gcEntrylogMetadataCachePath * @return server configuration. */ public ServerConfiguration setGcEntryLogMetadataCachePath(String gcEntrylogMetadataCachePath) { From 916beec222433e75eda1740937dfa78e3a270c1d Mon Sep 17 00:00:00 2001 From: chenhang Date: Sun, 26 Dec 2021 19:43:37 +0800 Subject: [PATCH 3/3] format code --- .../apache/bookkeeper/bookie/GarbageCollectorThread.java | 7 +++---- .../bookie/storage/ldb/PersistentEntryLogMetadataMap.java | 2 +- .../bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index 86f63001352..6949955158f 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -21,12 +21,12 @@ package org.apache.bookkeeper.bookie; +import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.IOException; -import java.nio.file.Files; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; @@ -50,7 +50,6 @@ import org.apache.commons.lang3.mutable.MutableLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; /** * This is the garbage collector thread that runs in the background to @@ -268,8 +267,8 @@ public void removeEntryLog(long logToRemove) { private EntryLogMetadataMap createEntryLogMetadataMap() throws IOException { if (conf.isGcEntryLogMetadataCacheEnabled()) { - String baseDir = Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath()) ? - this.ledgerDirsManager.getAllLedgerDirs().get(0).getPath() : conf.getGcEntryLogMetadataCachePath(); + String baseDir = Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath()) + ? this.ledgerDirsManager.getAllLedgerDirs().get(0).getPath() : conf.getGcEntryLogMetadataCachePath(); try { return new PersistentEntryLogMetadataMap(baseDir, conf); } catch (IOException e) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java index b7d1bd446cf..95b365970f9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java @@ -20,6 +20,7 @@ */ package org.apache.bookkeeper.bookie.storage.ldb; +import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; import io.netty.util.concurrent.FastThreadLocal; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -40,7 +41,6 @@ import org.apache.bookkeeper.conf.ServerConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; /** * Persistent entryLogMetadata-map that stores entry-loggers metadata into diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java index c5decc12500..052290103e2 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java @@ -179,7 +179,7 @@ public SingleDirectoryDbLedgerStorage(ServerConfiguration conf, LedgerManager le TransientLedgerInfo.LEDGER_INFO_CACHING_TIME_MINUTES, TimeUnit.MINUTES); entryLogger = new EntryLogger(conf, ledgerDirsManager, null, statsLogger, allocator); - gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager,this, statsLogger); + gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager, this, statsLogger); dbLedgerStorageStats = new DbLedgerStorageStats( ledgerDirStatsLogger,