diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java index ed57d7b7e16..71fbf44bbcd 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java @@ -25,6 +25,7 @@ import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_INDEX_SCOPE; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_LEDGER_SCOPE; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; @@ -1186,10 +1187,12 @@ public static boolean format(ServerConfiguration conf, // Clean up metadata directories if they are separate from the // ledger dirs - File metadataDir = new File(conf.getGcEntryLogMetadataCachePath()); - if (!cleanDir(metadataDir)) { - LOG.error("Formatting ledger metadata directory {} failed", metadataDir); - return false; + if (!Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath())) { + File metadataDir = new File(conf.getGcEntryLogMetadataCachePath()); + if (!cleanDir(metadataDir)) { + LOG.error("Formatting ledger metadata directory {} failed", metadataDir); + return false; + } } LOG.info("Bookie format completed successfully"); return true; diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index bf00566f1b3..6949955158f 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -21,7 +21,9 @@ package org.apache.bookkeeper.bookie; +import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.IOException; @@ -120,6 +122,7 @@ public class GarbageCollectorThread extends SafeRunnable { final GarbageCleaner garbageCleaner; final ServerConfiguration conf; + final LedgerDirsManager ledgerDirsManager; /** * Create a garbage collector thread. @@ -129,8 +132,10 @@ public class GarbageCollectorThread extends SafeRunnable { * @throws IOException */ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerManager, - final CompactableLedgerStorage ledgerStorage, StatsLogger statsLogger) throws IOException { - this(conf, ledgerManager, ledgerStorage, statsLogger, + final LedgerDirsManager ledgerDirsManager, + final CompactableLedgerStorage ledgerStorage, + StatsLogger statsLogger) throws IOException { + this(conf, ledgerManager, ledgerDirsManager, ledgerStorage, statsLogger, Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("GarbageCollectorThread"))); } @@ -143,6 +148,7 @@ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerMana */ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerManager, + final LedgerDirsManager ledgerDirsManager, final CompactableLedgerStorage ledgerStorage, StatsLogger statsLogger, ScheduledExecutorService gcExecutor) @@ -150,6 +156,7 @@ public GarbageCollectorThread(ServerConfiguration conf, this.gcExecutor = gcExecutor; this.conf = conf; + this.ledgerDirsManager = ledgerDirsManager; this.entryLogger = ledgerStorage.getEntryLogger(); this.entryLogMetaMap = createEntryLogMetadataMap(); this.ledgerStorage = ledgerStorage; @@ -260,11 +267,13 @@ public void removeEntryLog(long logToRemove) { private EntryLogMetadataMap createEntryLogMetadataMap() throws IOException { if (conf.isGcEntryLogMetadataCacheEnabled()) { - String baseDir = this.conf.getGcEntryLogMetadataCachePath(); + String baseDir = Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath()) + ? this.ledgerDirsManager.getAllLedgerDirs().get(0).getPath() : conf.getGcEntryLogMetadataCachePath(); try { return new PersistentEntryLogMetadataMap(baseDir, conf); } catch (IOException e) { - LOG.error("Failed to initialize persistent-metadata-map , clean up {}", baseDir, e); + LOG.error("Failed to initialize persistent-metadata-map , clean up {}", + baseDir + "/" + METADATA_CACHE, e); throw e; } } else { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java index 4cc34c76534..6d204b8eb40 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java @@ -192,7 +192,8 @@ public void initializeWithEntryLogger(ServerConfiguration conf, this.entryLogger.addListener(this); ledgerCache = new LedgerCacheImpl(conf, activeLedgers, null == indexDirsManager ? ledgerDirsManager : indexDirsManager, statsLogger); - gcThread = new GarbageCollectorThread(conf, ledgerManager, this, statsLogger.scope("gc")); + gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager, + this, statsLogger.scope("gc")); pageSize = conf.getPageSize(); ledgerDirsManager.addLedgerDirsListener(getLedgerDirsListener()); // Expose Stats diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java index 812ab845385..95b365970f9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java @@ -20,6 +20,7 @@ */ package org.apache.bookkeeper.bookie.storage.ldb; +import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE; import io.netty.util.concurrent.FastThreadLocal; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -77,14 +78,14 @@ protected DataInputStream initialValue() { }; public PersistentEntryLogMetadataMap(String metadataPath, ServerConfiguration conf) throws IOException { - LOG.info("Loading persistent entrylog metadata-map from {}", metadataPath); + LOG.info("Loading persistent entrylog metadata-map from {}", metadataPath + "/" + METADATA_CACHE); File dir = new File(metadataPath); if (!dir.mkdirs() && !dir.exists()) { String err = "Unable to create directory " + dir; LOG.error(err); throw new IOException(err); } - metadataMapDB = KeyValueStorageRocksDB.factory.newKeyValueStorage(metadataPath, "metadata-cache", + metadataMapDB = KeyValueStorageRocksDB.factory.newKeyValueStorage(metadataPath, METADATA_CACHE, DbConfigType.Small, conf); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java index 4643652c81f..052290103e2 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java @@ -179,7 +179,7 @@ public SingleDirectoryDbLedgerStorage(ServerConfiguration conf, LedgerManager le TransientLedgerInfo.LEDGER_INFO_CACHING_TIME_MINUTES, TimeUnit.MINUTES); entryLogger = new EntryLogger(conf, ledgerDirsManager, null, statsLogger, allocator); - gcThread = new GarbageCollectorThread(conf, ledgerManager, this, statsLogger); + gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager, this, statsLogger); dbLedgerStorageStats = new DbLedgerStorageStats( ledgerDirStatsLogger, diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 7af77d501f7..7a4cd0bc27f 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -17,7 +17,6 @@ */ package org.apache.bookkeeper.conf; -import static org.apache.bookkeeper.util.BookKeeperConstants.ENTRYLOG_INDEX_CACHE; import static org.apache.bookkeeper.util.BookKeeperConstants.MAX_LOG_SIZE_LIMIT; import com.google.common.annotations.Beta; @@ -511,17 +510,20 @@ public ServerConfiguration setGcEntryLogMetadataCacheEnabled( * gcPersistentEntrylogMetadataMapEnabled is true. * * @return entrylog metadata-map persistent store dir path.(default: it - * creates a sub-directory under a first available base ledger - * directory with name "entrylogIndexCache"). + * creates a sub-directory under each ledger + * directory with name "metadata-cache". If it set, it only works for one ledger directory + * configured for ledgerDirectories). */ public String getGcEntryLogMetadataCachePath() { - return getString(GC_ENTRYLOG_METADATA_CACHE_PATH, getLedgerDirNames()[0] + "/" + ENTRYLOG_INDEX_CACHE); + return getString(GC_ENTRYLOG_METADATA_CACHE_PATH, null); } /** * Set directory to persist Entrylog metadata if gcPersistentEntrylogMetadataMapEnabled is true. + * If it set, it only works for one ledger directory configured for ledgerDirectories. For multi ledgerDirectory + * configured, keep the default value is the best practice. * - * @param gcPersistentEntrylogMetadataMapPath. + * @param gcEntrylogMetadataCachePath * @return server configuration. */ public ServerConfiguration setGcEntryLogMetadataCachePath(String gcEntrylogMetadataCachePath) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java index a08ff59613c..bff0d62ce35 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java @@ -31,7 +31,7 @@ public class BookKeeperConstants { public static final String BOOKIE_STATUS_FILENAME = "BOOKIE_STATUS"; public static final String PASSWD = "passwd"; public static final String CURRENT_DIR = "current"; - public static final String ENTRYLOG_INDEX_CACHE = "entrylogIndexCache"; + public static final String METADATA_CACHE = "metadata-cache"; public static final String READONLY = "readonly"; // //////////////////////////