Support multi ledger directories for rocksdb backend entryMetadataMap

bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java

@@ -25,6 +25,7 @@
 import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_INDEX_SCOPE;
 import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_LEDGER_SCOPE;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.ByteBufAllocator;
@@ -1186,10 +1187,12 @@ public static boolean format(ServerConfiguration conf,
 
         // Clean up metadata directories if they are separate from the
         // ledger dirs
-        File metadataDir = new File(conf.getGcEntryLogMetadataCachePath());
-        if (!cleanDir(metadataDir)) {
-            LOG.error("Formatting ledger metadata directory {} failed", metadataDir);
-            return false;
+        if (!Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath())) {
+            File metadataDir = new File(conf.getGcEntryLogMetadataCachePath());
+            if (!cleanDir(metadataDir)) {
+                LOG.error("Formatting ledger metadata directory {} failed", metadataDir);
+                return false;
+            }
         }
         LOG.info("Bookie format completed successfully");
         return true;

bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java

@@ -21,7 +21,9 @@
 
 package org.apache.bookkeeper.bookie;
 
+import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
 import io.netty.util.concurrent.DefaultThreadFactory;
 
 import java.io.IOException;
@@ -120,6 +122,7 @@ public class GarbageCollectorThread extends SafeRunnable {
     final GarbageCleaner garbageCleaner;
 
     final ServerConfiguration conf;
+    final LedgerDirsManager ledgerDirsManager;
 
     /**
      * Create a garbage collector thread.
@@ -129,8 +132,10 @@ public class GarbageCollectorThread extends SafeRunnable {
      * @throws IOException
      */
     public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerManager,
-            final CompactableLedgerStorage ledgerStorage, StatsLogger statsLogger) throws IOException {
-        this(conf, ledgerManager, ledgerStorage, statsLogger,
+                                  final LedgerDirsManager ledgerDirsManager,
+                                  final CompactableLedgerStorage ledgerStorage,
+                                  StatsLogger statsLogger) throws IOException {
+        this(conf, ledgerManager, ledgerDirsManager, ledgerStorage, statsLogger,
                 Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory("GarbageCollectorThread")));
     }
 
@@ -143,13 +148,15 @@ public GarbageCollectorThread(ServerConfiguration conf, LedgerManager ledgerMana
      */
     public GarbageCollectorThread(ServerConfiguration conf,
                                   LedgerManager ledgerManager,
+                                  final LedgerDirsManager ledgerDirsManager,
                                   final CompactableLedgerStorage ledgerStorage,
                                   StatsLogger statsLogger,
                                   ScheduledExecutorService gcExecutor)
         throws IOException {
         this.gcExecutor = gcExecutor;
         this.conf = conf;
 
+        this.ledgerDirsManager = ledgerDirsManager;
         this.entryLogger = ledgerStorage.getEntryLogger();
         this.entryLogMetaMap = createEntryLogMetadataMap();
         this.ledgerStorage = ledgerStorage;
@@ -260,11 +267,13 @@ public void removeEntryLog(long logToRemove) {
 
     private EntryLogMetadataMap createEntryLogMetadataMap() throws IOException {
         if (conf.isGcEntryLogMetadataCacheEnabled()) {
-            String baseDir = this.conf.getGcEntryLogMetadataCachePath();
+            String baseDir = Strings.isNullOrEmpty(conf.getGcEntryLogMetadataCachePath())
+                ? this.ledgerDirsManager.getAllLedgerDirs().get(0).getPath() : conf.getGcEntryLogMetadataCachePath();
             try {
                 return new PersistentEntryLogMetadataMap(baseDir, conf);
             } catch (IOException e) {
-                LOG.error("Failed to initialize persistent-metadata-map , clean up {}", baseDir, e);
+                LOG.error("Failed to initialize persistent-metadata-map , clean up {}",
+                    baseDir + "/" + METADATA_CACHE, e);
                 throw e;
             }
         } else {

bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java

@@ -192,7 +192,8 @@ public void initializeWithEntryLogger(ServerConfiguration conf,
         this.entryLogger.addListener(this);
         ledgerCache = new LedgerCacheImpl(conf, activeLedgers,
                 null == indexDirsManager ? ledgerDirsManager : indexDirsManager, statsLogger);
-        gcThread = new GarbageCollectorThread(conf, ledgerManager, this, statsLogger.scope("gc"));
+        gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager,
+            this, statsLogger.scope("gc"));
         pageSize = conf.getPageSize();
         ledgerDirsManager.addLedgerDirsListener(getLedgerDirsListener());
         // Expose Stats

bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java

@@ -20,6 +20,7 @@
  */
 package org.apache.bookkeeper.bookie.storage.ldb;
 
+import static org.apache.bookkeeper.util.BookKeeperConstants.METADATA_CACHE;
 import io.netty.util.concurrent.FastThreadLocal;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -77,14 +78,14 @@ protected DataInputStream initialValue() {
     };
 
     public PersistentEntryLogMetadataMap(String metadataPath, ServerConfiguration conf) throws IOException {
-        LOG.info("Loading persistent entrylog metadata-map from {}", metadataPath);
+        LOG.info("Loading persistent entrylog metadata-map from {}", metadataPath + "/" + METADATA_CACHE);
         File dir = new File(metadataPath);
         if (!dir.mkdirs() && !dir.exists()) {
             String err = "Unable to create directory " + dir;
             LOG.error(err);
             throw new IOException(err);
         }
-        metadataMapDB = KeyValueStorageRocksDB.factory.newKeyValueStorage(metadataPath, "metadata-cache",
+        metadataMapDB = KeyValueStorageRocksDB.factory.newKeyValueStorage(metadataPath, METADATA_CACHE,
                 DbConfigType.Small, conf);
     }
 

bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java

@@ -179,7 +179,7 @@ public SingleDirectoryDbLedgerStorage(ServerConfiguration conf, LedgerManager le
                 TransientLedgerInfo.LEDGER_INFO_CACHING_TIME_MINUTES, TimeUnit.MINUTES);
 
         entryLogger = new EntryLogger(conf, ledgerDirsManager, null, statsLogger, allocator);
-        gcThread = new GarbageCollectorThread(conf, ledgerManager, this, statsLogger);
+        gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager, this, statsLogger);
 
         dbLedgerStorageStats = new DbLedgerStorageStats(
                 ledgerDirStatsLogger,

bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java

@@ -17,7 +17,6 @@
  */
 package org.apache.bookkeeper.conf;
 
-import static org.apache.bookkeeper.util.BookKeeperConstants.ENTRYLOG_INDEX_CACHE;
 import static org.apache.bookkeeper.util.BookKeeperConstants.MAX_LOG_SIZE_LIMIT;
 
 import com.google.common.annotations.Beta;
@@ -511,17 +510,20 @@ public ServerConfiguration setGcEntryLogMetadataCacheEnabled(
      * gcPersistentEntrylogMetadataMapEnabled is true.
      *
      * @return entrylog metadata-map persistent store dir path.(default: it
-     *         creates a sub-directory under a first available base ledger
-     *         directory with name "entrylogIndexCache").
+     *         creates a sub-directory under each ledger
+     *         directory with name "metadata-cache". If it set, it only works for one ledger directory
+     *         configured for ledgerDirectories).
      */
     public String getGcEntryLogMetadataCachePath() {
-        return getString(GC_ENTRYLOG_METADATA_CACHE_PATH, getLedgerDirNames()[0] + "/" + ENTRYLOG_INDEX_CACHE);
+        return getString(GC_ENTRYLOG_METADATA_CACHE_PATH, null);
     }
 
     /**
      * Set directory to persist Entrylog metadata if gcPersistentEntrylogMetadataMapEnabled is true.
+     * If it set, it only works for one ledger directory configured for ledgerDirectories. For multi ledgerDirectory
+     * configured, keep the default value is the best practice.
      *
-     * @param gcPersistentEntrylogMetadataMapPath.
+     * @param gcEntrylogMetadataCachePath
      * @return server configuration.
      */
     public ServerConfiguration setGcEntryLogMetadataCachePath(String gcEntrylogMetadataCachePath) {

bookkeeper-server/src/main/java/org/apache/bookkeeper/util/BookKeeperConstants.java

@@ -31,7 +31,7 @@ public class BookKeeperConstants {
     public static final String BOOKIE_STATUS_FILENAME = "BOOKIE_STATUS";
     public static final String PASSWD = "passwd";
     public static final String CURRENT_DIR = "current";
-    public static final String ENTRYLOG_INDEX_CACHE = "entrylogIndexCache";
+    public static final String METADATA_CACHE = "metadata-cache";
     public static final String READONLY = "readonly";
 
     // //////////////////////////