diff --git a/bookkeeper-dist/bkctl/build.gradle b/bookkeeper-dist/bkctl/build.gradle
index 27d27533eeb..af4cce6ecc0 100644
--- a/bookkeeper-dist/bkctl/build.gradle
+++ b/bookkeeper-dist/bkctl/build.gradle
@@ -72,6 +72,9 @@ distributions {
"log4j.cli.properties",
"log4j.shell.properties",
"nettyenv.sh",
+ "default_rocksdb.conf",
+ "entry_location_rocksdb.conf",
+ "ledger_metadata_rocksdb.conf",
)
}
}
diff --git a/bookkeeper-dist/src/assemble/bkctl.xml b/bookkeeper-dist/src/assemble/bkctl.xml
index cb4ea8f3a3b..1e7d5187557 100644
--- a/bookkeeper-dist/src/assemble/bkctl.xml
+++ b/bookkeeper-dist/src/assemble/bkctl.xml
@@ -41,6 +41,9 @@
log4j.cli.properties
log4j.shell.properties
nettyenv.sh
+ default_rocksdb.conf
+ entry_location_rocksdb.conf
+ ledger_metadata_rocksdb.conf
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
index 2ba63cc222c..0a563a1c9ce 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
@@ -379,7 +379,7 @@ public static void readLedgerIndexEntries(long ledgerId, ServerConfiguration ser
EntryLocationIndex entryLocationIndex = new EntryLocationIndex(serverConf,
(basePath, subPath, dbConfigType, conf1) ->
- new KeyValueStorageRocksDB(basePath, subPath, DbConfigType.Small, conf1, true),
+ new KeyValueStorageRocksDB(basePath, subPath, DbConfigType.Default, conf1, true),
ledgerBasePath, NullStatsLogger.INSTANCE);
try {
long lastEntryId = entryLocationIndex.getLastEntryInLedger(ledgerId);
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/EntryLocationIndex.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/EntryLocationIndex.java
index 6b7a20d7485..79784651e90 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/EntryLocationIndex.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/EntryLocationIndex.java
@@ -53,7 +53,7 @@ public class EntryLocationIndex implements Closeable {
public EntryLocationIndex(ServerConfiguration conf, KeyValueStorageFactory storageFactory, String basePath,
StatsLogger stats) throws IOException {
- locationsDb = storageFactory.newKeyValueStorage(basePath, "locations", DbConfigType.Huge, conf);
+ locationsDb = storageFactory.newKeyValueStorage(basePath, "locations", DbConfigType.EntryLocation, conf);
this.stats = new EntryLocationIndexStats(
stats,
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageFactory.java
index f3023c1ebae..08f68feff78 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageFactory.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageFactory.java
@@ -33,8 +33,9 @@ public interface KeyValueStorageFactory {
* Enum used to specify different config profiles in the underlying storage.
*/
enum DbConfigType {
- Small, // Used for ledgers db, doesn't need particular configuration
- Huge // Used for location index, lots of writes and much bigger dataset
+ Default, // Used for default,command until or test case
+ LedgerMetadata, // Used for ledgers db, doesn't need particular configuration
+ EntryLocation // Used for location index, lots of writes and much bigger dataset
}
KeyValueStorage newKeyValueStorage(String defaultBasePath, String subPath, DbConfigType dbConfigType,
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
index e6eb1978c9a..293004c4923 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
@@ -22,27 +22,21 @@
import static com.google.common.base.Preconditions.checkState;
-//CHECKSTYLE.OFF: IllegalImport
-import io.netty.util.internal.PlatformDependent;
-//CHECKSTYLE.ON: IllegalImport
-
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map.Entry;
-import java.util.concurrent.TimeUnit;
import org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageFactory.DbConfigType;
import org.apache.bookkeeper.conf.ServerConfiguration;
-import org.rocksdb.BlockBasedTableConfig;
-import org.rocksdb.BloomFilter;
-import org.rocksdb.Cache;
-import org.rocksdb.ChecksumType;
-import org.rocksdb.CompressionType;
-import org.rocksdb.InfoLogLevel;
-import org.rocksdb.LRUCache;
-import org.rocksdb.Options;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.DBOptions;
+import org.rocksdb.Env;
+import org.rocksdb.OptionsUtil;
import org.rocksdb.ReadOptions;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
@@ -65,25 +59,12 @@ public class KeyValueStorageRocksDB implements KeyValueStorage {
private final WriteOptions optionSync;
private final WriteOptions optionDontSync;
- private final Cache cache;
private final ReadOptions optionCache;
private final ReadOptions optionDontCache;
-
private final WriteBatch emptyBatch;
private static final String ROCKSDB_LOG_PATH = "dbStorage_rocksDB_logPath";
- private static final String ROCKSDB_LOG_LEVEL = "dbStorage_rocksDB_logLevel";
- private static final String ROCKSDB_LZ4_COMPRESSION_ENABLED = "dbStorage_rocksDB_lz4CompressionEnabled";
- private static final String ROCKSDB_WRITE_BUFFER_SIZE_MB = "dbStorage_rocksDB_writeBufferSizeMB";
- private static final String ROCKSDB_SST_SIZE_MB = "dbStorage_rocksDB_sstSizeInMB";
- private static final String ROCKSDB_BLOCK_SIZE = "dbStorage_rocksDB_blockSize";
- private static final String ROCKSDB_BLOOM_FILTERS_BITS_PER_KEY = "dbStorage_rocksDB_bloomFilterBitsPerKey";
- private static final String ROCKSDB_BLOCK_CACHE_SIZE = "dbStorage_rocksDB_blockCacheSize";
- private static final String ROCKSDB_NUM_LEVELS = "dbStorage_rocksDB_numLevels";
- private static final String ROCKSDB_NUM_FILES_IN_LEVEL0 = "dbStorage_rocksDB_numFilesInLevel0";
- private static final String ROCKSDB_MAX_SIZE_IN_LEVEL1_MB = "dbStorage_rocksDB_maxSizeInLevel1MB";
- private static final String ROCKSDB_FORMAT_VERSION = "dbStorage_rocksDB_format_version";
public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbConfigType, ServerConfiguration conf)
throws IOException {
@@ -105,104 +86,37 @@ public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbCo
this.optionDontCache = new ReadOptions();
this.emptyBatch = new WriteBatch();
- try (Options options = new Options()) {
- options.setCreateIfMissing(true);
-
- if (dbConfigType == DbConfigType.Huge) {
- // Set default RocksDB block-cache size to 10% / numberOfLedgers of direct memory, unless override
- int ledgerDirsSize = conf.getLedgerDirNames().length;
- long defaultRocksDBBlockCacheSizeBytes = PlatformDependent.maxDirectMemory() / ledgerDirsSize / 10;
- long blockCacheSize = DbLedgerStorage.getLongVariableOrDefault(conf, ROCKSDB_BLOCK_CACHE_SIZE,
- defaultRocksDBBlockCacheSizeBytes);
-
- long writeBufferSizeMB = conf.getInt(ROCKSDB_WRITE_BUFFER_SIZE_MB, 64);
- long sstSizeMB = conf.getInt(ROCKSDB_SST_SIZE_MB, 64);
- int numLevels = conf.getInt(ROCKSDB_NUM_LEVELS, -1);
- int numFilesInLevel0 = conf.getInt(ROCKSDB_NUM_FILES_IN_LEVEL0, 4);
- long maxSizeInLevel1MB = conf.getLong(ROCKSDB_MAX_SIZE_IN_LEVEL1_MB, 256);
- int blockSize = conf.getInt(ROCKSDB_BLOCK_SIZE, 64 * 1024);
- int bloomFilterBitsPerKey = conf.getInt(ROCKSDB_BLOOM_FILTERS_BITS_PER_KEY, 10);
- boolean lz4CompressionEnabled = conf.getBoolean(ROCKSDB_LZ4_COMPRESSION_ENABLED, true);
- int formatVersion = conf.getInt(ROCKSDB_FORMAT_VERSION, 2);
-
- if (lz4CompressionEnabled) {
- options.setCompressionType(CompressionType.LZ4_COMPRESSION);
- }
- options.setWriteBufferSize(writeBufferSizeMB * 1024 * 1024);
- options.setMaxWriteBufferNumber(4);
- if (numLevels > 0) {
- options.setNumLevels(numLevels);
- }
- options.setLevelZeroFileNumCompactionTrigger(numFilesInLevel0);
- options.setMaxBytesForLevelBase(maxSizeInLevel1MB * 1024 * 1024);
- options.setMaxBackgroundJobs(32);
- options.setIncreaseParallelism(32);
- options.setMaxTotalWalSize(512 * 1024 * 1024);
- options.setMaxOpenFiles(-1);
- options.setTargetFileSizeBase(sstSizeMB * 1024 * 1024);
- options.setDeleteObsoleteFilesPeriodMicros(TimeUnit.HOURS.toMicros(1));
-
- this.cache = new LRUCache(blockCacheSize);
- BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
- tableOptions.setBlockSize(blockSize);
- tableOptions.setBlockCache(cache);
- tableOptions.setFormatVersion(formatVersion);
- tableOptions.setChecksumType(ChecksumType.kxxHash);
- if (bloomFilterBitsPerKey > 0) {
- tableOptions.setFilterPolicy(new BloomFilter(bloomFilterBitsPerKey, false));
- }
-
- // Options best suited for HDDs
- tableOptions.setCacheIndexAndFilterBlocks(true);
- options.setLevelCompactionDynamicLevelBytes(true);
-
- options.setTableFormatConfig(tableOptions);
+ String dbFilePath = "";
+ DBOptions dbOptions = new DBOptions();
+ final List cfDescs = new ArrayList<>();
+ final List cfHandles = new ArrayList<>();
+ try {
+ if (dbConfigType == DbConfigType.EntryLocation) {
+ dbFilePath = conf.getEntryLocationRocksdbConf();
+ } else if (dbConfigType == DbConfigType.LedgerMetadata) {
+ dbFilePath = conf.getLedgerMetadataRocksdbConf();
} else {
- this.cache = null;
+ dbFilePath = conf.getDefaultRocksDBConf();
}
+ OptionsUtil.loadOptionsFromFile(dbFilePath, Env.getDefault(), dbOptions, cfDescs, false);
// Configure file path
String logPath = conf.getString(ROCKSDB_LOG_PATH, "");
if (!logPath.isEmpty()) {
Path logPathSetting = FileSystems.getDefault().getPath(logPath, subPath);
Files.createDirectories(logPathSetting);
log.info("RocksDB<{}> log path: {}", subPath, logPathSetting);
- options.setDbLogDir(logPathSetting.toString());
+ dbOptions.setDbLogDir(logPathSetting.toString());
}
String path = FileSystems.getDefault().getPath(basePath, subPath).toFile().toString();
- // Configure log level
- String logLevel = conf.getString(ROCKSDB_LOG_LEVEL, "info");
- switch (logLevel) {
- case "debug":
- options.setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL);
- break;
- case "info":
- options.setInfoLogLevel(InfoLogLevel.INFO_LEVEL);
- break;
- case "warn":
- options.setInfoLogLevel(InfoLogLevel.WARN_LEVEL);
- break;
- case "error":
- options.setInfoLogLevel(InfoLogLevel.ERROR_LEVEL);
- break;
- default:
- log.warn("Unrecognized RockDB log level: {}", logLevel);
- }
-
- // Keep log files for 1month
- options.setKeepLogFileNum(30);
- options.setLogFileTimeToRoll(TimeUnit.DAYS.toSeconds(1));
-
- try {
- if (readOnly) {
- db = RocksDB.openReadOnly(options, path);
- } else {
- db = RocksDB.open(options, path);
- }
- } catch (RocksDBException e) {
- throw new IOException("Error open RocksDB database", e);
+ if (readOnly) {
+ db = RocksDB.openReadOnly(dbOptions, path, cfDescs, cfHandles);
+ } else {
+ db = RocksDB.open(dbOptions, path, cfDescs, cfHandles);
}
+ } catch (RocksDBException e) {
+ throw new IOException("Error open RocksDB database", e);
}
optionSync.setSync(true);
@@ -215,9 +129,6 @@ public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbCo
@Override
public void close() throws IOException {
db.close();
- if (cache != null) {
- cache.close();
- }
optionSync.close();
optionDontSync.close();
optionCache.close();
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java
index eb7548ca9f2..c48410048c3 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgerMetadataIndex.java
@@ -70,7 +70,7 @@ public class LedgerMetadataIndex implements Closeable {
public LedgerMetadataIndex(ServerConfiguration conf, KeyValueStorageFactory storageFactory, String basePath,
StatsLogger stats) throws IOException {
- ledgersDb = storageFactory.newKeyValueStorage(basePath, "ledgers", DbConfigType.Small, conf);
+ ledgersDb = storageFactory.newKeyValueStorage(basePath, "ledgers", DbConfigType.LedgerMetadata, conf);
ledgers = new ConcurrentLongHashMap<>();
ledgersCount = new AtomicInteger();
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexCheckOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexCheckOp.java
index a48de342943..9a2a5f5cafd 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexCheckOp.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexCheckOp.java
@@ -59,7 +59,7 @@ public boolean initiate() throws IOException {
try {
KeyValueStorage index = new KeyValueStorageRocksDB(basePath, LedgersSubPath,
- DbConfigType.Small, conf, true);
+ DbConfigType.Default, conf, true);
// Read all ledgers from db
KeyValueStorage.CloseableIterator> iterator = index.iterator();
int ctr = 0;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexRebuildOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexRebuildOp.java
index 15410ef1193..7e10de8fe6c 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexRebuildOp.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LedgersIndexRebuildOp.java
@@ -91,7 +91,7 @@ public boolean initiate() {
+ "Starting to build a new ledgers index", ledgers.size());
try (KeyValueStorage newIndex = KeyValueStorageRocksDB.factory.newKeyValueStorage(
- basePath, tempLedgersSubPath, DbConfigType.Small, conf)) {
+ basePath, tempLedgersSubPath, DbConfigType.Default, conf)) {
LOG.info("Created ledgers index at temp location {}", tempPath);
for (Long ledgerId : ledgers) {
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java
index 7eae6da8a44..55c5c90c31e 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/LocationsIndexRebuildOp.java
@@ -77,7 +77,7 @@ public void initiate() throws IOException {
LOG.info("Found {} active ledgers in ledger manager", activeLedgers.size());
KeyValueStorage newIndex = KeyValueStorageRocksDB.factory.newKeyValueStorage(basePath, "locations",
- DbConfigType.Huge, conf);
+ DbConfigType.Default, conf);
int totalEntryLogs = entryLogs.size();
int completedEntryLogs = 0;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java
index 99f60691f55..ec289fd16c2 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/PersistentEntryLogMetadataMap.java
@@ -85,7 +85,7 @@ public PersistentEntryLogMetadataMap(String metadataPath, ServerConfiguration co
throw new IOException(err);
}
metadataMapDB = KeyValueStorageRocksDB.factory.newKeyValueStorage(metadataPath, METADATA_CACHE,
- DbConfigType.Small, conf);
+ DbConfigType.Default, conf);
}
@Override
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
index 3f099eb8a27..a26616c2acf 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
@@ -26,6 +26,7 @@
import io.netty.util.internal.PlatformDependent;
// CHECKSTYLE.ON: IllegalImport
import java.io.File;
+import java.net.URL;
import java.util.concurrent.TimeUnit;
import org.apache.bookkeeper.bookie.FileChannelProvider;
import org.apache.bookkeeper.bookie.InterleavedLedgerStorage;
@@ -318,6 +319,15 @@ public class ServerConfiguration extends AbstractConfiguration