From a8f4ab57f269e12734e42aebf5841ea45740855f Mon Sep 17 00:00:00 2001 From: chenhang Date: Tue, 21 Feb 2023 16:49:39 +0800 Subject: [PATCH 1/3] make rocksdb checksum type configurable --- .../storage/ldb/KeyValueStorageRocksDB.java | 7 +++- .../ldb/KeyValueStorageRocksDBTest.java | 38 +++++++++++++++++++ .../test_entry_location_rocksdb.conf | 16 ++++++++ conf/default_rocksdb.conf.default | 6 ++- conf/ledger_metadata_rocksdb.conf.default | 6 ++- 5 files changed, 70 insertions(+), 3 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java index deaed391464..d402e66e118 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java @@ -96,6 +96,7 @@ public class KeyValueStorageRocksDB implements KeyValueStorage { private static final String ROCKSDB_NUM_FILES_IN_LEVEL0 = "dbStorage_rocksDB_numFilesInLevel0"; private static final String ROCKSDB_MAX_SIZE_IN_LEVEL1_MB = "dbStorage_rocksDB_maxSizeInLevel1MB"; private static final String ROCKSDB_FORMAT_VERSION = "dbStorage_rocksDB_format_version"; + private static final String ROCKSDB_CHECKSUM_TYPE = "dbStorage_rocksDB_checksum_type"; public KeyValueStorageRocksDB(String basePath, String subPath, DbConfigType dbConfigType, ServerConfiguration conf) throws IOException { @@ -174,6 +175,7 @@ private RocksDB initializeRocksDBWithBookieConf(String basePath, String subPath, ServerConfiguration conf, boolean readOnly) throws IOException { Options options = new Options(); options.setCreateIfMissing(true); + ChecksumType checksumType = ChecksumType.valueOf(conf.getString(ROCKSDB_CHECKSUM_TYPE, "kxxHash")); if (dbConfigType == DbConfigType.EntryLocation) { /* Set default RocksDB block-cache size to 10% / numberOfLedgers of direct memory, unless override */ @@ -214,7 +216,7 @@ private RocksDB initializeRocksDBWithBookieConf(String basePath, String subPath, tableOptions.setBlockSize(blockSize); tableOptions.setBlockCache(cache); tableOptions.setFormatVersion(formatVersion); - tableOptions.setChecksumType(ChecksumType.kxxHash); + tableOptions.setChecksumType(checksumType); if (bloomFilterBitsPerKey > 0) { tableOptions.setFilterPolicy(new BloomFilter(bloomFilterBitsPerKey, false)); } @@ -226,6 +228,9 @@ private RocksDB initializeRocksDBWithBookieConf(String basePath, String subPath, options.setTableFormatConfig(tableOptions); } else { this.cache = null; + BlockBasedTableConfig tableOptions = new BlockBasedTableConfig(); + tableOptions.setChecksumType(checksumType); + options.setTableFormatConfig(tableOptions); } // Configure file path diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java index e7b4824913e..f3d678bfeb1 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java @@ -28,14 +28,18 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.List; +import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.conf.ServerConfiguration; import org.junit.Test; +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.ChecksumType; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyOptions; import org.rocksdb.CompressionType; import org.rocksdb.DBOptions; import org.rocksdb.Options; +@Slf4j public class KeyValueStorageRocksDBTest { @Test @@ -78,4 +82,38 @@ public void testRocksDBInitiateWithConfigurationFile() throws Exception { assertEquals(1, familyOptions.maxWriteBufferNumber()); rocksDB.close(); } + + @Test + public void testReadChecksumTypeFromBookieConfiguration() throws Exception { + ServerConfiguration configuration = new ServerConfiguration(); + configuration.setProperty("dbStorage_rocksDB_checksum_type", "kxxHash"); + configuration.setEntryLocationRocksdbConf("entry_location_rocksdb.conf"); + File tmpDir = Files.createTempDirectory("bk-kv-rocksdbtest-conf").toFile(); + Files.createDirectory(Paths.get(tmpDir.toString(), "subDir")); + KeyValueStorageRocksDB rocksDB = new KeyValueStorageRocksDB(tmpDir.toString(), "subDir", + KeyValueStorageFactory.DbConfigType.EntryLocation, configuration); + assertNull(rocksDB.getColumnFamilyDescriptors()); + + Options options = (Options) rocksDB.getOptions(); + assertEquals(ChecksumType.kxxHash, ((BlockBasedTableConfig) options.tableFormatConfig()).checksumType()); + } + + //@Test + public void testReadChecksumTypeFromConfigurationFile() throws Exception { + ServerConfiguration configuration = new ServerConfiguration(); + URL url = getClass().getClassLoader().getResource("test_entry_location_rocksdb.conf"); + configuration.setEntryLocationRocksdbConf(url.getPath()); + File tmpDir = Files.createTempDirectory("bk-kv-rocksdbtest-file").toFile(); + Files.createDirectory(Paths.get(tmpDir.toString(), "subDir")); + KeyValueStorageRocksDB rocksDB = new KeyValueStorageRocksDB(tmpDir.toString(), "subDir", + KeyValueStorageFactory.DbConfigType.EntryLocation, configuration); + assertNotNull(rocksDB.getColumnFamilyDescriptors()); + + List columnFamilyDescriptorList = rocksDB.getColumnFamilyDescriptors(); + ColumnFamilyOptions familyOptions = columnFamilyDescriptorList.get(0).getOptions(); + // There is a bug in RocksDB, which can't load BlockedBasedTableConfig from Options file. + // https://github.com/facebook/rocksdb/issues/5297 + // After the PR: https://github.com/facebook/rocksdb/pull/10826 merge, we can turn on this test. + assertEquals(ChecksumType.kxxHash, ((BlockBasedTableConfig) familyOptions.tableFormatConfig()).checksumType()); + } } diff --git a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf index 4047ef0d521..fa3cf9c8d90 100644 --- a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf @@ -31,3 +31,19 @@ write_buffer_size=1024 # set by jni: options.setMaxWriteBufferNumber max_write_buffer_number=1 + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setBlockSize + block_size=65536 + # set by jni: tableOptions.setBlockCache + block_cache=206150041 + # set by jni: tableOptions.setFormatVersion + format_version=2 + # set by jni: tableOptions.setChecksumType + checksum=kxxHash + # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder] + filter_policy=rocksdb.BloomFilter:10:false + # set by jni: tableOptions.setCacheIndexAndFilterBlocks + cache_index_and_filter_blocks=true + # set by jni: options.setLevelCompactionDynamicLevelBytes + level_compaction_dynamic_level_bytes=true \ No newline at end of file diff --git a/conf/default_rocksdb.conf.default b/conf/default_rocksdb.conf.default index 0f3a08779ed..e9b8e7c3ecd 100644 --- a/conf/default_rocksdb.conf.default +++ b/conf/default_rocksdb.conf.default @@ -26,4 +26,8 @@ [CFOptions "default"] # set by jni: options.setLogFileTimeToRoll - log_file_time_to_roll=86400 \ No newline at end of file + log_file_time_to_roll=86400 + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setChecksumType + checksum=kxxHash \ No newline at end of file diff --git a/conf/ledger_metadata_rocksdb.conf.default b/conf/ledger_metadata_rocksdb.conf.default index 0f3a08779ed..e9b8e7c3ecd 100644 --- a/conf/ledger_metadata_rocksdb.conf.default +++ b/conf/ledger_metadata_rocksdb.conf.default @@ -26,4 +26,8 @@ [CFOptions "default"] # set by jni: options.setLogFileTimeToRoll - log_file_time_to_roll=86400 \ No newline at end of file + log_file_time_to_roll=86400 + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setChecksumType + checksum=kxxHash \ No newline at end of file From e2b2d6b8d026a2a495678d0d439e5cf498d13b31 Mon Sep 17 00:00:00 2001 From: chenhang Date: Tue, 21 Feb 2023 16:51:09 +0800 Subject: [PATCH 2/3] format code --- .../bookie/storage/ldb/KeyValueStorageRocksDBTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java index f3d678bfeb1..73ec3006fed 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java @@ -28,7 +28,6 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.List; -import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.conf.ServerConfiguration; import org.junit.Test; import org.rocksdb.BlockBasedTableConfig; @@ -39,7 +38,6 @@ import org.rocksdb.DBOptions; import org.rocksdb.Options; -@Slf4j public class KeyValueStorageRocksDBTest { @Test From 830d57fe07502e935cb64d09ac577b5f9950fc40 Mon Sep 17 00:00:00 2001 From: chenhang Date: Mon, 6 Mar 2023 11:02:22 +0800 Subject: [PATCH 3/3] address comments --- .../bookie/storage/ldb/KeyValueStorageRocksDBTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java index 73ec3006fed..6a0e00b0c10 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java @@ -84,7 +84,6 @@ public void testRocksDBInitiateWithConfigurationFile() throws Exception { @Test public void testReadChecksumTypeFromBookieConfiguration() throws Exception { ServerConfiguration configuration = new ServerConfiguration(); - configuration.setProperty("dbStorage_rocksDB_checksum_type", "kxxHash"); configuration.setEntryLocationRocksdbConf("entry_location_rocksdb.conf"); File tmpDir = Files.createTempDirectory("bk-kv-rocksdbtest-conf").toFile(); Files.createDirectory(Paths.get(tmpDir.toString(), "subDir"));