From 3cafd1fc6be19472b0261c8a6c9ea1cee3c7fd35 Mon Sep 17 00:00:00 2001 From: duanlinlin Date: Fri, 19 Jul 2024 10:38:36 +0800 Subject: [PATCH 1/5] fix[rocksdb]: fix error rocksdb default config for CFOptions --- conf/default_rocksdb.conf.default | 5 +++-- conf/ledger_metadata_rocksdb.conf.default | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/conf/default_rocksdb.conf.default b/conf/default_rocksdb.conf.default index e9b8e7c3ecd..df433a33e96 100644 --- a/conf/default_rocksdb.conf.default +++ b/conf/default_rocksdb.conf.default @@ -23,11 +23,12 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 - -[CFOptions "default"] # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 + [CFOptions "default"] + #no default setting in CFOptions + [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setChecksumType checksum=kxxHash \ No newline at end of file diff --git a/conf/ledger_metadata_rocksdb.conf.default b/conf/ledger_metadata_rocksdb.conf.default index e9b8e7c3ecd..f84c0167ed7 100644 --- a/conf/ledger_metadata_rocksdb.conf.default +++ b/conf/ledger_metadata_rocksdb.conf.default @@ -23,10 +23,11 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 - -[CFOptions "default"] # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 + + [CFOptions "default"] + #no default setting in CFOptions [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setChecksumType From 9f1608ee6f49d7a4af32deaa2059964a0b3a1428 Mon Sep 17 00:00:00 2001 From: duanlinlin Date: Fri, 19 Jul 2024 10:57:51 +0800 Subject: [PATCH 2/5] fix[rocksdb]: update rocksdb config for unit test --- .../test/resources/conf/default_rocksdb.conf | 11 ++++++-- .../conf/ledger_metadata_rocksdb.conf | 11 ++++++-- .../test_entry_location_rocksdb.conf | 28 ++++++++++++++++--- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf b/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf index 0f3a08779ed..df433a33e96 100644 --- a/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/conf/default_rocksdb.conf @@ -23,7 +23,12 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 - -[CFOptions "default"] # set by jni: options.setLogFileTimeToRoll - log_file_time_to_roll=86400 \ No newline at end of file + log_file_time_to_roll=86400 + + [CFOptions "default"] + #no default setting in CFOptions + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setChecksumType + checksum=kxxHash \ No newline at end of file diff --git a/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf b/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf index 0f3a08779ed..f84c0167ed7 100644 --- a/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/conf/ledger_metadata_rocksdb.conf @@ -23,7 +23,12 @@ info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum keep_log_file_num=30 - -[CFOptions "default"] # set by jni: options.setLogFileTimeToRoll - log_file_time_to_roll=86400 \ No newline at end of file + log_file_time_to_roll=86400 + + [CFOptions "default"] + #no default setting in CFOptions + +[TableOptions/BlockBasedTable "default"] + # set by jni: tableOptions.setChecksumType + checksum=kxxHash \ No newline at end of file diff --git a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf index 9d1c3e08c47..6f6c1b4d052 100644 --- a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf @@ -19,18 +19,38 @@ [DBOptions] # set by jni: options.setCreateIfMissing create_if_missing=true + # set by jni: options.setInfoLogLevel + info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum - keep_log_file_num=1 + keep_log_file_num=30 + # set by jni: options.setLogFileTimeToRoll + log_file_time_to_roll=86400 + # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism + max_background_jobs=2 + # set by jni: options.setMaxSubcompactions + max_subcompactions=1 # set by jni: options.setMaxTotalWalSize - max_total_wal_size=1000 + max_total_wal_size=536870912 + # set by jni: options.setMaxOpenFiles + max_open_files=-1 + # set by jni: options.setDeleteObsoleteFilesPeriodMicros + delete_obsolete_files_period_micros=3600000000 [CFOptions "default"] # set by jni: options.setCompressionType compression=kLZ4Compression # set by jni: options.setWriteBufferSize - write_buffer_size=1024 + write_buffer_size=67108864 # set by jni: options.setMaxWriteBufferNumber - max_write_buffer_number=1 + max_write_buffer_number=4 + # set by jni: options.setNumLevels + num_levels=7 + # set by jni: options.setLevelZeroFileNumCompactionTrigger + level0_file_num_compaction_trigger=4 + # set by jni: options.setMaxBytesForLevelBase + max_bytes_for_level_base=268435456 + # set by jni: options.setTargetFileSizeBase + target_file_size_base=67108864 # set by jni: options.setLevelCompactionDynamicLevelBytes level_compaction_dynamic_level_bytes=true From b2d4872444e5afdc6bd682be58688d2c5927d9ad Mon Sep 17 00:00:00 2001 From: duanlinlin Date: Fri, 19 Jul 2024 11:05:49 +0800 Subject: [PATCH 3/5] fix[doc]: add comments for rocksdb configuration files --- conf/default_rocksdb.conf.default | 4 ++++ conf/entry_location_rocksdb.conf.default | 4 ++++ conf/ledger_metadata_rocksdb.conf.default | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/conf/default_rocksdb.conf.default b/conf/default_rocksdb.conf.default index df433a33e96..d8a77dd9d3f 100644 --- a/conf/default_rocksdb.conf.default +++ b/conf/default_rocksdb.conf.default @@ -16,6 +16,10 @@ # * limitations under the License. # */ +# When modifying this file, please also modify the configuration files(at +# bookkeeper-server/src/test/resources/conf) in the +# test case to ensure unit test coverage. + [DBOptions] # set by jni: options.setCreateIfMissing create_if_missing=true diff --git a/conf/entry_location_rocksdb.conf.default b/conf/entry_location_rocksdb.conf.default index 6f6c1b4d052..c90124ce93b 100644 --- a/conf/entry_location_rocksdb.conf.default +++ b/conf/entry_location_rocksdb.conf.default @@ -16,6 +16,10 @@ # * limitations under the License. # */ +# When modifying this file, please also modify the configuration files(at +# bookkeeper-server/src/test/resources/conf) in the +# test case to ensure unit test coverage. + [DBOptions] # set by jni: options.setCreateIfMissing create_if_missing=true diff --git a/conf/ledger_metadata_rocksdb.conf.default b/conf/ledger_metadata_rocksdb.conf.default index f84c0167ed7..0bf79349220 100644 --- a/conf/ledger_metadata_rocksdb.conf.default +++ b/conf/ledger_metadata_rocksdb.conf.default @@ -16,6 +16,10 @@ # * limitations under the License. # */ +# When modifying this file, please also modify the configuration files(at +# bookkeeper-server/src/test/resources/conf) in the +# test case to ensure unit test coverage. + [DBOptions] # set by jni: options.setCreateIfMissing create_if_missing=true From 0905f125a937bb2b848051a93f25b81ddf509464 Mon Sep 17 00:00:00 2001 From: duanlinlin Date: Fri, 19 Jul 2024 14:31:10 +0800 Subject: [PATCH 4/5] fix[test}: fix test cases --- .../src/test/resources/test_entry_location_rocksdb.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf index 6f6c1b4d052..cd16a085943 100644 --- a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf @@ -22,7 +22,7 @@ # set by jni: options.setInfoLogLevel info_log_level=INFO_LEVEL # set by jni: options.setKeepLogFileNum - keep_log_file_num=30 + keep_log_file_num=1 # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism @@ -30,7 +30,7 @@ # set by jni: options.setMaxSubcompactions max_subcompactions=1 # set by jni: options.setMaxTotalWalSize - max_total_wal_size=536870912 + max_total_wal_size=1000 # set by jni: options.setMaxOpenFiles max_open_files=-1 # set by jni: options.setDeleteObsoleteFilesPeriodMicros @@ -40,9 +40,9 @@ # set by jni: options.setCompressionType compression=kLZ4Compression # set by jni: options.setWriteBufferSize - write_buffer_size=67108864 + write_buffer_size=1024 # set by jni: options.setMaxWriteBufferNumber - max_write_buffer_number=4 + max_write_buffer_number=1 # set by jni: options.setNumLevels num_levels=7 # set by jni: options.setLevelZeroFileNumCompactionTrigger From 26ee64cb4ed7bb6dc7eec92bd398a734a2247431 Mon Sep 17 00:00:00 2001 From: duanlinlin Date: Fri, 19 Jul 2024 15:38:59 +0800 Subject: [PATCH 5/5] fix rocksdb config --- .../src/test/resources/conf/entry_location_rocksdb.conf | 4 ++-- .../src/test/resources/test_entry_location_rocksdb.conf | 4 ++-- conf/entry_location_rocksdb.conf.default | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf b/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf index 6f6c1b4d052..260111fa140 100644 --- a/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf @@ -26,7 +26,7 @@ # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism - max_background_jobs=2 + max_background_jobs=32 # set by jni: options.setMaxSubcompactions max_subcompactions=1 # set by jni: options.setMaxTotalWalSize @@ -57,7 +57,7 @@ [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setBlockSize block_size=65536 - # set by jni: tableOptions.setBlockCache +# set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10; block_cache=206150041 # set by jni: tableOptions.setFormatVersion format_version=2 diff --git a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf index cd16a085943..f0971b8d310 100644 --- a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf +++ b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf @@ -26,7 +26,7 @@ # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism - max_background_jobs=2 + max_background_jobs=32 # set by jni: options.setMaxSubcompactions max_subcompactions=1 # set by jni: options.setMaxTotalWalSize @@ -57,7 +57,7 @@ [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setBlockSize block_size=65536 - # set by jni: tableOptions.setBlockCache +# set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10; block_cache=206150041 # set by jni: tableOptions.setFormatVersion format_version=2 diff --git a/conf/entry_location_rocksdb.conf.default b/conf/entry_location_rocksdb.conf.default index c90124ce93b..4d5ce58a859 100644 --- a/conf/entry_location_rocksdb.conf.default +++ b/conf/entry_location_rocksdb.conf.default @@ -30,7 +30,7 @@ # set by jni: options.setLogFileTimeToRoll log_file_time_to_roll=86400 # set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism - max_background_jobs=2 + max_background_jobs=32 # set by jni: options.setMaxSubcompactions max_subcompactions=1 # set by jni: options.setMaxTotalWalSize @@ -61,7 +61,7 @@ [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setBlockSize block_size=65536 - # set by jni: tableOptions.setBlockCache + # set by jni: tableOptions.setBlockCache, default value is: maxDirectMemory() / ledgerDirsSize / 10; block_cache=206150041 # set by jni: tableOptions.setFormatVersion format_version=2