From 24e45ea57f14cfd814110420b85ba5042151e32e Mon Sep 17 00:00:00 2001 From: githubzilla Date: Tue, 30 Sep 2025 19:13:07 +0800 Subject: [PATCH 1/7] max_open_files could not be -1, otherwise DB will load all files --- .../rocksdb_cloud_data_store.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/eloq_data_store_service/rocksdb_cloud_data_store.cpp b/eloq_data_store_service/rocksdb_cloud_data_store.cpp index 0e4af21..86079df 100644 --- a/eloq_data_store_service/rocksdb_cloud_data_store.cpp +++ b/eloq_data_store_service/rocksdb_cloud_data_store.cpp @@ -603,6 +603,15 @@ bool RocksDBCloudDataStore::OpenCloudDB( cfs_impl->GetStorageProvider()); options.listeners.emplace_back(db_event_listener); + // The max_open_files default value is -1, it cause DB open all files on + // DB::Open() This behavior causes 2 effects, + // 1. DB::Open() will be slow + // 2. During DB::Open, some of the opened sst files keep in LRUCache will be + // deleted due to LRU policy, which causes DB::Open failed + // set max_open_files to 0 will conflict with + // skip_cloud_files_in_getchildren + options.max_open_files = 32; + // set ttl compaction filter assert(ttl_compaction_filter_ == nullptr); ttl_compaction_filter_ = std::make_unique(); @@ -693,6 +702,16 @@ bool RocksDBCloudDataStore::OpenCloudDB( // Enable auto compactions after blocking purger status = db_->SetOptions({{"disable_auto_compactions", "false"}}); + if (!status.ok()) + { + LOG(ERROR) << "Fail to enable auto compactions, error: " + << status.ToString(); + return false; + } + + status = db_->SetOptions( + {{"max_open_files", "-1"}}); // restore max_open_files to default value + if (cloud_config_.warm_up_thread_num_ != 0) { db_->WarmUp(cloud_config_.warm_up_thread_num_); From 194159c78f99f3897f0cd4a1f4a0c39f200d3916 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Thu, 9 Oct 2025 10:41:05 +0800 Subject: [PATCH 2/7] Increasing initial max_open_files --- eloq_data_store_service/rocksdb_cloud_data_store.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eloq_data_store_service/rocksdb_cloud_data_store.cpp b/eloq_data_store_service/rocksdb_cloud_data_store.cpp index 86079df..3637084 100644 --- a/eloq_data_store_service/rocksdb_cloud_data_store.cpp +++ b/eloq_data_store_service/rocksdb_cloud_data_store.cpp @@ -610,7 +610,8 @@ bool RocksDBCloudDataStore::OpenCloudDB( // deleted due to LRU policy, which causes DB::Open failed // set max_open_files to 0 will conflict with // skip_cloud_files_in_getchildren - options.max_open_files = 32; + // Given a smaller value here to avoid opening too many files + options.max_open_files = 128; // set ttl compaction filter assert(ttl_compaction_filter_ == nullptr); From d8f8478826cda4cb2ddc1c5a7ee0f8a6e6c431a5 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Thu, 9 Oct 2025 10:52:47 +0800 Subject: [PATCH 3/7] Fix comments --- .../rocksdb_cloud_data_store.cpp | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/eloq_data_store_service/rocksdb_cloud_data_store.cpp b/eloq_data_store_service/rocksdb_cloud_data_store.cpp index 3637084..8e350c3 100644 --- a/eloq_data_store_service/rocksdb_cloud_data_store.cpp +++ b/eloq_data_store_service/rocksdb_cloud_data_store.cpp @@ -674,7 +674,11 @@ bool RocksDBCloudDataStore::OpenCloudDB( { LOG(ERROR) << "Fail to pause background work, error: " << status.ToString(); - db_->ContinueBackgroundWork(); + // Clean up the partially initialized database + db_->Close(); + delete db_; + db_ = nullptr; + ttl_compaction_filter_ = nullptr; return false; } @@ -685,9 +689,14 @@ bool RocksDBCloudDataStore::OpenCloudDB( { LOG(ERROR) << "Fail to get current epoch from db, error: " << status.ToString(); - db_->ContinueBackgroundWork(); + // Clean up the partially initialized database + db_->Close(); + delete db_; + db_ = nullptr; + ttl_compaction_filter_ = nullptr; return false; } + if (current_epoch.empty()) { LOG(ERROR) << "Current epoch from db is empty"; @@ -707,12 +716,29 @@ bool RocksDBCloudDataStore::OpenCloudDB( { LOG(ERROR) << "Fail to enable auto compactions, error: " << status.ToString(); + // Clean up the partially initialized database + db_->Close(); + delete db_; + db_ = nullptr; + ttl_compaction_filter_ = nullptr; return false; } status = db_->SetOptions( {{"max_open_files", "-1"}}); // restore max_open_files to default value + if (!status.ok()) + { + LOG(ERROR) << "Fail to set max_open_files to -1, error: " + << status.ToString(); + // Clean up the partially initialized database + db_->Close(); + delete db_; + db_ = nullptr; + ttl_compaction_filter_ = nullptr; + return false; + } + if (cloud_config_.warm_up_thread_num_ != 0) { db_->WarmUp(cloud_config_.warm_up_thread_num_); From 5fae0e6140d33bc0411126743656b32684a5c282 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Thu, 9 Oct 2025 14:59:30 +0800 Subject: [PATCH 4/7] Fix SetOptions error --- eloq_data_store_service/rocksdb_cloud_data_store.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eloq_data_store_service/rocksdb_cloud_data_store.cpp b/eloq_data_store_service/rocksdb_cloud_data_store.cpp index 8e350c3..03a60d4 100644 --- a/eloq_data_store_service/rocksdb_cloud_data_store.cpp +++ b/eloq_data_store_service/rocksdb_cloud_data_store.cpp @@ -696,7 +696,6 @@ bool RocksDBCloudDataStore::OpenCloudDB( ttl_compaction_filter_ = nullptr; return false; } - if (current_epoch.empty()) { LOG(ERROR) << "Current epoch from db is empty"; @@ -724,7 +723,7 @@ bool RocksDBCloudDataStore::OpenCloudDB( return false; } - status = db_->SetOptions( + status = db_->SetDBOptions( {{"max_open_files", "-1"}}); // restore max_open_files to default value if (!status.ok()) From b90ffe51397e00b33730889e3424eebaa02c48ae Mon Sep 17 00:00:00 2001 From: githubzilla Date: Sun, 12 Oct 2025 13:01:41 +0800 Subject: [PATCH 5/7] Disable paranoid_check --- eloq_data_store_service/rocksdb_cloud_data_store.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/eloq_data_store_service/rocksdb_cloud_data_store.cpp b/eloq_data_store_service/rocksdb_cloud_data_store.cpp index 03a60d4..411ae00 100644 --- a/eloq_data_store_service/rocksdb_cloud_data_store.cpp +++ b/eloq_data_store_service/rocksdb_cloud_data_store.cpp @@ -445,6 +445,7 @@ bool RocksDBCloudDataStore::OpenCloudDB( options.create_missing_column_families = true; // boost write performance by enabling unordered write options.unordered_write = true; + options.paranoid_checks = false; // print db statistics every 60 seconds if (enable_stats_) From 9daac9a76b497ba342cae95db9647bb1b609a195 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Sun, 12 Oct 2025 13:49:56 +0800 Subject: [PATCH 6/7] Disable skip_cloud_in_get_children --- eloq_data_store_service/rocksdb_cloud_data_store.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eloq_data_store_service/rocksdb_cloud_data_store.cpp b/eloq_data_store_service/rocksdb_cloud_data_store.cpp index 411ae00..36d680b 100644 --- a/eloq_data_store_service/rocksdb_cloud_data_store.cpp +++ b/eloq_data_store_service/rocksdb_cloud_data_store.cpp @@ -331,7 +331,7 @@ bool RocksDBCloudDataStore::StartDB() cfs_options_.constant_sst_file_size_in_sst_file_manager = 64 * 1024 * 1024L; // Skip listing cloud files in GetChildren when DumpDBSummary to speed // up open db - cfs_options_.skip_cloud_files_in_getchildren = true; + // cfs_options_.skip_cloud_files_in_getchildren = true; DLOG(INFO) << "RocksDBCloudDataStore::StartDB, purger_periodicity_millis: " << cfs_options_.purger_periodicity_millis << " ms" @@ -445,7 +445,7 @@ bool RocksDBCloudDataStore::OpenCloudDB( options.create_missing_column_families = true; // boost write performance by enabling unordered write options.unordered_write = true; - options.paranoid_checks = false; + // options.paranoid_checks = false; // print db statistics every 60 seconds if (enable_stats_) From 5db677aeced59207a4d3e974927e076327a21720 Mon Sep 17 00:00:00 2001 From: githubzilla Date: Mon, 13 Oct 2025 13:55:20 +0800 Subject: [PATCH 7/7] Disable paranoid check --- eloq_data_store_service/rocksdb_cloud_data_store.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/eloq_data_store_service/rocksdb_cloud_data_store.cpp b/eloq_data_store_service/rocksdb_cloud_data_store.cpp index 36d680b..adaa3a1 100644 --- a/eloq_data_store_service/rocksdb_cloud_data_store.cpp +++ b/eloq_data_store_service/rocksdb_cloud_data_store.cpp @@ -329,9 +329,10 @@ bool RocksDBCloudDataStore::StartDB() // Temp fix for very slow open db issue // TODO(monkeyzilla): implement customized sst file manager cfs_options_.constant_sst_file_size_in_sst_file_manager = 64 * 1024 * 1024L; - // Skip listing cloud files in GetChildren when DumpDBSummary to speed - // up open db - // cfs_options_.skip_cloud_files_in_getchildren = true; + // Skip listing cloud files in GetChildren when DumpDBSummary, + // SanitizeOptions, Recover(CheckConsistency), WriteOptions to speed up open + // db + cfs_options_.skip_cloud_files_in_getchildren = true; DLOG(INFO) << "RocksDBCloudDataStore::StartDB, purger_periodicity_millis: " << cfs_options_.purger_periodicity_millis << " ms" @@ -445,7 +446,10 @@ bool RocksDBCloudDataStore::OpenCloudDB( options.create_missing_column_families = true; // boost write performance by enabling unordered write options.unordered_write = true; - // options.paranoid_checks = false; + // skip Consistency check, which compares the actual file size with the size + // recorded in the metadata, which can fail when skip_cloud_files_in_getchildren is + // set to true + options.paranoid_checks = false; // print db statistics every 60 seconds if (enable_stats_)