Skip to content

Fix slow open rocksdb cloud when db is large #88

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
githubzilla merged 4 commits into from
Sep 19, 2025
Merged

Fix slow open rocksdb cloud when db is large #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bd836cd
open in one second
githubzilla Sep 18, 2025
86a416b
Remove max_open_files setting
githubzilla Sep 19, 2025
ee1a381
Refine log
githubzilla Sep 19, 2025
eb8d41c
fix log
githubzilla Sep 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 15 additions & 11 deletions eloq_data_store_service/rocksdb_cloud_data_store.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,13 @@ bool RocksDBCloudDataStore::StartDB(std::string cookie, std::string prev_cookie)
cfs_options_.purger_periodicity_millis =
cloud_config_.purger_periodicity_millis_;

// Temp fix for very slow open db issue
// TODO(monkeyzilla): implement customized sst file manager
cfs_options_.constant_sst_file_size_in_sst_file_manager = 64 * 1024 * 1024L;
// Skip listing cloud files in GetChildren when DumpDBSummary to speed
// up open db
cfs_options_.skip_cloud_files_in_getchildren = true;

DLOG(INFO) << "RocksDBCloudDataStore::StartDB, purger_periodicity_millis: "
<< cfs_options_.purger_periodicity_millis << " ms"
<< ", run_purger: " << cfs_options_.run_purger;
Expand Down Expand Up @@ -499,13 +506,6 @@ bool RocksDBCloudDataStore::OpenCloudDB(
options.periodic_compaction_seconds = periodic_compaction_seconds_;
options.daily_offpeak_time_utc = dialy_offpeak_time_utc_;

// The max_open_files default value is -1, it cause DB open all files on
// DB::Open() This behavior causes 2 effects,
// 1. DB::Open() will be slow
// 2. During DB::Open, some of the opened sst files keep in LRUCache
// will be deleted due to LRU policy, which causes DB::Open failed
options.max_open_files = 0;

if (target_file_size_base_ > 0)
{
options.target_file_size_base = target_file_size_base_;
Expand Down Expand Up @@ -611,14 +611,18 @@ bool RocksDBCloudDataStore::OpenCloudDB(
return false;
}

// Restore skip_cloud_files_in_getchildren to false
// after DB::Open
rocksdb::CloudFileSystem *cfs =
dynamic_cast<rocksdb::CloudFileSystem *>(cloud_fs_.get());
auto &cfs_options_ref = cfs->GetMutableCloudFileSystemOptions();
cfs_options_ref.skip_cloud_files_in_getchildren = false;

if (cloud_config_.warm_up_thread_num_ != 0)
{
db_->WarmUp(cloud_config_.warm_up_thread_num_);
}

// Reset max_open_files to default value of -1 after DB::Open
db_->SetDBOptions({{"max_open_files", "-1"}});

LOG(INFO) << "RocksDB Cloud started";
return true;
}
Expand Down Expand Up @@ -835,4 +839,4 @@ inline int64_t RocksDBCloudDataStore::FindMaxTermFromCloudManifestFiles(

return max_term;
}
} // namespace EloqDS
} // namespace EloqDS
52 changes: 49 additions & 3 deletions eloq_data_store_service/rocksdb_cloud_dump.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ DEFINE_string(region, "us-east-1", "AWS region");
DEFINE_string(s3_endpoint, "", "Custom S3 endpoint URL (optional)");
DEFINE_string(db_path, "./db", "Local DB path");
DEFINE_bool(list_cf, false, "List all column families");
DEFINE_bool(opendb, false, "Open the DB only");
DEFINE_string(dump_keys, "", "Dump all keys from specified column family");
Comment thread
coderabbitai[bot] marked this conversation as resolved.
DEFINE_string(get_value,
"",
Expand Down Expand Up @@ -78,6 +79,7 @@ void print_usage(const char *prog_name)
" --s3_endpoint=URL Custom S3 endpoint URL (optional)\n"
" --db_path=PATH Local DB path (default: ./db)\n"
" --list_cf List all column families\n"
" --opendb Open the DB only\n"
" --dump_keys=CF Dump all keys from specified column "
"family\n"
" --get_value=CF,KEY Get value for a specific key in "
Expand All @@ -99,6 +101,7 @@ struct CmdLineParams
std::string s3_endpoint_url;
std::string db_path;
bool list_cf;
bool opendb;
std::string dump_keys_cf;
std::pair<std::string, std::string> get_value;
std::string cookie_on_open;
Expand All @@ -118,6 +121,7 @@ CmdLineParams parse_arguments()
params.s3_endpoint_url = FLAGS_s3_endpoint;
params.db_path = FLAGS_db_path;
params.list_cf = FLAGS_list_cf;
params.opendb = FLAGS_opendb;
params.dump_keys_cf = FLAGS_dump_keys;
params.cookie_on_open = FLAGS_cookie_on_open;

Expand Down Expand Up @@ -151,11 +155,11 @@ CmdLineParams parse_arguments()
{
throw std::runtime_error("Cookie on open is required");
}
if (!params.list_cf && params.dump_keys_cf.empty() &&
if (!params.list_cf && !params.opendb && params.dump_keys_cf.empty() &&
params.get_value.first.empty())
{
throw std::runtime_error(
"No action specified. Use --list_cf, --dump_keys, or --get_value");
"No action specified. Use --list_cf, --opendb, --dump_keys, or --get_value");
}

return params;
Expand Down Expand Up @@ -379,6 +383,13 @@ int main(int argc, char **argv)
cfs_options.dest_bucket.SetRegion(params.region);
cfs_options.dest_bucket.SetObjectPath(params.object_path);
cfs_options.cookie_on_open = params.cookie_on_open;
// Temp fix for very slow open db issue
// TODO: implement customized sst file manager
cfs_options.constant_sst_file_size_in_sst_file_manager =
64 * 1024 * 1024L;
// Skip listing cloud files in GetChildren when DumpDBSummary to speed
// up open db
cfs_options.skip_cloud_files_in_getchildren = true;

// Add sst_file_cache for accelerating random access on sst files
cfs_options.sst_file_cache =
Expand Down Expand Up @@ -419,6 +430,9 @@ int main(int argc, char **argv)
options.info_log_level = rocksdb::InfoLogLevel::INFO_LEVEL;
options.max_open_files = -1; // Allow unlimited open files
options.disable_auto_compactions = true; // Disable auto compactions
options.best_efforts_recovery = false;
options.skip_checking_sst_file_sizes_on_db_open = true;
options.skip_stats_update_on_db_open = true;

// Open the DB
rocksdb::DBCloud *db = nullptr;
Expand Down Expand Up @@ -457,6 +471,8 @@ int main(int argc, char **argv)
cf_name, rocksdb::ColumnFamilyOptions()));
}

LOG(INFO) << "Opening database with " << cf_descs.size()
<< " column families...";
// Open the database with all column families
std::vector<rocksdb::ColumnFamilyHandle *> cf_handles;
status = rocksdb::DBCloud::Open(
Expand All @@ -470,6 +486,36 @@ int main(int argc, char **argv)
status.ToString());
}

rocksdb::CloudFileSystem *cloud_fs_ptr =
dynamic_cast<rocksdb::CloudFileSystem *>(cloud_fs.get());
auto &cfs_options_ref =
cloud_fs_ptr->GetMutableCloudFileSystemOptions();
// Restore skip_cloud_files_in_getchildren to false
cfs_options_ref.skip_cloud_files_in_getchildren = false;

if (params.opendb)
{
LOG(INFO) << "Database opened successfully.";
// Clean up resources
for (auto cf_handle : cf_handles)
{
db->DestroyColumnFamilyHandle(cf_handle);
}

db->Close();
DLOG(INFO) << "DB closed";
delete db;
DLOG(INFO) << "DB deleted";
cloud_env = nullptr;
DLOG(INFO) << "cloud_env reset";
cloud_fs = nullptr;
DLOG(INFO) << "cloud_fs reset";
LOG(INFO) << "Database closed successfully.";
Aws::ShutdownAPI(aws_options);
google::ShutdownGoogleLogging();
return 0;
}

// Build a map from column family name to handle
std::unordered_map<std::string, rocksdb::ColumnFamilyHandle *> cf_map;
for (size_t i = 0; i < column_families.size(); i++)
Expand Down Expand Up @@ -592,4 +638,4 @@ int main(int argc, char **argv)
google::ShutdownGoogleLogging();

return 0;
}
}