From e95f06004cd60d5dfe238c0c1dfb4332edbf33a5 Mon Sep 17 00:00:00 2001 From: Dylan Forciea Date: Mon, 24 Jun 2024 13:56:29 -0500 Subject: [PATCH 1/8] [REDRES-820] Add rocks options for compaction --- v8/options.go | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/v8/options.go b/v8/options.go index f9a5e54..b682a3e 100644 --- a/v8/options.go +++ b/v8/options.go @@ -1059,6 +1059,64 @@ func (opts *Options) EnableStatistics() { C.rocksdb_options_enable_statistics(opts.c) } +// SetPeriodicCompactionSeconds sets periodic_compaction_seconds option. +// +// This option has different meanings for different compaction styles: +// +// Leveled: files older than `periodic_compaction_seconds` will be picked up +// +// for compaction and will be re-written to the same level as they were +// before. +// +// FIFO: not supported. Setting this option has no effect for FIFO compaction. +// +// Universal: when there are files older than `periodic_compaction_seconds`, +// +// rocksdb will try to do as large a compaction as possible including the +// last level. Such compaction is only skipped if only last level is to +// be compacted and no file in last level is older than +// `periodic_compaction_seconds`. See more in +// UniversalCompactionBuilder::PickPeriodicCompaction(). +// For backward compatibility, the effective value of this option takes +// into account the value of option `ttl`. The logic is as follows: +// - both options are set to 30 days if they have the default value. +// - if both options are zero, zero is picked. Otherwise, we take the min +// value among non-zero options values (i.e. takes the stricter limit). +// +// One main use of the feature is to make sure a file goes through compaction +// filters periodically. Users can also use the feature to clear up SST +// files using old format. +// +// A file's age is computed by looking at file_creation_time or creation_time +// table properties in order, if they have valid non-zero values; if not, the +// age is based on the file's last modified time (given by the underlying +// Env). +// +// This option only supports block based table format for any compaction +// style. +// +// unit: seconds. Ex: 7 days = 7 * 24 * 60 * 60 +// +// Values: +// 0: Turn off Periodic compactions. +// UINT64_MAX - 1 (0xfffffffffffffffe) is special flag to allow RocksDB to +// pick default. +// +// Default: 30 days if using block based table format + compaction filter + +// +// leveled compaction or block based table format + universal compaction. +// 0 (disabled) otherwise. +// +// Dynamically changeable through SetOptions() API +func (opts *Options) SetPeriodicCompactionSeconds(v uint64) { + C.rocksdb_options_set_periodic_compaction_seconds(opts.c, C.uint64_t(v)) +} + +// GetPeriodicCompactionSeconds gets periodic periodic_compaction_seconds option. +func (opts *Options) GetPeriodicCompactionSeconds() uint64 { + return uint64(C.rocksdb_options_get_periodic_compaction_seconds(opts.c)) +} + // PrepareForBulkLoad prepare the DB for bulk loading. // // All data will be in level 0 without any automatic compaction. @@ -1221,6 +1279,20 @@ func (opts *Options) SetAtomicFlush(value bool) { C.rocksdb_options_set_atomic_flush(opts.c, C.uchar(btoi(value))) } +// AddCompactOnDeletionCollectorFactory marks a SST +// file as need-compaction when it observe at least "D" deletion +// entries in any "N" consecutive entries or the ratio of tombstone +// entries in the whole file >= the specified deletion ratio. +func (opts *Options) AddCompactOnDeletionCollectorFactory(windowSize, numDelsTrigger uint) { + C.rocksdb_options_add_compact_on_deletion_collector_factory(opts.c, C.size_t(windowSize), C.size_t(numDelsTrigger)) +} + +// AddCompactOnDeletionCollectorFactoryWithRatio similar to AddCompactOnDeletionCollectorFactory +// with specific deletion ratio. +func (opts *Options) AddCompactOnDeletionCollectorFactoryWithRatio(windowSize, numDelsTrigger uint, deletionRatio float64) { + C.rocksdb_options_add_compact_on_deletion_collector_factory_del_ratio(opts.c, C.size_t(windowSize), C.size_t(numDelsTrigger), C.double(deletionRatio)) +} + // SetMaxSubcompactions represents the maximum number of threads that will // concurrently perform a compaction job by breaking it into multiple, // smaller ones that are run simultaneously. From 767cf311ef4ae891237a239757d35f7cf368574b Mon Sep 17 00:00:00 2001 From: Dylan Forciea Date: Mon, 24 Jun 2024 14:40:31 -0500 Subject: [PATCH 2/8] set period compaction not present in 8.5.4 yet --- v8/options.go | 58 --------------------------------------------------- 1 file changed, 58 deletions(-) diff --git a/v8/options.go b/v8/options.go index b682a3e..fca6b39 100644 --- a/v8/options.go +++ b/v8/options.go @@ -1059,64 +1059,6 @@ func (opts *Options) EnableStatistics() { C.rocksdb_options_enable_statistics(opts.c) } -// SetPeriodicCompactionSeconds sets periodic_compaction_seconds option. -// -// This option has different meanings for different compaction styles: -// -// Leveled: files older than `periodic_compaction_seconds` will be picked up -// -// for compaction and will be re-written to the same level as they were -// before. -// -// FIFO: not supported. Setting this option has no effect for FIFO compaction. -// -// Universal: when there are files older than `periodic_compaction_seconds`, -// -// rocksdb will try to do as large a compaction as possible including the -// last level. Such compaction is only skipped if only last level is to -// be compacted and no file in last level is older than -// `periodic_compaction_seconds`. See more in -// UniversalCompactionBuilder::PickPeriodicCompaction(). -// For backward compatibility, the effective value of this option takes -// into account the value of option `ttl`. The logic is as follows: -// - both options are set to 30 days if they have the default value. -// - if both options are zero, zero is picked. Otherwise, we take the min -// value among non-zero options values (i.e. takes the stricter limit). -// -// One main use of the feature is to make sure a file goes through compaction -// filters periodically. Users can also use the feature to clear up SST -// files using old format. -// -// A file's age is computed by looking at file_creation_time or creation_time -// table properties in order, if they have valid non-zero values; if not, the -// age is based on the file's last modified time (given by the underlying -// Env). -// -// This option only supports block based table format for any compaction -// style. -// -// unit: seconds. Ex: 7 days = 7 * 24 * 60 * 60 -// -// Values: -// 0: Turn off Periodic compactions. -// UINT64_MAX - 1 (0xfffffffffffffffe) is special flag to allow RocksDB to -// pick default. -// -// Default: 30 days if using block based table format + compaction filter + -// -// leveled compaction or block based table format + universal compaction. -// 0 (disabled) otherwise. -// -// Dynamically changeable through SetOptions() API -func (opts *Options) SetPeriodicCompactionSeconds(v uint64) { - C.rocksdb_options_set_periodic_compaction_seconds(opts.c, C.uint64_t(v)) -} - -// GetPeriodicCompactionSeconds gets periodic periodic_compaction_seconds option. -func (opts *Options) GetPeriodicCompactionSeconds() uint64 { - return uint64(C.rocksdb_options_get_periodic_compaction_seconds(opts.c)) -} - // PrepareForBulkLoad prepare the DB for bulk loading. // // All data will be in level 0 without any automatic compaction. From e2bdf7456215dfa943e2316cf32afec3a483a2a8 Mon Sep 17 00:00:00 2001 From: Dylan Forciea Date: Tue, 25 Jun 2024 09:46:38 -0500 Subject: [PATCH 3/8] Add functions for getting base DB --- v8/transactiondb.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/v8/transactiondb.go b/v8/transactiondb.go index 047bdae..eb12f04 100644 --- a/v8/transactiondb.go +++ b/v8/transactiondb.go @@ -128,6 +128,19 @@ func (db *TransactionDB) ReleaseSnapshot(snapshot *Snapshot) { snapshot.c = nil } +// GetBaseDB gets base db. +func (db *TransactionDB) GetBaseDB() *DB { + base := C.rocksdb_transactiondb_get_base_db(db.c) + return &DB{c: base} +} + +// CloseBaseDBOfTransactionDB closes base db of TransactionDB. +func CloseBaseDBOfTransactionDB(db *DB) { + if db != nil && db.c != nil { + C.rocksdb_transactiondb_close_base_db(db.c) + } +} + // TransactionBegin begins a new transaction // with the WriteOptions and TransactionOptions given. func (db *TransactionDB) TransactionBegin( From 7028d122f0324754d6c36b83c77ad4b09533e9ae Mon Sep 17 00:00:00 2001 From: Dylan Forciea Date: Wed, 26 Jun 2024 08:16:39 -0500 Subject: [PATCH 4/8] Add SetOptionsCF call --- v8/db.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/v8/db.go b/v8/db.go index ae3e298..00a1984 100755 --- a/v8/db.go +++ b/v8/db.go @@ -883,6 +883,41 @@ func (db *DB) SetOptions(keys, values []string) error { return nil } +// SetOptionsCF dynamically changes options through the SetOptions API for specific Column Family. +func (db *DB) SetOptionsCF(cf *ColumnFamilyHandle, keys, values []string) (err error) { + numKeys := len(keys) + if numKeys == 0 { + return nil + } + + cKeys := make([]*C.char, numKeys) + cValues := make([]*C.char, numKeys) + for i := range keys { + cKeys[i] = C.CString(keys[i]) + cValues[i] = C.CString(values[i]) + } + + var cErr *C.char + + C.rocksdb_set_options_cf( + db.c, + cf.c, + C.int(numKeys), + &cKeys[0], + &cValues[0], + &cErr, + ) + err = fromCError(cErr) + + // free before return + for i := range cKeys { + C.free(unsafe.Pointer(cKeys[i])) + C.free(unsafe.Pointer(cValues[i])) + } + + return +} + // LiveFileMetadata is a metadata which is associated with each SST file. type LiveFileMetadata struct { Name string From b70dda79ffb07f738d70ec960ca4c2598f4c2d02 Mon Sep 17 00:00:00 2001 From: Dylan Forciea Date: Wed, 26 Jun 2024 09:15:31 -0500 Subject: [PATCH 5/8] Add missing func --- v8/util.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/v8/util.go b/v8/util.go index fc138cd..fbacd51 100644 --- a/v8/util.go +++ b/v8/util.go @@ -64,3 +64,12 @@ func charSlice(data **C.char, len C.int) []*C.char { func sizeSlice(data *C.size_t, len C.int) []C.size_t { return unsafe.Slice(data, int(len)) } + +// fromCError returns go error and free c_err if need. +func fromCError(cErr *C.char) (err error) { + if cErr != nil { + err = errors.New(C.GoString(cErr)) + C.rocksdb_free(unsafe.Pointer(cErr)) + } + return +} From a9df07494ee4f1149e56564b51f5ced12d62cda9 Mon Sep 17 00:00:00 2001 From: Dylan Forciea Date: Wed, 26 Jun 2024 09:20:13 -0500 Subject: [PATCH 6/8] Add missing include --- v8/util.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v8/util.go b/v8/util.go index fbacd51..d2500e3 100644 --- a/v8/util.go +++ b/v8/util.go @@ -1,6 +1,7 @@ package gorocksdb // #include +// #include "rocksdb/c.h" import "C" import "unsafe" From 7571dabe4a235596417c19be162b6096ed6db79a Mon Sep 17 00:00:00 2001 From: Dylan Forciea Date: Wed, 26 Jun 2024 09:26:19 -0500 Subject: [PATCH 7/8] More imports --- v8/util.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/v8/util.go b/v8/util.go index d2500e3..3d801ce 100644 --- a/v8/util.go +++ b/v8/util.go @@ -2,9 +2,12 @@ package gorocksdb // #include // #include "rocksdb/c.h" - import "C" -import "unsafe" + +import ( + "errors" + "unsafe" +) // btoi converts a bool value to int. func btoi(b bool) int { From 3b1c13b5de415f11383afbc1f5403430073b650b Mon Sep 17 00:00:00 2001 From: Dylan Forciea Date: Wed, 26 Jun 2024 10:11:39 -0500 Subject: [PATCH 8/8] Fix up error handling --- v8/db.go | 7 +++++-- v8/util.go | 17 ++--------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/v8/db.go b/v8/db.go index 00a1984..3ce274f 100755 --- a/v8/db.go +++ b/v8/db.go @@ -734,7 +734,7 @@ func (db *DB) CreateColumnFamilyWithTTL(opts *Options, name string, ttl int) (*C var ( cErr *C.char cName = C.CString(name) - cTtl = C.int(ttl) + cTtl = C.int(ttl) ) defer C.free(unsafe.Pointer(cName)) cHandle := C.rocksdb_create_column_family_with_ttl(db.c, opts.c, cName, cTtl, &cErr) @@ -907,7 +907,10 @@ func (db *DB) SetOptionsCF(cf *ColumnFamilyHandle, keys, values []string) (err e &cValues[0], &cErr, ) - err = fromCError(cErr) + if cErr != nil { + err = errors.New(C.GoString(cErr)) + C.rocksdb_free(unsafe.Pointer(cErr)) + } // free before return for i := range cKeys { diff --git a/v8/util.go b/v8/util.go index 3d801ce..fc138cd 100644 --- a/v8/util.go +++ b/v8/util.go @@ -1,13 +1,9 @@ package gorocksdb // #include -// #include "rocksdb/c.h" -import "C" -import ( - "errors" - "unsafe" -) +import "C" +import "unsafe" // btoi converts a bool value to int. func btoi(b bool) int { @@ -68,12 +64,3 @@ func charSlice(data **C.char, len C.int) []*C.char { func sizeSlice(data *C.size_t, len C.int) []C.size_t { return unsafe.Slice(data, int(len)) } - -// fromCError returns go error and free c_err if need. -func fromCError(cErr *C.char) (err error) { - if cErr != nil { - err = errors.New(C.GoString(cErr)) - C.rocksdb_free(unsafe.Pointer(cErr)) - } - return -}