From 066237cf889819e2a79acc849497cc1275be4658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Calixte?= Date: Thu, 22 Jun 2023 15:10:41 +0200 Subject: [PATCH 1/5] Allow usage of GetApproximateMemoryUsageByType with TransactionDB --- memory_usage.go | 16 +++++++++++-- memory_usage_test.go | 56 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/memory_usage.go b/memory_usage.go index 7b9a6ad6..fda07792 100644 --- a/memory_usage.go +++ b/memory_usage.go @@ -20,16 +20,28 @@ type MemoryUsage struct { CacheTotal uint64 } +type DBForMemoryUsage interface { + getDBforMemoryUsage() *C.rocksdb_t +} + +func (db *DB) getDBforMemoryUsage() *C.rocksdb_t { + return db.c +} + +func (db *TransactionDB) getDBforMemoryUsage() *C.rocksdb_t { + return (*C.rocksdb_t)(db.c) +} + // GetApproximateMemoryUsageByType returns summary // memory usage stats for given databases and caches. -func GetApproximateMemoryUsageByType(dbs []*DB, caches []*Cache) (*MemoryUsage, error) { +func GetApproximateMemoryUsageByType(dbs []DBForMemoryUsage, caches []*Cache) (*MemoryUsage, error) { // register memory consumers consumers := C.rocksdb_memory_consumers_create() defer C.rocksdb_memory_consumers_destroy(consumers) for _, db := range dbs { if db != nil { - C.rocksdb_memory_consumers_add_db(consumers, db.c) + C.rocksdb_memory_consumers_add_db(consumers, (db.getDBforMemoryUsage())) } } for _, cache := range caches { diff --git a/memory_usage_test.go b/memory_usage_test.go index 3f15be92..00f3b035 100644 --- a/memory_usage_test.go +++ b/memory_usage_test.go @@ -25,7 +25,7 @@ func TestMemoryUsage(t *testing.T) { defer db.Close() // take first memory usage snapshot - mu1, err := GetApproximateMemoryUsageByType([]*DB{db}, []*Cache{cache}) + mu1, err := GetApproximateMemoryUsageByType([]DBForMemoryUsage{db}, []*Cache{cache}) ensure.Nil(t, err) // perform IO operations that will affect in-memory tables (and maybe cache as well) @@ -50,7 +50,59 @@ func TestMemoryUsage(t *testing.T) { ensure.Nil(t, err) // take second memory usage snapshot - mu2, err := GetApproximateMemoryUsageByType([]*DB{db}, []*Cache{cache}) + mu2, err := GetApproximateMemoryUsageByType([]DBForMemoryUsage{db}, []*Cache{cache}) + ensure.Nil(t, err) + + // the amount of memory used by memtables should increase after write/read; + // cache memory usage is not likely to be changed, perhaps because requested key is kept by memtable + assert.True(t, mu2.MemTableTotal > mu1.MemTableTotal) + assert.True(t, mu2.MemTableUnflushed > mu1.MemTableUnflushed) + assert.True(t, mu2.CacheTotal >= mu1.CacheTotal) + assert.True(t, mu2.MemTableReadersTotal >= mu1.MemTableReadersTotal) +} + +func TestMemoryUsageTransactionDB(t *testing.T) { + // create database with cache + cache := NewLRUCache(8 * 1024 * 1024) + bbto := NewDefaultBlockBasedTableOptions() + bbto.SetBlockCache(cache) + defer bbto.Destroy() + defer cache.Destroy() + + applyOpts := func(opts *Options, transactionDBOpts *TransactionDBOptions) { + opts.SetBlockBasedTableFactory(bbto) + } + + db := newTestTransactionDB(t, "TestMemoryUsage", applyOpts) + defer db.Close() + + // take first memory usage snapshot + mu1, err := GetApproximateMemoryUsageByType([]DBForMemoryUsage{db}, []*Cache{cache}) + ensure.Nil(t, err) + + // perforx`m IO operations that will affect in-memory tables (and maybe cache as well) + wo := NewDefaultWriteOptions() + defer wo.Destroy() + ro := NewDefaultReadOptions() + defer ro.Destroy() + + key := []byte("key") + value := make([]byte, 1024) + _, err = rand.Read(value) + ensure.Nil(t, err) + + err = db.Put(wo, key, value) + ensure.Nil(t, err) + + // A single Put is not enough to increase approximate memtable usage. + err = db.Put(wo, key, value) + ensure.Nil(t, err) + + _, err = db.Get(ro, key) + ensure.Nil(t, err) + + // take second memory usage snapshot + mu2, err := GetApproximateMemoryUsageByType([]DBForMemoryUsage{db}, []*Cache{cache}) ensure.Nil(t, err) // the amount of memory used by memtables should increase after write/read; From 2753e074ef6973742dc195ea411622e59d34edfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Calixte?= Date: Fri, 23 Jun 2023 15:23:40 +0200 Subject: [PATCH 2/5] avoid changing public function signature --- memory_usage.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/memory_usage.go b/memory_usage.go index fda07792..e2704fda 100644 --- a/memory_usage.go +++ b/memory_usage.go @@ -34,7 +34,17 @@ func (db *TransactionDB) getDBforMemoryUsage() *C.rocksdb_t { // GetApproximateMemoryUsageByType returns summary // memory usage stats for given databases and caches. -func GetApproximateMemoryUsageByType(dbs []DBForMemoryUsage, caches []*Cache) (*MemoryUsage, error) { +func GetApproximateMemoryUsageByType(dbs []*DB, caches []*Cache) (*MemoryUsage, error) { + dbsForMemoryUsage := make([]DBForMemoryUsage, 0, len(dbs)) + for _, db := range dbs { + dbsForMemoryUsage = append(dbsForMemoryUsage, db) + } + return GetApproximateMemoryUsageByTypeGenericDB(dbsForMemoryUsage, caches) +} + +// GetApproximateMemoryUsageByTypeGenericDB returns summary +// memory usage stats for given databases and caches. +func GetApproximateMemoryUsageByTypeGenericDB(dbs []DBForMemoryUsage, caches []*Cache) (*MemoryUsage, error) { // register memory consumers consumers := C.rocksdb_memory_consumers_create() defer C.rocksdb_memory_consumers_destroy(consumers) From 2c028093f197f9e3987486e92f6f4a3ec0446363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Calixte?= Date: Fri, 23 Jun 2023 15:24:39 +0200 Subject: [PATCH 3/5] update unit tests --- memory_usage_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/memory_usage_test.go b/memory_usage_test.go index 00f3b035..c6d05c34 100644 --- a/memory_usage_test.go +++ b/memory_usage_test.go @@ -25,7 +25,7 @@ func TestMemoryUsage(t *testing.T) { defer db.Close() // take first memory usage snapshot - mu1, err := GetApproximateMemoryUsageByType([]DBForMemoryUsage{db}, []*Cache{cache}) + mu1, err := GetApproximateMemoryUsageByType([]*DB{db}, []*Cache{cache}) ensure.Nil(t, err) // perform IO operations that will affect in-memory tables (and maybe cache as well) @@ -50,7 +50,7 @@ func TestMemoryUsage(t *testing.T) { ensure.Nil(t, err) // take second memory usage snapshot - mu2, err := GetApproximateMemoryUsageByType([]DBForMemoryUsage{db}, []*Cache{cache}) + mu2, err := GetApproximateMemoryUsageByType([]*DB{db}, []*Cache{cache}) ensure.Nil(t, err) // the amount of memory used by memtables should increase after write/read; @@ -77,7 +77,7 @@ func TestMemoryUsageTransactionDB(t *testing.T) { defer db.Close() // take first memory usage snapshot - mu1, err := GetApproximateMemoryUsageByType([]DBForMemoryUsage{db}, []*Cache{cache}) + mu1, err := GetApproximateMemoryUsageByTypeGenericDB([]DBForMemoryUsage{db}, []*Cache{cache}) ensure.Nil(t, err) // perforx`m IO operations that will affect in-memory tables (and maybe cache as well) @@ -102,7 +102,7 @@ func TestMemoryUsageTransactionDB(t *testing.T) { ensure.Nil(t, err) // take second memory usage snapshot - mu2, err := GetApproximateMemoryUsageByType([]DBForMemoryUsage{db}, []*Cache{cache}) + mu2, err := GetApproximateMemoryUsageByTypeGenericDB([]DBForMemoryUsage{db}, []*Cache{cache}) ensure.Nil(t, err) // the amount of memory used by memtables should increase after write/read; From a4d4b5bb054e8051f4d315fb0659865af51282d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Calixte?= Date: Fri, 23 Jun 2023 18:31:06 +0200 Subject: [PATCH 4/5] better naming --- memory_usage.go | 18 +++++++++--------- memory_usage_test.go | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/memory_usage.go b/memory_usage.go index e2704fda..b73ed0cd 100644 --- a/memory_usage.go +++ b/memory_usage.go @@ -20,38 +20,38 @@ type MemoryUsage struct { CacheTotal uint64 } -type DBForMemoryUsage interface { - getDBforMemoryUsage() *C.rocksdb_t +type NativeDB interface { + getNativeDB() *C.rocksdb_t } -func (db *DB) getDBforMemoryUsage() *C.rocksdb_t { +func (db *DB) getNativeDB() *C.rocksdb_t { return db.c } -func (db *TransactionDB) getDBforMemoryUsage() *C.rocksdb_t { +func (db *TransactionDB) getNativeDB() *C.rocksdb_t { return (*C.rocksdb_t)(db.c) } // GetApproximateMemoryUsageByType returns summary // memory usage stats for given databases and caches. func GetApproximateMemoryUsageByType(dbs []*DB, caches []*Cache) (*MemoryUsage, error) { - dbsForMemoryUsage := make([]DBForMemoryUsage, 0, len(dbs)) + dbsForMemoryUsage := make([]NativeDB, 0, len(dbs)) for _, db := range dbs { dbsForMemoryUsage = append(dbsForMemoryUsage, db) } - return GetApproximateMemoryUsageByTypeGenericDB(dbsForMemoryUsage, caches) + return GetApproximateMemoryUsageByTypeNativeDB(dbsForMemoryUsage, caches) } -// GetApproximateMemoryUsageByTypeGenericDB returns summary +// GetApproximateMemoryUsageByTypeNativeDB returns summary // memory usage stats for given databases and caches. -func GetApproximateMemoryUsageByTypeGenericDB(dbs []DBForMemoryUsage, caches []*Cache) (*MemoryUsage, error) { +func GetApproximateMemoryUsageByTypeNativeDB(dbs []NativeDB, caches []*Cache) (*MemoryUsage, error) { // register memory consumers consumers := C.rocksdb_memory_consumers_create() defer C.rocksdb_memory_consumers_destroy(consumers) for _, db := range dbs { if db != nil { - C.rocksdb_memory_consumers_add_db(consumers, (db.getDBforMemoryUsage())) + C.rocksdb_memory_consumers_add_db(consumers, (db.getNativeDB())) } } for _, cache := range caches { diff --git a/memory_usage_test.go b/memory_usage_test.go index c6d05c34..fd541d6a 100644 --- a/memory_usage_test.go +++ b/memory_usage_test.go @@ -77,7 +77,7 @@ func TestMemoryUsageTransactionDB(t *testing.T) { defer db.Close() // take first memory usage snapshot - mu1, err := GetApproximateMemoryUsageByTypeGenericDB([]DBForMemoryUsage{db}, []*Cache{cache}) + mu1, err := GetApproximateMemoryUsageByTypeNativeDB([]NativeDB{db}, []*Cache{cache}) ensure.Nil(t, err) // perforx`m IO operations that will affect in-memory tables (and maybe cache as well) @@ -102,7 +102,7 @@ func TestMemoryUsageTransactionDB(t *testing.T) { ensure.Nil(t, err) // take second memory usage snapshot - mu2, err := GetApproximateMemoryUsageByTypeGenericDB([]DBForMemoryUsage{db}, []*Cache{cache}) + mu2, err := GetApproximateMemoryUsageByTypeNativeDB([]NativeDB{db}, []*Cache{cache}) ensure.Nil(t, err) // the amount of memory used by memtables should increase after write/read; From 0ac53e5e575aa7ef4383079f399b5dca5859638d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Calixte?= Date: Mon, 26 Jun 2023 10:14:38 +0200 Subject: [PATCH 5/5] naming --- memory_usage.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/memory_usage.go b/memory_usage.go index b73ed0cd..a2fad4ef 100644 --- a/memory_usage.go +++ b/memory_usage.go @@ -35,11 +35,11 @@ func (db *TransactionDB) getNativeDB() *C.rocksdb_t { // GetApproximateMemoryUsageByType returns summary // memory usage stats for given databases and caches. func GetApproximateMemoryUsageByType(dbs []*DB, caches []*Cache) (*MemoryUsage, error) { - dbsForMemoryUsage := make([]NativeDB, 0, len(dbs)) + nativeDBs := make([]NativeDB, 0, len(dbs)) for _, db := range dbs { - dbsForMemoryUsage = append(dbsForMemoryUsage, db) + nativeDBs = append(nativeDBs, db) } - return GetApproximateMemoryUsageByTypeNativeDB(dbsForMemoryUsage, caches) + return GetApproximateMemoryUsageByTypeNativeDB(nativeDBs, caches) } // GetApproximateMemoryUsageByTypeNativeDB returns summary