From 02e913ed79cf65915bc5d541339460583ff89234 Mon Sep 17 00:00:00 2001 From: bibby Date: Tue, 10 Sep 2019 15:36:32 -0400 Subject: [PATCH 1/8] Adding PartialMergeMulti - the merge operator now supports PartialMergeMulti, which is given a list of operands. - if PartialMergeMulti returns false, it will still call PartialMerge in a loop, so libraries can provide a simple implementation that returns false to preserve existing behaviour --- merge_operator.go | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/merge_operator.go b/merge_operator.go index 33f83948..c811c7c1 100644 --- a/merge_operator.go +++ b/merge_operator.go @@ -43,6 +43,25 @@ type MergeOperator interface { // correct order once a base-value (a Put/Delete/End-of-Database) is seen. PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) + // This function performs merge on multiple operands + // when all of the operands are themselves merge operation types + // that you would have passed to a db.Merge() call in the same order + // (i.e.: db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), + // ... db.Merge(key, operand[n])). + // + // PartialMerge should combine them into a single merge operation. + // The return value should be constructed such that a call to + // db.Merge(key, new_value) would yield the same result as a call + // to db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), + // ... db.Merge(key, operand[n])). + // + // If it is impossible or infeasible to combine the operations, return false. + // The library will attempt to call PartialMerge with each operand if + // PartialMergeMulti returns false. + // The library will internally keep track of the operations, and apply them in the + // correct order once a base-value (a Put/Delete/End-of-Database) is seen. + PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) + // The name of the MergeOperator. Name() string } @@ -62,6 +81,9 @@ func (mo nativeMergeOperator) FullMerge(key, existingValue []byte, operands [][] func (mo nativeMergeOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { return nil, false } +func (mo nativeMergeOperator) PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) { + return nil, false +} func (mo nativeMergeOperator) Name() string { return "" } // Hold references to merge operators. @@ -110,13 +132,17 @@ func gorocksdb_mergeoperator_partial_merge_multi(idx int, cKey *C.char, cKeyLen success := true merger := mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator - leftOperand := operands[0] - for i := 1; i < int(cNumOperands); i++ { - newValue, success = merger.PartialMerge(key, leftOperand, operands[i]) - if !success { - break + // attempt a merge multi operation, otherwise use partial merge + newValue, success = merger.PartialMergeMulti(key, operands) + if !success { + leftOperand := operands[0] + for i := 1; i < int(cNumOperands); i++ { + newValue, success = merger.PartialMerge(key, leftOperand, operands[i]) + if !success { + break + } + leftOperand = newValue } - leftOperand = newValue } newValueLen := len(newValue) From b554c6b6e4e479211dd65ee1c75e14f0775309a0 Mon Sep 17 00:00:00 2001 From: bibby Date: Tue, 10 Sep 2019 16:51:37 -0400 Subject: [PATCH 2/8] fixing the test --- merge_operator_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/merge_operator_test.go b/merge_operator_test.go index fd7e0887..ca63efc6 100644 --- a/merge_operator_test.go +++ b/merge_operator_test.go @@ -41,8 +41,9 @@ func TestMergeOperator(t *testing.T) { } type mockMergeOperator struct { - fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) - partialMerge func(key, leftOperand, rightOperand []byte) ([]byte, bool) + fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) + partialMerge func(key, leftOperand, rightOperand []byte) ([]byte, bool) + partialMergeMulti func(key []byte, operands [][]byte) ([]byte, bool) } func (m *mockMergeOperator) Name() string { return "gorocksdb.test" } @@ -52,3 +53,6 @@ func (m *mockMergeOperator) FullMerge(key, existingValue []byte, operands [][]by func (m *mockMergeOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { return m.partialMerge(key, leftOperand, rightOperand) } +func (m *mockMergeOperator) PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) { + return nil, false +} From 39c1afb2ff0726dd0cdc761007b0062fe4b8ea02 Mon Sep 17 00:00:00 2001 From: bibby Date: Tue, 10 Sep 2019 21:36:41 -0400 Subject: [PATCH 3/8] taking a more opinionated approach and just ditching the left right operator format --- merge_operator.go | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/merge_operator.go b/merge_operator.go index c811c7c1..d28e244a 100644 --- a/merge_operator.go +++ b/merge_operator.go @@ -28,21 +28,6 @@ type MergeOperator interface { // internal corruption. This will be treated as an error by the library. FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) - // This function performs merge(left_op, right_op) - // when both the operands are themselves merge operation types - // that you would have passed to a db.Merge() call in the same order - // (i.e.: db.Merge(key,left_op), followed by db.Merge(key,right_op)). - // - // PartialMerge should combine them into a single merge operation. - // The return value should be constructed such that a call to - // db.Merge(key, new_value) would yield the same result as a call - // to db.Merge(key, left_op) followed by db.Merge(key, right_op). - // - // If it is impossible or infeasible to combine the two operations, return false. - // The library will internally keep track of the operations, and apply them in the - // correct order once a base-value (a Put/Delete/End-of-Database) is seen. - PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) - // This function performs merge on multiple operands // when all of the operands are themselves merge operation types // that you would have passed to a db.Merge() call in the same order @@ -60,7 +45,7 @@ type MergeOperator interface { // PartialMergeMulti returns false. // The library will internally keep track of the operations, and apply them in the // correct order once a base-value (a Put/Delete/End-of-Database) is seen. - PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) + PartialMerge(key []byte, operands [][]byte) ([]byte, bool) // The name of the MergeOperator. Name() string @@ -78,10 +63,7 @@ type nativeMergeOperator struct { func (mo nativeMergeOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { return nil, false } -func (mo nativeMergeOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { - return nil, false -} -func (mo nativeMergeOperator) PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) { +func (mo nativeMergeOperator) PartialMerge(key []byte, operands [][]byte) ([]byte, bool) { return nil, false } func (mo nativeMergeOperator) Name() string { return "" } @@ -131,19 +113,7 @@ func gorocksdb_mergeoperator_partial_merge_multi(idx int, cKey *C.char, cKeyLen var newValue []byte success := true - merger := mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator - // attempt a merge multi operation, otherwise use partial merge - newValue, success = merger.PartialMergeMulti(key, operands) - if !success { - leftOperand := operands[0] - for i := 1; i < int(cNumOperands); i++ { - newValue, success = merger.PartialMerge(key, leftOperand, operands[i]) - if !success { - break - } - leftOperand = newValue - } - } + newValue, success = mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator.PartialMerge(key, operands) newValueLen := len(newValue) *cNewValueLen = C.size_t(newValueLen) From 1384bf7789b26cb79d7c26df41b6c010d7b4ccfc Mon Sep 17 00:00:00 2001 From: bibby Date: Wed, 11 Sep 2019 11:12:17 -0400 Subject: [PATCH 4/8] Update PartialMerge to be PartialMergeMulti - this matches the naming in the C++ libary --- merge_operator.go | 12 +++++------- merge_operator_test.go | 6 +----- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/merge_operator.go b/merge_operator.go index d28e244a..d4c93107 100644 --- a/merge_operator.go +++ b/merge_operator.go @@ -28,24 +28,22 @@ type MergeOperator interface { // internal corruption. This will be treated as an error by the library. FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) - // This function performs merge on multiple operands + // PartialMergeMulti performs merge on multiple operands // when all of the operands are themselves merge operation types // that you would have passed to a db.Merge() call in the same order // (i.e.: db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), // ... db.Merge(key, operand[n])). // - // PartialMerge should combine them into a single merge operation. + // PartialMergeMulti should combine them into a single merge operation. // The return value should be constructed such that a call to // db.Merge(key, new_value) would yield the same result as a call // to db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), // ... db.Merge(key, operand[n])). // // If it is impossible or infeasible to combine the operations, return false. - // The library will attempt to call PartialMerge with each operand if - // PartialMergeMulti returns false. // The library will internally keep track of the operations, and apply them in the // correct order once a base-value (a Put/Delete/End-of-Database) is seen. - PartialMerge(key []byte, operands [][]byte) ([]byte, bool) + PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) // The name of the MergeOperator. Name() string @@ -63,7 +61,7 @@ type nativeMergeOperator struct { func (mo nativeMergeOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { return nil, false } -func (mo nativeMergeOperator) PartialMerge(key []byte, operands [][]byte) ([]byte, bool) { +func (mo nativeMergeOperator) PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) { return nil, false } func (mo nativeMergeOperator) Name() string { return "" } @@ -113,7 +111,7 @@ func gorocksdb_mergeoperator_partial_merge_multi(idx int, cKey *C.char, cKeyLen var newValue []byte success := true - newValue, success = mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator.PartialMerge(key, operands) + newValue, success = mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator.PartialMergeMulti(key, operands) newValueLen := len(newValue) *cNewValueLen = C.size_t(newValueLen) diff --git a/merge_operator_test.go b/merge_operator_test.go index ca63efc6..376b3697 100644 --- a/merge_operator_test.go +++ b/merge_operator_test.go @@ -42,7 +42,6 @@ func TestMergeOperator(t *testing.T) { type mockMergeOperator struct { fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) - partialMerge func(key, leftOperand, rightOperand []byte) ([]byte, bool) partialMergeMulti func(key []byte, operands [][]byte) ([]byte, bool) } @@ -50,9 +49,6 @@ func (m *mockMergeOperator) Name() string { return "gorocksdb.test" } func (m *mockMergeOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { return m.fullMerge(key, existingValue, operands) } -func (m *mockMergeOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { - return m.partialMerge(key, leftOperand, rightOperand) -} func (m *mockMergeOperator) PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) { - return nil, false + return m.partialMergeMulti(key, operands) } From a3b2f44e63542a95a3d0e3499e662df8174f4213 Mon Sep 17 00:00:00 2001 From: bibby Date: Thu, 12 Sep 2019 13:54:54 -0400 Subject: [PATCH 5/8] New MergeMultiOperator interface - this interface adds a PartialMergeMulti function that will be used instead of PartialMerge --- merge_multi_operator.go | 27 +++++++++++++++++++++++++++ merge_operator.go | 34 +++++++++++++++++++++++----------- merge_operator_test.go | 8 ++++---- 3 files changed, 54 insertions(+), 15 deletions(-) create mode 100644 merge_multi_operator.go diff --git a/merge_multi_operator.go b/merge_multi_operator.go new file mode 100644 index 00000000..3ffbeb08 --- /dev/null +++ b/merge_multi_operator.go @@ -0,0 +1,27 @@ +package gorocksdb + +// #include "rocksdb/c.h" +import "C" + +// MergeMultiOperator is similar to a MergeOperator but implements PartialMergeMulti +// When given a MergeMultiOperator, the PartialMergeMulti will always +// be used instead of PartialMerge +type MergeMultiOperator interface { + MergeOperator + // PartialMerge performs merge on multiple operands + // when all of the operands are themselves merge operation types + // that you would have passed to a db.Merge() call in the same order + // (i.e.: db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), + // ... db.Merge(key, operand[n])). + // + // PartialMerge should combine them into a single merge operation. + // The return value should be constructed such that a call to + // db.Merge(key, new_value) would yield the same result as a call + // to db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), + // ... db.Merge(key, operand[n])). + // + // If it is impossible or infeasible to combine the operations, return false. + // The library will internally keep track of the operations, and apply them in the + // correct order once a base-value (a Put/Delete/End-of-Database) is seen. + PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) +} diff --git a/merge_operator.go b/merge_operator.go index d4c93107..8e5294b8 100644 --- a/merge_operator.go +++ b/merge_operator.go @@ -28,22 +28,20 @@ type MergeOperator interface { // internal corruption. This will be treated as an error by the library. FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) - // PartialMergeMulti performs merge on multiple operands - // when all of the operands are themselves merge operation types + // This function performs merge(left_op, right_op) + // when both the operands are themselves merge operation types // that you would have passed to a db.Merge() call in the same order - // (i.e.: db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), - // ... db.Merge(key, operand[n])). + // (i.e.: db.Merge(key,left_op), followed by db.Merge(key,right_op)). // - // PartialMergeMulti should combine them into a single merge operation. + // PartialMerge should combine them into a single merge operation. // The return value should be constructed such that a call to // db.Merge(key, new_value) would yield the same result as a call - // to db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), - // ... db.Merge(key, operand[n])). + // to db.Merge(key, left_op) followed by db.Merge(key, right_op). // - // If it is impossible or infeasible to combine the operations, return false. + // If it is impossible or infeasible to combine the two operations, return false. // The library will internally keep track of the operations, and apply them in the // correct order once a base-value (a Put/Delete/End-of-Database) is seen. - PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) + PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) // The name of the MergeOperator. Name() string @@ -61,7 +59,7 @@ type nativeMergeOperator struct { func (mo nativeMergeOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { return nil, false } -func (mo nativeMergeOperator) PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) { +func (mo nativeMergeOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { return nil, false } func (mo nativeMergeOperator) Name() string { return "" } @@ -111,7 +109,21 @@ func gorocksdb_mergeoperator_partial_merge_multi(idx int, cKey *C.char, cKeyLen var newValue []byte success := true - newValue, success = mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator.PartialMergeMulti(key, operands) + merger := mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator + + // if there is a MergeMultiOperator, use it, otherwise use the MergeOperator + if mmo, ok := merger.(MergeMultiOperator); ok { + newValue, success = mmo.PartialMergeMulti(key, operands) + } else { + leftOperand := operands[0] + for i := 1; i < int(cNumOperands); i++ { + newValue, success = merger.PartialMerge(key, leftOperand, operands[i]) + if !success { + break + } + leftOperand = newValue + } + } newValueLen := len(newValue) *cNewValueLen = C.size_t(newValueLen) diff --git a/merge_operator_test.go b/merge_operator_test.go index 376b3697..fd7e0887 100644 --- a/merge_operator_test.go +++ b/merge_operator_test.go @@ -41,14 +41,14 @@ func TestMergeOperator(t *testing.T) { } type mockMergeOperator struct { - fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) - partialMergeMulti func(key []byte, operands [][]byte) ([]byte, bool) + fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) + partialMerge func(key, leftOperand, rightOperand []byte) ([]byte, bool) } func (m *mockMergeOperator) Name() string { return "gorocksdb.test" } func (m *mockMergeOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { return m.fullMerge(key, existingValue, operands) } -func (m *mockMergeOperator) PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) { - return m.partialMergeMulti(key, operands) +func (m *mockMergeOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { + return m.partialMerge(key, leftOperand, rightOperand) } From 20539af54cfadee38080564e951c08be43ad1b05 Mon Sep 17 00:00:00 2001 From: bibby Date: Thu, 12 Sep 2019 14:48:18 -0400 Subject: [PATCH 6/8] Don't embed the MergeOperator in MergeMultiOperator --- merge_multi_operator.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/merge_multi_operator.go b/merge_multi_operator.go index 3ffbeb08..2e9f0071 100644 --- a/merge_multi_operator.go +++ b/merge_multi_operator.go @@ -3,11 +3,10 @@ package gorocksdb // #include "rocksdb/c.h" import "C" -// MergeMultiOperator is similar to a MergeOperator but implements PartialMergeMulti -// When given a MergeMultiOperator, the PartialMergeMulti will always -// be used instead of PartialMerge +// MergeMultiOperator implements PartialMergeMulti(key []byte, operands [][]byte) ([]byte, err) +// When a MergeOperator implements this interface, PartialMergeMulti +// will be used instead of PartialMerge type MergeMultiOperator interface { - MergeOperator // PartialMerge performs merge on multiple operands // when all of the operands are themselves merge operation types // that you would have passed to a db.Merge() call in the same order From c1b2da75cf354a4168316a91d07c484473fa7cae Mon Sep 17 00:00:00 2001 From: bibby Date: Fri, 13 Sep 2019 12:45:15 -0400 Subject: [PATCH 7/8] Adding a test for merge multi - this test forces both a PartialMergeMulti and a FullMerge to occur --- merge_multi_operator_test.go | 82 ++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 merge_multi_operator_test.go diff --git a/merge_multi_operator_test.go b/merge_multi_operator_test.go new file mode 100644 index 00000000..ec20d9c4 --- /dev/null +++ b/merge_multi_operator_test.go @@ -0,0 +1,82 @@ +package gorocksdb + +import ( + "testing" + + "github.com/facebookgo/ensure" +) + +func TestMergeMultiOperator(t *testing.T) { + var ( + givenKey = []byte("hello") + startingVal = []byte("foo") + mergeVal1 = []byte("bar") + mergeVal2 = []byte("baz") + fMergeResult = []byte("foobarbaz") + pMergeResult = []byte("barbaz") + ) + + merger := &mockMergeMultiOperator{ + fullMerge: func(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + ensure.DeepEqual(&fatalAsError{t}, key, givenKey) + ensure.DeepEqual(&fatalAsError{t}, existingValue, startingVal) + ensure.DeepEqual(&fatalAsError{t}, operands[0], pMergeResult) + return fMergeResult, true + }, + partialMerge: func(key, leftOperand, rightOperand []byte) ([]byte, bool) { + t.FailNow() // this should never be called + return nil, false + }, + partialMergeMulti: func(key []byte, operands [][]byte) ([]byte, bool) { + ensure.DeepEqual(&fatalAsError{t}, key, givenKey) + ensure.DeepEqual(&fatalAsError{t}, operands[0], mergeVal1) + ensure.DeepEqual(&fatalAsError{t}, operands[1], mergeVal2) + return pMergeResult, true + }, + } + db := newTestDB(t, "TestMergeOperator", func(opts *Options) { + opts.SetMergeOperator(merger) + }) + defer db.Close() + + wo := NewDefaultWriteOptions() + defer wo.Destroy() + + // insert a starting value and compact to trigger merges + ensure.Nil(t, db.Put(wo, givenKey, startingVal)) + + // trigger a compaction to ensure that a merge is performed + db.CompactRange(Range{nil, nil}) + + // we expect these two operands to be passed to merge multi + ensure.Nil(t, db.Merge(wo, givenKey, mergeVal1)) + ensure.Nil(t, db.Merge(wo, givenKey, mergeVal2)) + + // trigger a compaction to ensure that a + // partial and full merge are performed + db.CompactRange(Range{nil, nil}) + + ro := NewDefaultReadOptions() + v1, err := db.Get(ro, givenKey) + defer v1.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v1.Data(), fMergeResult) + +} + +type mockMergeMultiOperator struct { + fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) + partialMerge func(key, leftOperand, rightOperand []byte) ([]byte, bool) + partialMergeMulti func(key []byte, operands [][]byte) ([]byte, bool) +} + +func (m *mockMergeMultiOperator) Name() string { return "gorocksdb.test" } +func (m *mockMergeMultiOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + return m.fullMerge(key, existingValue, operands) +} +func (m *mockMergeMultiOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { + return m.partialMerge(key, leftOperand, rightOperand) +} +func (m *mockMergeMultiOperator) PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) { + return m.partialMergeMulti(key, operands) +} From d0110a460a0f2d20242216872e40f34728993f53 Mon Sep 17 00:00:00 2001 From: bibby Date: Fri, 13 Sep 2019 13:25:59 -0400 Subject: [PATCH 8/8] Separate the interfaces - remove PartialMerge from the MergeOperator interface (it really is optional) - add 2 new optional interfaces for a MergeOperator, PartialMerger and MultiMerger. Implementing either of these allows PartialMerges to occur, otherwise FullMerge is used. --- merge_operator.go | 44 ++++++++++--- merge_operator_test.go | 137 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 171 insertions(+), 10 deletions(-) diff --git a/merge_operator.go b/merge_operator.go index 8e5294b8..2de7f9ab 100644 --- a/merge_operator.go +++ b/merge_operator.go @@ -28,6 +28,14 @@ type MergeOperator interface { // internal corruption. This will be treated as an error by the library. FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) + // The name of the MergeOperator. + Name() string +} + +// PartialMerger implements PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, err) +// When a MergeOperator implements this interface, PartialMerge will be called in addition +// to FullMerge for compactions across levels +type PartialMerger interface { // This function performs merge(left_op, right_op) // when both the operands are themselves merge operation types // that you would have passed to a db.Merge() call in the same order @@ -42,9 +50,28 @@ type MergeOperator interface { // The library will internally keep track of the operations, and apply them in the // correct order once a base-value (a Put/Delete/End-of-Database) is seen. PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) +} - // The name of the MergeOperator. - Name() string +// MultiMerger implements PartialMergeMulti(key []byte, operands [][]byte) ([]byte, err) +// When a MergeOperator implements this interface, PartialMergeMulti will be called in addition +// to FullMerge for compactions across levels +type MultiMerger interface { + // PartialMerge performs merge on multiple operands + // when all of the operands are themselves merge operation types + // that you would have passed to a db.Merge() call in the same order + // (i.e.: db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), + // ... db.Merge(key, operand[n])). + // + // PartialMerge should combine them into a single merge operation. + // The return value should be constructed such that a call to + // db.Merge(key, new_value) would yield the same result as a call + // to db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), + // ... db.Merge(key, operand[n])). + // + // If it is impossible or infeasible to combine the operations, return false. + // The library will internally keep track of the operations, and apply them in the + // correct order once a base-value (a Put/Delete/End-of-Database) is seen. + PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) } // NewNativeMergeOperator creates a MergeOperator object. @@ -111,18 +138,21 @@ func gorocksdb_mergeoperator_partial_merge_multi(idx int, cKey *C.char, cKeyLen merger := mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator - // if there is a MergeMultiOperator, use it, otherwise use the MergeOperator - if mmo, ok := merger.(MergeMultiOperator); ok { - newValue, success = mmo.PartialMergeMulti(key, operands) - } else { + // check if this MergeOperator supports partial or multi merges + switch v := merger.(type) { + case MultiMerger: + newValue, success = v.PartialMergeMulti(key, operands) + case PartialMerger: leftOperand := operands[0] for i := 1; i < int(cNumOperands); i++ { - newValue, success = merger.PartialMerge(key, leftOperand, operands[i]) + newValue, success = v.PartialMerge(key, leftOperand, operands[i]) if !success { break } leftOperand = newValue } + default: + success = false } newValueLen := len(newValue) diff --git a/merge_operator_test.go b/merge_operator_test.go index fd7e0887..9dad6f78 100644 --- a/merge_operator_test.go +++ b/merge_operator_test.go @@ -40,15 +40,146 @@ func TestMergeOperator(t *testing.T) { ensure.DeepEqual(t, v1.Data(), givenMerged) } +func TestPartialMergeOperator(t *testing.T) { + var ( + givenKey = []byte("hello") + startingVal = []byte("foo") + mergeVal1 = []byte("bar") + mergeVal2 = []byte("baz") + fMergeResult = []byte("foobarbaz") + pMergeResult = []byte("barbaz") + ) + + merger := &mockMergePartialOperator{ + fullMerge: func(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + ensure.DeepEqual(&fatalAsError{t}, key, givenKey) + ensure.DeepEqual(&fatalAsError{t}, existingValue, startingVal) + ensure.DeepEqual(&fatalAsError{t}, operands[0], pMergeResult) + return fMergeResult, true + }, + partialMerge: func(key, leftOperand, rightOperand []byte) ([]byte, bool) { + ensure.DeepEqual(&fatalAsError{t}, key, givenKey) + ensure.DeepEqual(&fatalAsError{t}, leftOperand, mergeVal1) + ensure.DeepEqual(&fatalAsError{t}, rightOperand, mergeVal2) + return pMergeResult, true + }, + } + db := newTestDB(t, "TestMergeOperator", func(opts *Options) { + opts.SetMergeOperator(merger) + }) + defer db.Close() + + wo := NewDefaultWriteOptions() + defer wo.Destroy() + + // insert a starting value and compact to trigger merges + ensure.Nil(t, db.Put(wo, givenKey, startingVal)) + + // trigger a compaction to ensure that a merge is performed + db.CompactRange(Range{nil, nil}) + + // we expect these two operands to be passed to merge partial + ensure.Nil(t, db.Merge(wo, givenKey, mergeVal1)) + ensure.Nil(t, db.Merge(wo, givenKey, mergeVal2)) + + // trigger a compaction to ensure that a + // partial and full merge are performed + db.CompactRange(Range{nil, nil}) + + ro := NewDefaultReadOptions() + v1, err := db.Get(ro, givenKey) + defer v1.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v1.Data(), fMergeResult) + +} + +func TestMergeMultiOperator(t *testing.T) { + var ( + givenKey = []byte("hello") + startingVal = []byte("foo") + mergeVal1 = []byte("bar") + mergeVal2 = []byte("baz") + fMergeResult = []byte("foobarbaz") + pMergeResult = []byte("barbaz") + ) + + merger := &mockMergeMultiOperator{ + fullMerge: func(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + ensure.DeepEqual(&fatalAsError{t}, key, givenKey) + ensure.DeepEqual(&fatalAsError{t}, existingValue, startingVal) + ensure.DeepEqual(&fatalAsError{t}, operands[0], pMergeResult) + return fMergeResult, true + }, + partialMergeMulti: func(key []byte, operands [][]byte) ([]byte, bool) { + ensure.DeepEqual(&fatalAsError{t}, key, givenKey) + ensure.DeepEqual(&fatalAsError{t}, operands[0], mergeVal1) + ensure.DeepEqual(&fatalAsError{t}, operands[1], mergeVal2) + return pMergeResult, true + }, + } + db := newTestDB(t, "TestMergeOperator", func(opts *Options) { + opts.SetMergeOperator(merger) + }) + defer db.Close() + + wo := NewDefaultWriteOptions() + defer wo.Destroy() + + // insert a starting value and compact to trigger merges + ensure.Nil(t, db.Put(wo, givenKey, startingVal)) + + // trigger a compaction to ensure that a merge is performed + db.CompactRange(Range{nil, nil}) + + // we expect these two operands to be passed to merge multi + ensure.Nil(t, db.Merge(wo, givenKey, mergeVal1)) + ensure.Nil(t, db.Merge(wo, givenKey, mergeVal2)) + + // trigger a compaction to ensure that a + // partial and full merge are performed + db.CompactRange(Range{nil, nil}) + + ro := NewDefaultReadOptions() + v1, err := db.Get(ro, givenKey) + defer v1.Free() + ensure.Nil(t, err) + ensure.DeepEqual(t, v1.Data(), fMergeResult) + +} + +// Mock Objects type mockMergeOperator struct { - fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) - partialMerge func(key, leftOperand, rightOperand []byte) ([]byte, bool) + fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) } func (m *mockMergeOperator) Name() string { return "gorocksdb.test" } func (m *mockMergeOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { return m.fullMerge(key, existingValue, operands) } -func (m *mockMergeOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { + +type mockMergeMultiOperator struct { + fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) + partialMergeMulti func(key []byte, operands [][]byte) ([]byte, bool) +} + +func (m *mockMergeMultiOperator) Name() string { return "gorocksdb.multi" } +func (m *mockMergeMultiOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + return m.fullMerge(key, existingValue, operands) +} +func (m *mockMergeMultiOperator) PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) { + return m.partialMergeMulti(key, operands) +} + +type mockMergePartialOperator struct { + fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) + partialMerge func(key, leftOperand, rightOperand []byte) ([]byte, bool) +} + +func (m *mockMergePartialOperator) Name() string { return "gorocksdb.partial" } +func (m *mockMergePartialOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + return m.fullMerge(key, existingValue, operands) +} +func (m *mockMergePartialOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { return m.partialMerge(key, leftOperand, rightOperand) }