From ef5ef2bfd4e34e69cb0693043504bf7a538ce936 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 6 Jan 2023 03:26:05 +0700 Subject: [PATCH 01/10] fix: fix for merk caching in contract caching (irony) --- Cargo.lock | 30 ++--- packages/rs-drive/Cargo.toml | 6 +- packages/rs-drive/src/drive/contract/mod.rs | 127 +++++++++++++++++- .../src/drive/fee_pools/epochs/proposers.rs | 7 +- .../src/drive/fee_pools/epochs/start_block.rs | 7 +- packages/rs-drive/src/drive/fee_pools/mod.rs | 1 + .../drive/fee_pools/pending_epoch_updates.rs | 3 + .../rs-drive/src/drive/grove_operations.rs | 7 +- packages/rs-drive/src/drive/identity/mod.rs | 7 +- .../src/drive/identity/withdrawal_queue.rs | 7 +- packages/rs-drive/src/drive/mod.rs | 13 ++ 11 files changed, 188 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0200f978946..43a8637a7cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -574,7 +574,7 @@ dependencies = [ [[package]] name = "costs" version = "1.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=89df137eb8b50f315d820c252ba6b7be61af46b4#89df137eb8b50f315d820c252ba6b7be61af46b4" +source = "git+https://github.com/dashpay/grovedb?rev=af53d3729d19ad5bd1912970fbb599b264b9046b#af53d3729d19ad5bd1912970fbb599b264b9046b" dependencies = [ "integer-encoding", "intmap", @@ -784,7 +784,7 @@ dependencies = [ [[package]] name = "dashcore" version = "0.29.1" -source = "git+https://github.com/dashevo/rust-dashcore?branch=master#0cf3939e25a66b3cfdb5ba354fb4437d6b0d9d03" +source = "git+https://github.com/dashevo/rust-dashcore?branch=master#51548a4a1b9eca7430f5f3caf94d9784886ff2e9" dependencies = [ "anyhow", "bech32", @@ -800,7 +800,7 @@ dependencies = [ [[package]] name = "dashcore" version = "0.29.1" -source = "git+https://github.com/dashpay/rust-dashcore?branch=master#0cf3939e25a66b3cfdb5ba354fb4437d6b0d9d03" +source = "git+https://github.com/dashpay/rust-dashcore?branch=master#51548a4a1b9eca7430f5f3caf94d9784886ff2e9" dependencies = [ "anyhow", "bech32", @@ -1253,8 +1253,8 @@ dependencies = [ [[package]] name = "grovedb" -version = "0.9.0" -source = "git+https://github.com/dashpay/grovedb?rev=89df137eb8b50f315d820c252ba6b7be61af46b4#89df137eb8b50f315d820c252ba6b7be61af46b4" +version = "0.9.2" +source = "git+https://github.com/dashpay/grovedb?rev=af53d3729d19ad5bd1912970fbb599b264b9046b#af53d3729d19ad5bd1912970fbb599b264b9046b" dependencies = [ "bincode", "costs", @@ -1687,8 +1687,8 @@ dependencies = [ [[package]] name = "merk" -version = "0.9.0" -source = "git+https://github.com/dashpay/grovedb?rev=89df137eb8b50f315d820c252ba6b7be61af46b4#89df137eb8b50f315d820c252ba6b7be61af46b4" +version = "0.9.2" +source = "git+https://github.com/dashpay/grovedb?rev=af53d3729d19ad5bd1912970fbb599b264b9046b#af53d3729d19ad5bd1912970fbb599b264b9046b" dependencies = [ "blake3", "byteorder", @@ -1833,9 +1833,9 @@ checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" [[package]] name = "nom" -version = "7.1.1" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c" dependencies = [ "memchr", "minimal-lexical", @@ -2010,9 +2010,9 @@ dependencies = [ [[package]] name = "object" -version = "0.30.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239da7f290cfa979f43f85a8efeee9a8a76d0827c356d37f9d3d7254d6b537fb" +checksum = "8d864c91689fdc196779b98dba0aceac6118594c2df6ee5d943eb6a8df4d107a" dependencies = [ "memchr", ] @@ -2729,7 +2729,7 @@ dependencies = [ [[package]] name = "storage" version = "1.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=89df137eb8b50f315d820c252ba6b7be61af46b4#89df137eb8b50f315d820c252ba6b7be61af46b4" +source = "git+https://github.com/dashpay/grovedb?rev=af53d3729d19ad5bd1912970fbb599b264b9046b#af53d3729d19ad5bd1912970fbb599b264b9046b" dependencies = [ "blake3", "costs", @@ -2962,9 +2962,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "38a54aca0c15d014013256222ba0ebed095673f89345dd79119d912eb561b7a8" dependencies = [ "autocfg", "bytes", @@ -3116,7 +3116,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "visualize" version = "0.1.0" -source = "git+https://github.com/dashpay/grovedb?rev=89df137eb8b50f315d820c252ba6b7be61af46b4#89df137eb8b50f315d820c252ba6b7be61af46b4" +source = "git+https://github.com/dashpay/grovedb?rev=af53d3729d19ad5bd1912970fbb599b264b9046b#af53d3729d19ad5bd1912970fbb599b264b9046b" dependencies = [ "hex", "itertools", diff --git a/packages/rs-drive/Cargo.toml b/packages/rs-drive/Cargo.toml index dc09f6ed05c..a88519ff5ad 100644 --- a/packages/rs-drive/Cargo.toml +++ b/packages/rs-drive/Cargo.toml @@ -36,15 +36,15 @@ rust_decimal_macros = "1.25.0" [dependencies.grovedb] git = "https://github.com/dashpay/grovedb" -rev = "89df137eb8b50f315d820c252ba6b7be61af46b4" +rev = "af53d3729d19ad5bd1912970fbb599b264b9046b" [dependencies.storage] git = "https://github.com/dashpay/grovedb" -rev = "89df137eb8b50f315d820c252ba6b7be61af46b4" +rev = "af53d3729d19ad5bd1912970fbb599b264b9046b" [dependencies.costs] git = "https://github.com/dashpay/grovedb" -rev = "89df137eb8b50f315d820c252ba6b7be61af46b4" +rev = "af53d3729d19ad5bd1912970fbb599b264b9046b" [dev-dependencies] criterion = "0.3.5" diff --git a/packages/rs-drive/src/drive/contract/mod.rs b/packages/rs-drive/src/drive/contract/mod.rs index 4692209574b..e3a34b7d86b 100644 --- a/packages/rs-drive/src/drive/contract/mod.rs +++ b/packages/rs-drive/src/drive/contract/mod.rs @@ -851,9 +851,15 @@ impl Drive { epoch: Option<&Epoch>, transaction: TransactionArg, ) -> CostResult>, Error> { - let CostContext { value, cost } = - self.grove - .get(contract_root_path(&contract_id), &[0], transaction); + // As we want deterministic costs, we want the cost to always be the same for + // fetching the contract. + // We need to pass allow cache to false + let CostContext { value, cost } = self.grove.get_caching_optional( + contract_root_path(&contract_id), + &[0], + false, + transaction, + ); match value { Ok(Element::Item(stored_contract_bytes, element_flag)) => { @@ -1304,6 +1310,121 @@ mod tests { .expect("expected to apply contract successfully"); } + #[test] + fn test_contract_cache_returns_same_cost_after_reload() { + let tmp_dir = TempDir::new().unwrap(); + let drive: Drive = Drive::open(tmp_dir, None).expect("expected to open Drive successfully"); + + drive + .create_initial_state_structure(None) + .expect("expected to create root tree successfully"); + + let contract_path = "tests/supporting_files/contract/references/references.json"; + + // let's construct the grovedb structure for the dashpay data contract + let contract_cbor = json_document_to_cbor(contract_path, Some(1)); + let ref_contract = ::from_cbor(&contract_cbor, None) + .expect("expected to deserialize the contract"); + + let transaction = drive.grove.start_transaction(); + + // Create a contract first + drive + .apply_contract( + &ref_contract, + contract_cbor.clone(), + BlockInfo::default(), + true, + StorageFlags::optional_default_as_ref(), + Some(&transaction), + ) + .expect("expected to apply contract successfully"); + + let contract_path = "tests/supporting_files/contract/deepNested/deep-nested50.json"; + // let's construct the grovedb structure for the dashpay data contract + let contract_cbor = json_document_to_cbor(contract_path, Some(1)); + let deep_contract = ::from_cbor(&contract_cbor, None) + .expect("expected to deserialize the contract"); + drive + .apply_contract( + &deep_contract, + contract_cbor.clone(), + BlockInfo::default(), + true, + StorageFlags::optional_default_as_ref(), + Some(&transaction), + ) + .expect("expected to apply contract successfully"); + + let deep_contract_fetch_info = drive + .get_contract_with_fetch_info( + deep_contract.id().to_buffer(), + Some(&Epoch::new(0)), + Some(&transaction), + ) + .expect("got contract") + .1 + .expect("got contract fetch info"); + let ref_contract_fetch_info = drive + .get_contract_with_fetch_info( + ref_contract.id().to_buffer(), + Some(&Epoch::new(0)), + Some(&transaction), + ) + .expect("got contract") + .1 + .expect("got contract fetch info"); + + transaction.commit().expect("expected to commit"); + + let deep_contract_fetch_info_after_prune = drive + .get_contract_with_fetch_info( + deep_contract.id().to_buffer(), + Some(&Epoch::new(0)), + None, + ) + .expect("got contract") + .1 + .expect("got contract fetch info"); + let ref_contract_fetch_info_after_prune = drive + .get_contract_with_fetch_info(ref_contract.id().to_buffer(), Some(&Epoch::new(0)), None) + .expect("got contract") + .1 + .expect("got contract fetch info"); + + assert_eq!( + deep_contract_fetch_info, + deep_contract_fetch_info_after_prune + ); + assert_eq!(ref_contract_fetch_info, ref_contract_fetch_info_after_prune); + + drive.drop_cache(); + + let deep_contract_fetch_info_after_cache_reload = drive + .get_contract_with_fetch_info( + deep_contract.id().to_buffer(), + Some(&Epoch::new(0)), + None, + ) + .expect("got contract") + .1 + .expect("got contract fetch info"); + let ref_contract_fetch_info_after_cache_reload = drive + .get_contract_with_fetch_info(ref_contract.id().to_buffer(), Some(&Epoch::new(0)), None) + .expect("got contract") + .1 + .expect("got contract fetch info"); + + assert_eq!( + deep_contract_fetch_info, + deep_contract_fetch_info_after_cache_reload + ); + assert_eq!( + ref_contract_fetch_info, + ref_contract_fetch_info_after_cache_reload + ); + } + mod get_contract_with_fetch_info_and_add_to_operations { use super::*; diff --git a/packages/rs-drive/src/drive/fee_pools/epochs/proposers.rs b/packages/rs-drive/src/drive/fee_pools/epochs/proposers.rs index 0e0002ef604..dd549a4bab5 100644 --- a/packages/rs-drive/src/drive/fee_pools/epochs/proposers.rs +++ b/packages/rs-drive/src/drive/fee_pools/epochs/proposers.rs @@ -113,7 +113,12 @@ impl Drive { let key_elements = self .grove - .query_raw(&path_query, QueryKeyElementPairResultType, transaction) + .query_raw( + &path_query, + transaction.is_some(), + QueryKeyElementPairResultType, + transaction, + ) .unwrap() .map_err(Error::GroveDB)? .0 diff --git a/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs b/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs index 7aa5d0d532e..626baa4dc46 100644 --- a/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs +++ b/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs @@ -103,7 +103,12 @@ impl Drive { let (result_items, _) = self .grove - .query_raw(&path_query, QueryPathKeyElementTrioResultType, transaction) + .query_raw( + &path_query, + transaction.is_some(), + QueryPathKeyElementTrioResultType, + transaction, + ) .unwrap() .map_err(Error::GroveDB)?; diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index 3064d20ee1b..a5fabeca0b7 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -123,6 +123,7 @@ impl Drive { .grove .query_raw( &PathQuery::new_unsized(pools_vec_path(), epochs_query), + transaction.is_some(), QueryResultType::QueryElementResultType, transaction, ) diff --git a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs index f4efa29b423..630ece00ab5 100644 --- a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs +++ b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs @@ -62,6 +62,7 @@ impl Drive { .grove .query_raw( &PathQuery::new_unsized(pools_pending_updates_path(), query), + transaction.is_some(), QueryResultType::QueryKeyElementPairResultType, transaction, ) @@ -114,6 +115,7 @@ impl Drive { .grove .query_raw( &PathQuery::new_unsized(pools_pending_updates_path(), query), + transaction.is_some(), QueryResultType::QueryKeyElementPairResultType, transaction, ) @@ -163,6 +165,7 @@ impl Drive { .grove .query_raw( &PathQuery::new_unsized(pools_pending_updates_path(), query), + transaction.is_some(), QueryResultType::QueryKeyElementPairResultType, transaction, ) diff --git a/packages/rs-drive/src/drive/grove_operations.rs b/packages/rs-drive/src/drive/grove_operations.rs index 97ead714db1..8ebe4ba5e57 100644 --- a/packages/rs-drive/src/drive/grove_operations.rs +++ b/packages/rs-drive/src/drive/grove_operations.rs @@ -381,7 +381,9 @@ impl Drive { transaction: TransactionArg, drive_operations: &mut Vec, ) -> Result<(Vec>, u16), Error> { - let CostContext { value, cost } = self.grove.query_item_value(path_query, transaction); + let CostContext { value, cost } = + self.grove + .query_item_value(path_query, transaction.is_some(), transaction); drive_operations.push(CalculatedCostOperation(cost)); value.map_err(Error::GroveDB) } @@ -396,7 +398,8 @@ impl Drive { drive_operations: &mut Vec, ) -> Result<(QueryResultElements, u16), Error> { let CostContext { value, cost } = - self.grove.query_raw(path_query, result_type, transaction); + self.grove + .query_raw(path_query, transaction.is_some(), result_type, transaction); drive_operations.push(CalculatedCostOperation(cost)); value.map_err(Error::GroveDB) } diff --git a/packages/rs-drive/src/drive/identity/mod.rs b/packages/rs-drive/src/drive/identity/mod.rs index 31e38e1baa5..f1e04b1728d 100644 --- a/packages/rs-drive/src/drive/identity/mod.rs +++ b/packages/rs-drive/src/drive/identity/mod.rs @@ -196,7 +196,12 @@ impl Drive { }; let (result_items, _) = self .grove - .query_raw(&path_query, QueryElementResultType, transaction) + .query_raw( + &path_query, + transaction.is_some(), + QueryElementResultType, + transaction, + ) .unwrap() .map_err(Error::GroveDB)?; diff --git a/packages/rs-drive/src/drive/identity/withdrawal_queue.rs b/packages/rs-drive/src/drive/identity/withdrawal_queue.rs index 779e0ec95eb..e1e0f57f151 100644 --- a/packages/rs-drive/src/drive/identity/withdrawal_queue.rs +++ b/packages/rs-drive/src/drive/identity/withdrawal_queue.rs @@ -159,7 +159,12 @@ impl Drive { let result_items = self .grove - .query_raw(&path_query, QueryKeyElementPairResultType, transaction) + .query_raw( + &path_query, + transaction.is_some(), + QueryKeyElementPairResultType, + transaction, + ) .unwrap() .map_err(Error::GroveDB)? .0 diff --git a/packages/rs-drive/src/drive/mod.rs b/packages/rs-drive/src/drive/mod.rs index 6c5b9fc316b..f8cf9a76fb6 100644 --- a/packages/rs-drive/src/drive/mod.rs +++ b/packages/rs-drive/src/drive/mod.rs @@ -172,6 +172,19 @@ impl Drive { } } + pub fn drop_cache(&self) { + let genesis_time_ms = self.config.default_genesis_time; + let data_contracts_global_cache_size = self.config.data_contracts_global_cache_size; + let data_contracts_block_cache_size = self.config.data_contracts_block_cache_size; + self.cache.replace(DriveCache { + cached_contracts: DataContractCache::new( + data_contracts_global_cache_size, + data_contracts_block_cache_size, + ), + genesis_time_ms, + }); + } + /// Commits a transaction. pub fn commit_transaction(&self, transaction: Transaction) -> Result<(), Error> { self.grove From 8bd4e916b84c9eb1cdd8fc437c49b7df1519bb90 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 6 Jan 2023 03:51:14 +0700 Subject: [PATCH 02/10] more fixes --- packages/js-drive/lib/storage/GroveDBStore.js | 5 +---- packages/rs-drive-nodejs/GroveDB.js | 5 +++-- packages/rs-drive-nodejs/src/lib.rs | 9 ++++++--- packages/rs-drive/src/drive/mod.rs | 1 + 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/js-drive/lib/storage/GroveDBStore.js b/packages/js-drive/lib/storage/GroveDBStore.js index d29eb371bd7..6a58f8f3560 100644 --- a/packages/js-drive/lib/storage/GroveDBStore.js +++ b/packages/js-drive/lib/storage/GroveDBStore.js @@ -238,10 +238,7 @@ class GroveDBStore { let items; try { - [items] = await this.db.query( - query, - options.useTransaction || false, - ); + [items] = await this.db.query(query, options.useTransaction || false); } catch (e) { if ( e.message.startsWith('grovedb: path key not found') diff --git a/packages/rs-drive-nodejs/GroveDB.js b/packages/rs-drive-nodejs/GroveDB.js index 6b5bb2edfbe..51d10024622 100644 --- a/packages/rs-drive-nodejs/GroveDB.js +++ b/packages/rs-drive-nodejs/GroveDB.js @@ -194,10 +194,11 @@ class GroveDB { * * @param {PathQuery} query * @param {boolean} [useTransaction=false] + * @param {boolean} [allowCache=false] * @return {Promise<*>} */ - async query(query, useTransaction = false) { - return groveDbQueryAsync.call(this.db, query, useTransaction); + async query(query, useTransaction = false, allowCache = true) { + return groveDbQueryAsync.call(this.db, query, allowCache, useTransaction); } /** diff --git a/packages/rs-drive-nodejs/src/lib.rs b/packages/rs-drive-nodejs/src/lib.rs index 937bb160d36..6594c6c028f 100644 --- a/packages/rs-drive-nodejs/src/lib.rs +++ b/packages/rs-drive-nodejs/src/lib.rs @@ -1576,12 +1576,15 @@ impl PlatformWrapper { fn js_grove_db_query(mut cx: FunctionContext) -> JsResult { let js_path_query = cx.argument::(0)?; - let js_using_transaction = cx.argument::(1)?; + let js_allow_cache = cx.argument::(1)?; + let js_using_transaction = cx.argument::(2)?; - let js_callback = cx.argument::(2)?.root(&mut cx); + let js_callback = cx.argument::(3)?.root(&mut cx); let path_query = converter::js_path_query_to_path_query(js_path_query, &mut cx)?; + let allow_cache = js_allow_cache.value(&mut cx); + let using_transaction = js_using_transaction.value(&mut cx); let db = cx @@ -1603,7 +1606,7 @@ impl PlatformWrapper { let result = transaction_result.and_then(|transaction_arg| { grove_db - .query_item_value(&path_query, transaction_arg) + .query_item_value(&path_query, allow_cache, transaction_arg) .unwrap() .map_err(Error::GroveDB) .map_err(|err| err.to_string()) diff --git a/packages/rs-drive/src/drive/mod.rs b/packages/rs-drive/src/drive/mod.rs index f8cf9a76fb6..4d2423ef532 100644 --- a/packages/rs-drive/src/drive/mod.rs +++ b/packages/rs-drive/src/drive/mod.rs @@ -172,6 +172,7 @@ impl Drive { } } + /// Drops the drive cache pub fn drop_cache(&self) { let genesis_time_ms = self.config.default_genesis_time; let data_contracts_global_cache_size = self.config.data_contracts_global_cache_size; From 81abd16a56d8026a5da5ed3b09be26fc8d0e94d6 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 6 Jan 2023 03:51:39 +0700 Subject: [PATCH 03/10] small fmt --- .../validation/basic/find_duplicates_by_id.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rs-dpp/src/document/state_transition/documents_batch_transition/validation/basic/find_duplicates_by_id.rs b/packages/rs-dpp/src/document/state_transition/documents_batch_transition/validation/basic/find_duplicates_by_id.rs index da190ef2082..4d65f2f61eb 100644 --- a/packages/rs-dpp/src/document/state_transition/documents_batch_transition/validation/basic/find_duplicates_by_id.rs +++ b/packages/rs-dpp/src/document/state_transition/documents_batch_transition/validation/basic/find_duplicates_by_id.rs @@ -41,6 +41,7 @@ fn create_fingerprint(document_transition: &JsonValue) -> Option { #[cfg(test)] mod test { + use crate::document::document_transition::{DocumentTransitionObjectLike, JsonValue}; use crate::{ document::document_transition::{ DocumentCreateTransition, DocumentDeleteTransition, DocumentReplaceTransition, @@ -48,7 +49,6 @@ mod test { }, tests::utils::generate_random_identifier_struct, }; - use crate::document::document_transition::{DocumentTransitionObjectLike, JsonValue}; use super::find_duplicates_by_id; From 475b7a119670036a3b03a876c44d3e8f58ac05b2 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 6 Jan 2023 17:55:54 +0800 Subject: [PATCH 04/10] test: fix test and small refactoring (#711) --- packages/js-drive/lib/storage/GroveDBStore.js | 7 +- packages/rs-drive-nodejs/GroveDB.js | 6 +- packages/rs-drive-nodejs/src/lib.rs | 6 +- packages/rs-drive-nodejs/test/GroveDB.spec.js | 45 +-- packages/rs-drive/src/drive/contract/mod.rs | 281 +++++++++--------- 5 files changed, 168 insertions(+), 177 deletions(-) diff --git a/packages/js-drive/lib/storage/GroveDBStore.js b/packages/js-drive/lib/storage/GroveDBStore.js index 6a58f8f3560..35a1eb474d9 100644 --- a/packages/js-drive/lib/storage/GroveDBStore.js +++ b/packages/js-drive/lib/storage/GroveDBStore.js @@ -231,6 +231,7 @@ class GroveDBStore { * * @param {PathQuery} query * @param {Object} [options] + * @param {boolean} [options.skipCache=false] * @param {boolean} [options.useTransaction=false] * @return {Promise>} */ @@ -238,7 +239,11 @@ class GroveDBStore { let items; try { - [items] = await this.db.query(query, options.useTransaction || false); + [items] = await this.db.query( + query, + options.skipCache || false, + options.useTransaction || false, + ); } catch (e) { if ( e.message.startsWith('grovedb: path key not found') diff --git a/packages/rs-drive-nodejs/GroveDB.js b/packages/rs-drive-nodejs/GroveDB.js index 51d10024622..100ab9963ff 100644 --- a/packages/rs-drive-nodejs/GroveDB.js +++ b/packages/rs-drive-nodejs/GroveDB.js @@ -193,12 +193,12 @@ class GroveDB { * Get data using query. * * @param {PathQuery} query + * @param {boolean} [skipCache=false] * @param {boolean} [useTransaction=false] - * @param {boolean} [allowCache=false] * @return {Promise<*>} */ - async query(query, useTransaction = false, allowCache = true) { - return groveDbQueryAsync.call(this.db, query, allowCache, useTransaction); + async query(query, skipCache = false, useTransaction = false) { + return groveDbQueryAsync.call(this.db, query, skipCache, useTransaction); } /** diff --git a/packages/rs-drive-nodejs/src/lib.rs b/packages/rs-drive-nodejs/src/lib.rs index 6594c6c028f..a52377c838c 100644 --- a/packages/rs-drive-nodejs/src/lib.rs +++ b/packages/rs-drive-nodejs/src/lib.rs @@ -1576,14 +1576,14 @@ impl PlatformWrapper { fn js_grove_db_query(mut cx: FunctionContext) -> JsResult { let js_path_query = cx.argument::(0)?; - let js_allow_cache = cx.argument::(1)?; + let js_skip_cache = cx.argument::(1)?; let js_using_transaction = cx.argument::(2)?; let js_callback = cx.argument::(3)?.root(&mut cx); let path_query = converter::js_path_query_to_path_query(js_path_query, &mut cx)?; - let allow_cache = js_allow_cache.value(&mut cx); + let skip_cache = js_skip_cache.value(&mut cx); let using_transaction = js_using_transaction.value(&mut cx); @@ -1606,7 +1606,7 @@ impl PlatformWrapper { let result = transaction_result.and_then(|transaction_arg| { grove_db - .query_item_value(&path_query, allow_cache, transaction_arg) + .query_item_value(&path_query, !skip_cache, transaction_arg) .unwrap() .map_err(Error::GroveDB) .map_err(|err| err.to_string()) diff --git a/packages/rs-drive-nodejs/test/GroveDB.spec.js b/packages/rs-drive-nodejs/test/GroveDB.spec.js index 9e1bcfb2df3..85726052775 100644 --- a/packages/rs-drive-nodejs/test/GroveDB.spec.js +++ b/packages/rs-drive-nodejs/test/GroveDB.spec.js @@ -434,7 +434,7 @@ describe('GroveDB', () => { }); }); - describe('#query for Item subtrees', () => { + describe('#query', () => { let aValue; let aKey; let bValue; @@ -836,37 +836,21 @@ describe('GroveDB', () => { expect(skipped).to.equals(0); }); - }); - describe('#query', () => { - let dPath; - let dKey; - let ePath; - - let daValue; - let dbValue; - let dcValue; - let eaValue; - let eaKey; - let ebValue; - - beforeEach(async () => { - await groveDb.insert( - rootTreePath, - treeKey, - { type: 'tree', epoch: 0 }, - ); + it('should be able to retrieve data with subquery', async () => { + // Prepare tree structure + const dKey = Buffer.from('dKey'); + const daValue = Buffer.from('da'); + const dbValue = Buffer.from('db'); + const dcValue = Buffer.from('dc'); + const eaValue = Buffer.from('ea'); + const eaKey = Buffer.from('eaKey'); + const ebValue = Buffer.from('eb'); - dKey = Buffer.from('dKey'); - daValue = Buffer.from('da'); - dbValue = Buffer.from('db'); - dcValue = Buffer.from('dc'); - eaValue = Buffer.from('ea'); - eaKey = Buffer.from('eaKey'); - ebValue = Buffer.from('eb'); + const dPath = [...itemTreePath]; - dPath = [...itemTreePath]; dPath.push(dKey); + await groveDb.insert( itemTreePath, dKey, @@ -892,7 +876,8 @@ describe('GroveDB', () => { ); const eKey = Buffer.from('eKey'); - ePath = [...itemTreePath]; + + const ePath = [...itemTreePath]; ePath.push(eKey); await groveDb.insert( itemTreePath, @@ -911,9 +896,7 @@ describe('GroveDB', () => { Buffer.from('ebKey'), { type: 'item', epoch: 0, value: ebValue }, ); - }); - it('should be able to retrieve data with subquery', async () => { // This should give us only last subtree and apply subquery to it const query = { path: itemTreePath, diff --git a/packages/rs-drive/src/drive/contract/mod.rs b/packages/rs-drive/src/drive/contract/mod.rs index e3a34b7d86b..8d0fe7bd237 100644 --- a/packages/rs-drive/src/drive/contract/mod.rs +++ b/packages/rs-drive/src/drive/contract/mod.rs @@ -1045,6 +1045,8 @@ mod tests { }; use crate::drive::Drive; + use crate::common::helpers::setup::setup_drive_with_initial_state_structure; + fn setup_deep_nested_contract() -> (Drive, Contract, Vec) { // Todo: make TempDir based on _prefix let tmp_dir = TempDir::new().unwrap(); @@ -1310,126 +1312,12 @@ mod tests { .expect("expected to apply contract successfully"); } - #[test] - fn test_contract_cache_returns_same_cost_after_reload() { - let tmp_dir = TempDir::new().unwrap(); - let drive: Drive = Drive::open(tmp_dir, None).expect("expected to open Drive successfully"); - - drive - .create_initial_state_structure(None) - .expect("expected to create root tree successfully"); - - let contract_path = "tests/supporting_files/contract/references/references.json"; - - // let's construct the grovedb structure for the dashpay data contract - let contract_cbor = json_document_to_cbor(contract_path, Some(1)); - let ref_contract = ::from_cbor(&contract_cbor, None) - .expect("expected to deserialize the contract"); - - let transaction = drive.grove.start_transaction(); - - // Create a contract first - drive - .apply_contract( - &ref_contract, - contract_cbor.clone(), - BlockInfo::default(), - true, - StorageFlags::optional_default_as_ref(), - Some(&transaction), - ) - .expect("expected to apply contract successfully"); - - let contract_path = "tests/supporting_files/contract/deepNested/deep-nested50.json"; - // let's construct the grovedb structure for the dashpay data contract - let contract_cbor = json_document_to_cbor(contract_path, Some(1)); - let deep_contract = ::from_cbor(&contract_cbor, None) - .expect("expected to deserialize the contract"); - drive - .apply_contract( - &deep_contract, - contract_cbor.clone(), - BlockInfo::default(), - true, - StorageFlags::optional_default_as_ref(), - Some(&transaction), - ) - .expect("expected to apply contract successfully"); - - let deep_contract_fetch_info = drive - .get_contract_with_fetch_info( - deep_contract.id().to_buffer(), - Some(&Epoch::new(0)), - Some(&transaction), - ) - .expect("got contract") - .1 - .expect("got contract fetch info"); - let ref_contract_fetch_info = drive - .get_contract_with_fetch_info( - ref_contract.id().to_buffer(), - Some(&Epoch::new(0)), - Some(&transaction), - ) - .expect("got contract") - .1 - .expect("got contract fetch info"); - - transaction.commit().expect("expected to commit"); - - let deep_contract_fetch_info_after_prune = drive - .get_contract_with_fetch_info( - deep_contract.id().to_buffer(), - Some(&Epoch::new(0)), - None, - ) - .expect("got contract") - .1 - .expect("got contract fetch info"); - let ref_contract_fetch_info_after_prune = drive - .get_contract_with_fetch_info(ref_contract.id().to_buffer(), Some(&Epoch::new(0)), None) - .expect("got contract") - .1 - .expect("got contract fetch info"); - - assert_eq!( - deep_contract_fetch_info, - deep_contract_fetch_info_after_prune - ); - assert_eq!(ref_contract_fetch_info, ref_contract_fetch_info_after_prune); - - drive.drop_cache(); - - let deep_contract_fetch_info_after_cache_reload = drive - .get_contract_with_fetch_info( - deep_contract.id().to_buffer(), - Some(&Epoch::new(0)), - None, - ) - .expect("got contract") - .1 - .expect("got contract fetch info"); - let ref_contract_fetch_info_after_cache_reload = drive - .get_contract_with_fetch_info(ref_contract.id().to_buffer(), Some(&Epoch::new(0)), None) - .expect("got contract") - .1 - .expect("got contract fetch info"); - - assert_eq!( - deep_contract_fetch_info, - deep_contract_fetch_info_after_cache_reload - ); - assert_eq!( - ref_contract_fetch_info, - ref_contract_fetch_info_after_cache_reload - ); - } - - mod get_contract_with_fetch_info_and_add_to_operations { + mod get_contract_with_fetch_info { use super::*; + use dpp::prelude::Identifier; #[test] - fn test_get_contract_from_global_and_transactional_cache() { + fn should_get_contract_from_global_and_block_cache() { let (drive, mut contract, _) = setup_reference_contract(); let transaction = drive.grove.start_transaction(); @@ -1466,41 +1354,156 @@ mod tests { } #[test] - fn test_get_non_existent_contract() { - let tmp_dir = TempDir::new().unwrap(); - let drive: Drive = - Drive::open(tmp_dir, None).expect("expected to open Drive successfully"); + fn should_return_none_if_contract_not_exist() { + let drive = setup_drive_with_initial_state_structure(); - drive - .create_initial_state_structure(None) - .expect("expected to create state structure"); - let contract_id = rand::thread_rng().gen::<[u8; 32]>(); + let result = drive + .get_contract_with_fetch_info([0; 32], None, None) + .expect("should get contract"); + + assert!(result.0.is_none()); + assert!(result.1.is_none()); + } + + #[test] + fn should_return_fees_for_non_existing_contract_if_epoch_is_passed() { + let drive = setup_drive_with_initial_state_structure(); let result = drive - .get_contract_with_fetch_info(contract_id, None, None) + .get_contract_with_fetch_info([0; 32], Some(&Epoch::new(0)), None) .expect("should get contract"); + assert!(matches!( + result.0, + Some(FeeResult { + processing_fee: 6000, + .. + }) + )); + assert!(result.1.is_none()); } #[test] - fn test_get_non_existent_contract_has_fees() { - let tmp_dir = TempDir::new().unwrap(); - let drive: Drive = - Drive::open(tmp_dir, None).expect("expected to open Drive successfully"); + fn should_return_then_same_costs_for_cached_and_non_cached_contracts_in_merk() { + // Merk trees have own cache and depends on does contract node cached or not + // we get could get different costs. To avoid of it we fetch contracts without tree caching + + let (drive, mut ref_contract, ref_contract_cbor) = setup_reference_contract(); + let transaction = drive.grove.start_transaction(); + + // Create more contracts to trigger re-balancing + for i in 0..150u8 { + ref_contract.id = Identifier::from([i; 32]); + + drive + .apply_contract( + &ref_contract, + ref_contract_cbor.clone(), + BlockInfo::default(), + true, + StorageFlags::optional_default_as_ref(), + Some(&transaction), + ) + .expect("expected to apply contract successfully"); + } + + // Create a deep placed contract + let contract_path = "tests/supporting_files/contract/deepNested/deep-nested50.json"; + let contract_cbor = json_document_to_cbor(contract_path, Some(1)); + let deep_contract = ::from_cbor(&contract_cbor, None) + .expect("expected to deserialize the contract"); drive - .create_initial_state_structure(None) - .expect("expected to create state structure"); - let contract_id = rand::thread_rng().gen::<[u8; 32]>(); + .apply_contract( + &deep_contract, + contract_cbor.clone(), + BlockInfo::default(), + true, + StorageFlags::optional_default_as_ref(), + Some(&transaction), + ) + .expect("expected to apply contract successfully"); - let result = drive - .get_contract_with_fetch_info(contract_id, Some(&Epoch::new(0)), None) - .expect("should get contract"); + // Get fetch info to compare + let ref_contract_fetch_info = drive + .get_contract_with_fetch_info( + Identifier::from([0; 32]).to_buffer(), + Some(&Epoch::new(0)), + Some(&transaction), + ) + .expect("got contract") + .1 + .expect("got contract fetch info"); - let fees = result.0; - assert!(fees.is_some()); - assert_eq!(fees.unwrap().processing_fee, 6000) + let deep_contract_fetch_info = drive + .get_contract_with_fetch_info( + deep_contract.id().to_buffer(), + Some(&Epoch::new(0)), + Some(&transaction), + ) + .expect("got contract") + .1 + .expect("got contract fetch info"); + + transaction.commit().expect("expected to commit"); + + let deep_contract_fetch_info_after_prune = drive + .get_contract_with_fetch_info( + deep_contract.id().to_buffer(), + Some(&Epoch::new(0)), + None, + ) + .expect("got contract") + .1 + .expect("got contract fetch info"); + let ref_contract_fetch_info_after_prune = drive + .get_contract_with_fetch_info( + ref_contract.id().to_buffer(), + Some(&Epoch::new(0)), + None, + ) + .expect("got contract") + .1 + .expect("got contract fetch info"); + + assert_eq!( + deep_contract_fetch_info, + deep_contract_fetch_info_after_prune + ); + + assert_eq!(ref_contract_fetch_info, ref_contract_fetch_info_after_prune); + + drive.drop_cache(); + + let deep_contract_fetch_info_after_cache_reload = drive + .get_contract_with_fetch_info( + deep_contract.id().to_buffer(), + Some(&Epoch::new(0)), + None, + ) + .expect("got contract") + .1 + .expect("got contract fetch info"); + + let ref_contract_fetch_info_after_cache_reload = drive + .get_contract_with_fetch_info( + ref_contract.id().to_buffer(), + Some(&Epoch::new(0)), + None, + ) + .expect("got contract") + .1 + .expect("got contract fetch info"); + + assert_eq!( + deep_contract_fetch_info, + deep_contract_fetch_info_after_cache_reload + ); + assert_eq!( + ref_contract_fetch_info, + ref_contract_fetch_info_after_cache_reload + ); } } } From 43e9046fc055e54407934534c3cf2d376acf06cd Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 6 Jan 2023 22:36:47 +0800 Subject: [PATCH 05/10] test: additional test to prove the fix --- packages/rs-drive/src/drive/contract/mod.rs | 141 ++++++++++++++++---- 1 file changed, 117 insertions(+), 24 deletions(-) diff --git a/packages/rs-drive/src/drive/contract/mod.rs b/packages/rs-drive/src/drive/contract/mod.rs index 8d0fe7bd237..6f58040d8a3 100644 --- a/packages/rs-drive/src/drive/contract/mod.rs +++ b/packages/rs-drive/src/drive/contract/mod.rs @@ -1315,6 +1315,8 @@ mod tests { mod get_contract_with_fetch_info { use super::*; use dpp::prelude::Identifier; + use serde::Serialize; + use std::borrow::Borrow; #[test] fn should_get_contract_from_global_and_block_cache() { @@ -1385,11 +1387,17 @@ mod tests { } #[test] - fn should_return_then_same_costs_for_cached_and_non_cached_contracts_in_merk() { + fn should_always_have_then_same_cost() { // Merk trees have own cache and depends on does contract node cached or not // we get could get different costs. To avoid of it we fetch contracts without tree caching - let (drive, mut ref_contract, ref_contract_cbor) = setup_reference_contract(); + let (drive, mut ref_contract, _) = setup_reference_contract(); + + /* + * Firstly, we create multiple contracts during block processing (in transaction) + */ + + let ref_contract_id_buffer = Identifier::from([0; 32]).to_buffer(); let transaction = drive.grove.start_transaction(); @@ -1397,10 +1405,12 @@ mod tests { for i in 0..150u8 { ref_contract.id = Identifier::from([i; 32]); + let ref_contract_cbor = ref_contract.to_cbor().expect("should serialize contract"); + drive .apply_contract( &ref_contract, - ref_contract_cbor.clone(), + ref_contract_cbor, BlockInfo::default(), true, StorageFlags::optional_default_as_ref(), @@ -1417,7 +1427,7 @@ mod tests { drive .apply_contract( &deep_contract, - contract_cbor.clone(), + contract_cbor, BlockInfo::default(), true, StorageFlags::optional_default_as_ref(), @@ -1425,10 +1435,9 @@ mod tests { ) .expect("expected to apply contract successfully"); - // Get fetch info to compare - let ref_contract_fetch_info = drive + let ref_contract_fetch_info_transactional = drive .get_contract_with_fetch_info( - Identifier::from([0; 32]).to_buffer(), + ref_contract_id_buffer, Some(&Epoch::new(0)), Some(&transaction), ) @@ -1436,7 +1445,7 @@ mod tests { .1 .expect("got contract fetch info"); - let deep_contract_fetch_info = drive + let deep_contract_fetch_info_transactional = drive .get_contract_with_fetch_info( deep_contract.id().to_buffer(), Some(&Epoch::new(0)), @@ -1446,9 +1455,22 @@ mod tests { .1 .expect("got contract fetch info"); + /* + * Then we commit the block + */ + + // Commit transaction and merge block (transactional) cache to global cache transaction.commit().expect("expected to commit"); - let deep_contract_fetch_info_after_prune = drive + let mut drive_cache = drive.cache.borrow_mut(); + drive_cache.cached_contracts.merge_block_cache(); + drop(drive_cache); + + /* + * Contracts fetched with user query and during block execution must have equal costs + */ + + let deep_contract_fetch_info = drive .get_contract_with_fetch_info( deep_contract.id().to_buffer(), Some(&Epoch::new(0)), @@ -1457,9 +1479,39 @@ mod tests { .expect("got contract") .1 .expect("got contract fetch info"); - let ref_contract_fetch_info_after_prune = drive + + let ref_contract_fetch_info = drive + .get_contract_with_fetch_info(ref_contract_id_buffer, Some(&Epoch::new(0)), None) + .expect("got contract") + .1 + .expect("got contract fetch info"); + + assert_eq!( + deep_contract_fetch_info_transactional, + deep_contract_fetch_info + ); + + assert_eq!( + ref_contract_fetch_info_transactional, + ref_contract_fetch_info + ); + + /* + * User restarts the node + */ + + // Drop cache so contract will be fetched once again + drive.drop_cache(); + + /* + * Other nodes weren't restarted so contracts queried by user after restart + * must have the same costs as transactional contracts and contracts before + * restart + */ + + let deep_contract_fetch_info_without_cache = drive .get_contract_with_fetch_info( - ref_contract.id().to_buffer(), + deep_contract.id().to_buffer(), Some(&Epoch::new(0)), None, ) @@ -1467,42 +1519,83 @@ mod tests { .1 .expect("got contract fetch info"); + let ref_contract_fetch_info_without_cache = drive + .get_contract_with_fetch_info(ref_contract_id_buffer, Some(&Epoch::new(0)), None) + .expect("got contract") + .1 + .expect("got contract fetch info"); + assert_eq!( - deep_contract_fetch_info, - deep_contract_fetch_info_after_prune + deep_contract_fetch_info_transactional, + deep_contract_fetch_info_without_cache + ); + assert_eq!( + ref_contract_fetch_info_transactional, + ref_contract_fetch_info_without_cache ); - assert_eq!(ref_contract_fetch_info, ref_contract_fetch_info_after_prune); - + /* + * Let's imagine that many blocks were executed and the node is restarted again + */ drive.drop_cache(); - let deep_contract_fetch_info_after_cache_reload = drive + /* + * Drive executes a new block + */ + + let transaction = drive.grove.start_transaction(); + + // Create more contracts to trigger re-balancing + for i in 150..200u8 { + ref_contract.id = Identifier::from([i; 32]); + + let ref_contract_cbor = ref_contract.to_cbor().expect("should serialize contract"); + + drive + .apply_contract( + &ref_contract, + ref_contract_cbor, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_ref(), + None, + ) + .expect("expected to apply contract successfully"); + } + + /* + * Other nodes weren't restarted so contracts fetched during block execution + * should have the same cost as previously fetched contracts + */ + + let deep_contract_fetch_info_transactional2 = drive .get_contract_with_fetch_info( deep_contract.id().to_buffer(), Some(&Epoch::new(0)), - None, + Some(&transaction), ) .expect("got contract") .1 .expect("got contract fetch info"); - let ref_contract_fetch_info_after_cache_reload = drive + let ref_contract_fetch_info_transactional2 = drive .get_contract_with_fetch_info( - ref_contract.id().to_buffer(), + ref_contract_id_buffer, Some(&Epoch::new(0)), - None, + Some(&transaction), ) .expect("got contract") .1 .expect("got contract fetch info"); assert_eq!( - deep_contract_fetch_info, - deep_contract_fetch_info_after_cache_reload + ref_contract_fetch_info_transactional, + ref_contract_fetch_info_transactional2, ); + assert_eq!( - ref_contract_fetch_info, - ref_contract_fetch_info_after_cache_reload + deep_contract_fetch_info_transactional, + deep_contract_fetch_info_transactional2 ); } } From 9e0bbb7d74f2daed97ef5387e5ef14e45b7abe3b Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 6 Jan 2023 22:43:15 +0800 Subject: [PATCH 06/10] style: remove unused imports --- packages/rs-drive/src/drive/contract/mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/rs-drive/src/drive/contract/mod.rs b/packages/rs-drive/src/drive/contract/mod.rs index 6f58040d8a3..02480d46f8c 100644 --- a/packages/rs-drive/src/drive/contract/mod.rs +++ b/packages/rs-drive/src/drive/contract/mod.rs @@ -1315,8 +1315,6 @@ mod tests { mod get_contract_with_fetch_info { use super::*; use dpp::prelude::Identifier; - use serde::Serialize; - use std::borrow::Borrow; #[test] fn should_get_contract_from_global_and_block_cache() { From e5aabc27c7f0930628aa24ff57e5a01e14cf3442 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sat, 7 Jan 2023 10:44:51 +0800 Subject: [PATCH 07/10] test: additional test to prove the fix (#714) --- packages/rs-drive/src/drive/contract/mod.rs | 139 ++++++++++++++++---- 1 file changed, 115 insertions(+), 24 deletions(-) diff --git a/packages/rs-drive/src/drive/contract/mod.rs b/packages/rs-drive/src/drive/contract/mod.rs index 8d0fe7bd237..02480d46f8c 100644 --- a/packages/rs-drive/src/drive/contract/mod.rs +++ b/packages/rs-drive/src/drive/contract/mod.rs @@ -1385,11 +1385,17 @@ mod tests { } #[test] - fn should_return_then_same_costs_for_cached_and_non_cached_contracts_in_merk() { + fn should_always_have_then_same_cost() { // Merk trees have own cache and depends on does contract node cached or not // we get could get different costs. To avoid of it we fetch contracts without tree caching - let (drive, mut ref_contract, ref_contract_cbor) = setup_reference_contract(); + let (drive, mut ref_contract, _) = setup_reference_contract(); + + /* + * Firstly, we create multiple contracts during block processing (in transaction) + */ + + let ref_contract_id_buffer = Identifier::from([0; 32]).to_buffer(); let transaction = drive.grove.start_transaction(); @@ -1397,10 +1403,12 @@ mod tests { for i in 0..150u8 { ref_contract.id = Identifier::from([i; 32]); + let ref_contract_cbor = ref_contract.to_cbor().expect("should serialize contract"); + drive .apply_contract( &ref_contract, - ref_contract_cbor.clone(), + ref_contract_cbor, BlockInfo::default(), true, StorageFlags::optional_default_as_ref(), @@ -1417,7 +1425,7 @@ mod tests { drive .apply_contract( &deep_contract, - contract_cbor.clone(), + contract_cbor, BlockInfo::default(), true, StorageFlags::optional_default_as_ref(), @@ -1425,10 +1433,9 @@ mod tests { ) .expect("expected to apply contract successfully"); - // Get fetch info to compare - let ref_contract_fetch_info = drive + let ref_contract_fetch_info_transactional = drive .get_contract_with_fetch_info( - Identifier::from([0; 32]).to_buffer(), + ref_contract_id_buffer, Some(&Epoch::new(0)), Some(&transaction), ) @@ -1436,7 +1443,7 @@ mod tests { .1 .expect("got contract fetch info"); - let deep_contract_fetch_info = drive + let deep_contract_fetch_info_transactional = drive .get_contract_with_fetch_info( deep_contract.id().to_buffer(), Some(&Epoch::new(0)), @@ -1446,9 +1453,22 @@ mod tests { .1 .expect("got contract fetch info"); + /* + * Then we commit the block + */ + + // Commit transaction and merge block (transactional) cache to global cache transaction.commit().expect("expected to commit"); - let deep_contract_fetch_info_after_prune = drive + let mut drive_cache = drive.cache.borrow_mut(); + drive_cache.cached_contracts.merge_block_cache(); + drop(drive_cache); + + /* + * Contracts fetched with user query and during block execution must have equal costs + */ + + let deep_contract_fetch_info = drive .get_contract_with_fetch_info( deep_contract.id().to_buffer(), Some(&Epoch::new(0)), @@ -1457,9 +1477,39 @@ mod tests { .expect("got contract") .1 .expect("got contract fetch info"); - let ref_contract_fetch_info_after_prune = drive + + let ref_contract_fetch_info = drive + .get_contract_with_fetch_info(ref_contract_id_buffer, Some(&Epoch::new(0)), None) + .expect("got contract") + .1 + .expect("got contract fetch info"); + + assert_eq!( + deep_contract_fetch_info_transactional, + deep_contract_fetch_info + ); + + assert_eq!( + ref_contract_fetch_info_transactional, + ref_contract_fetch_info + ); + + /* + * User restarts the node + */ + + // Drop cache so contract will be fetched once again + drive.drop_cache(); + + /* + * Other nodes weren't restarted so contracts queried by user after restart + * must have the same costs as transactional contracts and contracts before + * restart + */ + + let deep_contract_fetch_info_without_cache = drive .get_contract_with_fetch_info( - ref_contract.id().to_buffer(), + deep_contract.id().to_buffer(), Some(&Epoch::new(0)), None, ) @@ -1467,42 +1517,83 @@ mod tests { .1 .expect("got contract fetch info"); + let ref_contract_fetch_info_without_cache = drive + .get_contract_with_fetch_info(ref_contract_id_buffer, Some(&Epoch::new(0)), None) + .expect("got contract") + .1 + .expect("got contract fetch info"); + assert_eq!( - deep_contract_fetch_info, - deep_contract_fetch_info_after_prune + deep_contract_fetch_info_transactional, + deep_contract_fetch_info_without_cache + ); + assert_eq!( + ref_contract_fetch_info_transactional, + ref_contract_fetch_info_without_cache ); - assert_eq!(ref_contract_fetch_info, ref_contract_fetch_info_after_prune); - + /* + * Let's imagine that many blocks were executed and the node is restarted again + */ drive.drop_cache(); - let deep_contract_fetch_info_after_cache_reload = drive + /* + * Drive executes a new block + */ + + let transaction = drive.grove.start_transaction(); + + // Create more contracts to trigger re-balancing + for i in 150..200u8 { + ref_contract.id = Identifier::from([i; 32]); + + let ref_contract_cbor = ref_contract.to_cbor().expect("should serialize contract"); + + drive + .apply_contract( + &ref_contract, + ref_contract_cbor, + BlockInfo::default(), + true, + StorageFlags::optional_default_as_ref(), + None, + ) + .expect("expected to apply contract successfully"); + } + + /* + * Other nodes weren't restarted so contracts fetched during block execution + * should have the same cost as previously fetched contracts + */ + + let deep_contract_fetch_info_transactional2 = drive .get_contract_with_fetch_info( deep_contract.id().to_buffer(), Some(&Epoch::new(0)), - None, + Some(&transaction), ) .expect("got contract") .1 .expect("got contract fetch info"); - let ref_contract_fetch_info_after_cache_reload = drive + let ref_contract_fetch_info_transactional2 = drive .get_contract_with_fetch_info( - ref_contract.id().to_buffer(), + ref_contract_id_buffer, Some(&Epoch::new(0)), - None, + Some(&transaction), ) .expect("got contract") .1 .expect("got contract fetch info"); assert_eq!( - deep_contract_fetch_info, - deep_contract_fetch_info_after_cache_reload + ref_contract_fetch_info_transactional, + ref_contract_fetch_info_transactional2, ); + assert_eq!( - ref_contract_fetch_info, - ref_contract_fetch_info_after_cache_reload + deep_contract_fetch_info_transactional, + deep_contract_fetch_info_transactional2 ); } } From 7071b089159abe8bc8a9370899a9248bf3c1a3e5 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sat, 7 Jan 2023 12:03:26 +0800 Subject: [PATCH 08/10] test: do not pass epoch on user call --- packages/rs-drive/src/drive/contract/mod.rs | 50 +++++++++++++-------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/packages/rs-drive/src/drive/contract/mod.rs b/packages/rs-drive/src/drive/contract/mod.rs index 02480d46f8c..11e22d19f25 100644 --- a/packages/rs-drive/src/drive/contract/mod.rs +++ b/packages/rs-drive/src/drive/contract/mod.rs @@ -113,7 +113,7 @@ pub fn add_init_contracts_structure_operations(batch: &mut GroveDbOpBatch) { } /// Contract and fetch information -#[derive(Default, PartialEq, Debug)] +#[derive(Default, PartialEq, Debug, Clone)] pub struct ContractFetchInfo { /// The contract pub contract: Contract, @@ -1433,7 +1433,7 @@ mod tests { ) .expect("expected to apply contract successfully"); - let ref_contract_fetch_info_transactional = drive + let mut ref_contract_fetch_info_transactional = drive .get_contract_with_fetch_info( ref_contract_id_buffer, Some(&Epoch::new(0)), @@ -1443,7 +1443,7 @@ mod tests { .1 .expect("got contract fetch info"); - let deep_contract_fetch_info_transactional = drive + let mut deep_contract_fetch_info_transactional = drive .get_contract_with_fetch_info( deep_contract.id().to_buffer(), Some(&Epoch::new(0)), @@ -1469,17 +1469,13 @@ mod tests { */ let deep_contract_fetch_info = drive - .get_contract_with_fetch_info( - deep_contract.id().to_buffer(), - Some(&Epoch::new(0)), - None, - ) + .get_contract_with_fetch_info(deep_contract.id().to_buffer(), None, None) .expect("got contract") .1 .expect("got contract fetch info"); let ref_contract_fetch_info = drive - .get_contract_with_fetch_info(ref_contract_id_buffer, Some(&Epoch::new(0)), None) + .get_contract_with_fetch_info(ref_contract_id_buffer, None, None) .expect("got contract") .1 .expect("got contract fetch info"); @@ -1508,21 +1504,28 @@ mod tests { */ let deep_contract_fetch_info_without_cache = drive - .get_contract_with_fetch_info( - deep_contract.id().to_buffer(), - Some(&Epoch::new(0)), - None, - ) + .get_contract_with_fetch_info(deep_contract.id().to_buffer(), None, None) .expect("got contract") .1 .expect("got contract fetch info"); let ref_contract_fetch_info_without_cache = drive - .get_contract_with_fetch_info(ref_contract_id_buffer, Some(&Epoch::new(0)), None) + .get_contract_with_fetch_info(ref_contract_id_buffer, None, None) .expect("got contract") .1 .expect("got contract fetch info"); + // Remove fees to match with fetch with epoch provided + let mut deep_contract_fetch_info_transactional_without_arc = + Arc::make_mut(&mut deep_contract_fetch_info_transactional); + + deep_contract_fetch_info_transactional_without_arc.fee = None; + + let mut ref_contract_fetch_info_transactional_without_arc = + Arc::make_mut(&mut ref_contract_fetch_info_transactional); + + ref_contract_fetch_info_transactional_without_arc.fee = None; + assert_eq!( deep_contract_fetch_info_transactional, deep_contract_fetch_info_without_cache @@ -1556,7 +1559,7 @@ mod tests { BlockInfo::default(), true, StorageFlags::optional_default_as_ref(), - None, + Some(&transaction), ) .expect("expected to apply contract successfully"); } @@ -1566,7 +1569,7 @@ mod tests { * should have the same cost as previously fetched contracts */ - let deep_contract_fetch_info_transactional2 = drive + let mut deep_contract_fetch_info_transactional2 = drive .get_contract_with_fetch_info( deep_contract.id().to_buffer(), Some(&Epoch::new(0)), @@ -1576,7 +1579,7 @@ mod tests { .1 .expect("got contract fetch info"); - let ref_contract_fetch_info_transactional2 = drive + let mut ref_contract_fetch_info_transactional2 = drive .get_contract_with_fetch_info( ref_contract_id_buffer, Some(&Epoch::new(0)), @@ -1586,6 +1589,17 @@ mod tests { .1 .expect("got contract fetch info"); + // Remove fees to match with fetch with epoch provided + let mut deep_contract_fetch_info_transactional_without_arc = + Arc::make_mut(&mut deep_contract_fetch_info_transactional2); + + deep_contract_fetch_info_transactional_without_arc.fee = None; + + let mut ref_contract_fetch_info_transactional_without_arc = + Arc::make_mut(&mut ref_contract_fetch_info_transactional2); + + ref_contract_fetch_info_transactional_without_arc.fee = None; + assert_eq!( ref_contract_fetch_info_transactional, ref_contract_fetch_info_transactional2, From 649b2e1a9fa4cd1e0c972a94f68d655ea971fcb6 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 9 Jan 2023 15:22:45 +0700 Subject: [PATCH 09/10] fixes for deterministic querying --- packages/rs-drive/src/drive/contract/mod.rs | 16 +++++++++++++--- packages/rs-drive/src/drive/document/insert.rs | 6 +++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/rs-drive/src/drive/contract/mod.rs b/packages/rs-drive/src/drive/contract/mod.rs index 11e22d19f25..dfd8ddfe347 100644 --- a/packages/rs-drive/src/drive/contract/mod.rs +++ b/packages/rs-drive/src/drive/contract/mod.rs @@ -854,7 +854,7 @@ impl Drive { // As we want deterministic costs, we want the cost to always be the same for // fetching the contract. // We need to pass allow cache to false - let CostContext { value, cost } = self.grove.get_caching_optional( + let CostContext { value, cost } = self.grove.get_raw_caching_optional( contract_root_path(&contract_id), &[0], false, @@ -1606,8 +1606,18 @@ mod tests { ); assert_eq!( - deep_contract_fetch_info_transactional, - deep_contract_fetch_info_transactional2 + deep_contract_fetch_info_transactional.contract, + deep_contract_fetch_info_transactional2.contract + ); + + assert_eq!( + deep_contract_fetch_info_transactional.storage_flags, + deep_contract_fetch_info_transactional2.storage_flags + ); + + assert_eq!( + deep_contract_fetch_info_transactional.cost, + deep_contract_fetch_info_transactional2.cost ); } } diff --git a/packages/rs-drive/src/drive/document/insert.rs b/packages/rs-drive/src/drive/document/insert.rs index 271fd08050c..a542879feba 100644 --- a/packages/rs-drive/src/drive/document/insert.rs +++ b/packages/rs-drive/src/drive/document/insert.rs @@ -1357,7 +1357,7 @@ mod tests { .expect("expected to insert a document successfully"); let added_bytes = storage_fee / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; - assert_eq!((3247, 2914000), (added_bytes, processing_fee)); + assert_eq!((3247, 2838400), (added_bytes, processing_fee)); } #[test] @@ -1405,7 +1405,7 @@ mod tests { .expect("expected to insert a document successfully"); let added_bytes = storage_fee / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; - assert_eq!((1428, 1895000), (added_bytes, processing_fee)); + assert_eq!((1428, 1865400), (added_bytes, processing_fee)); } #[test] @@ -1672,7 +1672,7 @@ mod tests { .expect("expected to insert a document successfully"); let added_bytes = storage_fee / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; - assert_eq!((1986, 2604600), (added_bytes, processing_fee)); + assert_eq!((1986, 2556600), (added_bytes, processing_fee)); drive .grove From 2ce3f3bde0abc6863adbf6d12a66aa790f638eec Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 9 Jan 2023 17:58:01 +0700 Subject: [PATCH 10/10] updated grove version --- Cargo.lock | 14 +++++++------- packages/rs-drive/Cargo.toml | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 43a8637a7cf..b8bad86fdb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -574,7 +574,7 @@ dependencies = [ [[package]] name = "costs" version = "1.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=af53d3729d19ad5bd1912970fbb599b264b9046b#af53d3729d19ad5bd1912970fbb599b264b9046b" +source = "git+https://github.com/dashpay/grovedb?rev=a9e168767c318b94b6edd119c51542c43e8772b2#a9e168767c318b94b6edd119c51542c43e8772b2" dependencies = [ "integer-encoding", "intmap", @@ -1253,8 +1253,8 @@ dependencies = [ [[package]] name = "grovedb" -version = "0.9.2" -source = "git+https://github.com/dashpay/grovedb?rev=af53d3729d19ad5bd1912970fbb599b264b9046b#af53d3729d19ad5bd1912970fbb599b264b9046b" +version = "0.9.3" +source = "git+https://github.com/dashpay/grovedb?rev=a9e168767c318b94b6edd119c51542c43e8772b2#a9e168767c318b94b6edd119c51542c43e8772b2" dependencies = [ "bincode", "costs", @@ -1687,8 +1687,8 @@ dependencies = [ [[package]] name = "merk" -version = "0.9.2" -source = "git+https://github.com/dashpay/grovedb?rev=af53d3729d19ad5bd1912970fbb599b264b9046b#af53d3729d19ad5bd1912970fbb599b264b9046b" +version = "0.9.3" +source = "git+https://github.com/dashpay/grovedb?rev=a9e168767c318b94b6edd119c51542c43e8772b2#a9e168767c318b94b6edd119c51542c43e8772b2" dependencies = [ "blake3", "byteorder", @@ -2729,7 +2729,7 @@ dependencies = [ [[package]] name = "storage" version = "1.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=af53d3729d19ad5bd1912970fbb599b264b9046b#af53d3729d19ad5bd1912970fbb599b264b9046b" +source = "git+https://github.com/dashpay/grovedb?rev=a9e168767c318b94b6edd119c51542c43e8772b2#a9e168767c318b94b6edd119c51542c43e8772b2" dependencies = [ "blake3", "costs", @@ -3116,7 +3116,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "visualize" version = "0.1.0" -source = "git+https://github.com/dashpay/grovedb?rev=af53d3729d19ad5bd1912970fbb599b264b9046b#af53d3729d19ad5bd1912970fbb599b264b9046b" +source = "git+https://github.com/dashpay/grovedb?rev=a9e168767c318b94b6edd119c51542c43e8772b2#a9e168767c318b94b6edd119c51542c43e8772b2" dependencies = [ "hex", "itertools", diff --git a/packages/rs-drive/Cargo.toml b/packages/rs-drive/Cargo.toml index a88519ff5ad..3a31095caa0 100644 --- a/packages/rs-drive/Cargo.toml +++ b/packages/rs-drive/Cargo.toml @@ -36,15 +36,15 @@ rust_decimal_macros = "1.25.0" [dependencies.grovedb] git = "https://github.com/dashpay/grovedb" -rev = "af53d3729d19ad5bd1912970fbb599b264b9046b" +rev = "a9e168767c318b94b6edd119c51542c43e8772b2" [dependencies.storage] git = "https://github.com/dashpay/grovedb" -rev = "af53d3729d19ad5bd1912970fbb599b264b9046b" +rev = "a9e168767c318b94b6edd119c51542c43e8772b2" [dependencies.costs] git = "https://github.com/dashpay/grovedb" -rev = "af53d3729d19ad5bd1912970fbb599b264b9046b" +rev = "a9e168767c318b94b6edd119c51542c43e8772b2" [dev-dependencies] criterion = "0.3.5"