From bed038093a4029be3f9697de02e058880e7698dd Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Mon, 12 Dec 2022 23:59:09 +0800 Subject: [PATCH 01/54] feat!: credit refunds --- .../AbstractStateTransition.js | 14 +++- .../StateTransitionExecutionContext.js | 27 +++++++ .../fee/calculateOperationFees.js | 20 +++++- .../fee/calculateStateTransitionFee.js | 46 ++++++++++-- .../fee/operations/AbstractOperation.js | 8 +++ .../fee/operations/PreCalculatedOperation.js | 7 ++ .../fee/operations/ReadOperation.js | 7 ++ .../SignatureVerificationOperation.js | 7 ++ .../handlers/prepareProposalHandlerFactory.js | 14 ++-- .../handlers/proposal/deliverTxFactory.js | 72 +++++++++---------- .../abci/handlers/proposal/endBlockFactory.js | 30 ++++---- .../proposal/fees/addToFeeTxResults.js | 32 +++++++++ .../handlers/proposal/fees/aggregateFees.js | 20 ------ .../proposal/fees/aggregateOperationFees.js | 42 ----------- .../proposal/processProposalFactory.js | 15 ++-- .../unserializeStateTransitionFactory.js | 15 ++-- packages/rs-drive-nodejs/FeeResult.js | 11 +++ packages/rs-drive-nodejs/src/fee_result.rs | 35 +++++++++ packages/rs-drive-nodejs/src/lib.rs | 1 + .../rs-drive/src/drive/document/delete.rs | 7 +- .../rs-drive/src/drive/document/insert.rs | 8 +-- .../rs-drive/src/drive/document/update.rs | 6 +- packages/rs-drive/src/fee/mod.rs | 11 ++- packages/rs-drive/src/fee/op.rs | 26 ++++--- ...rom_epochs_by_identities.rs => refunds.rs} | 47 +++++++++--- 25 files changed, 355 insertions(+), 173 deletions(-) create mode 100644 packages/js-drive/lib/abci/handlers/proposal/fees/addToFeeTxResults.js delete mode 100644 packages/js-drive/lib/abci/handlers/proposal/fees/aggregateFees.js delete mode 100644 packages/js-drive/lib/abci/handlers/proposal/fees/aggregateOperationFees.js rename packages/rs-drive/src/fee/{removed_bytes_from_epochs_by_identities.rs => refunds.rs} (62%) diff --git a/packages/js-dpp/lib/stateTransition/AbstractStateTransition.js b/packages/js-dpp/lib/stateTransition/AbstractStateTransition.js index 4fdc332dc98..f7ad8208eb9 100644 --- a/packages/js-dpp/lib/stateTransition/AbstractStateTransition.js +++ b/packages/js-dpp/lib/stateTransition/AbstractStateTransition.js @@ -112,6 +112,14 @@ class AbstractStateTransition { return rawStateTransition; } + /** + * @abstract + * @return {Identifier} + */ + getOwnerId() { + throw new Error('Not implemented'); + } + /** * Get state transition as JSON * @@ -287,10 +295,12 @@ class AbstractStateTransition { /** * Calculate ST fee in credits * + * @param {Object} options + * @param {boolean} [options.useCache=false] * @return {number} */ - calculateFee() { - return calculateStateTransitionFee(this); + calculateFee(options = {}) { + return calculateStateTransitionFee(this, options); } /** diff --git a/packages/js-dpp/lib/stateTransition/StateTransitionExecutionContext.js b/packages/js-dpp/lib/stateTransition/StateTransitionExecutionContext.js index fd3bb3720b9..75b9af2ce0b 100644 --- a/packages/js-dpp/lib/stateTransition/StateTransitionExecutionContext.js +++ b/packages/js-dpp/lib/stateTransition/StateTransitionExecutionContext.js @@ -9,6 +9,7 @@ class StateTransitionExecutionContext { */ this.dryOperations = []; this.dryRun = false; + this.lastCalculatedFees = null; } /** @@ -64,6 +65,32 @@ class StateTransitionExecutionContext { isDryRun() { return this.dryRun; } + + /** + * @return {{ + * storageFee: number, + * processingFee: number, + * feeRefunds: {identifier: Buffer, creditsPerEpoch: Object}[], + * feeRefundsSum: number, + * total: number, + * }|null} + */ + getLastCalculatedFeeDetails() { + return this.lastCalculatedFees; + } + + /** + * @param {{ + * storageFee:number, + * processingFee:number, + * feeRefunds: {identifier: Buffer, creditsPerEpoch: Object}[], + * feeRefundsSum: number, + * total: number, + * }} lastCalculatedFees + */ + setLastCalculatedFeeDetails(lastCalculatedFees) { + this.lastCalculatedFees = lastCalculatedFees; + } } module.exports = StateTransitionExecutionContext; diff --git a/packages/js-dpp/lib/stateTransition/fee/calculateOperationFees.js b/packages/js-dpp/lib/stateTransition/fee/calculateOperationFees.js index a915f2a1dea..0af2c2e6b0b 100644 --- a/packages/js-dpp/lib/stateTransition/fee/calculateOperationFees.js +++ b/packages/js-dpp/lib/stateTransition/fee/calculateOperationFees.js @@ -7,15 +7,32 @@ const { * * @param {AbstractOperation[]} operations * - * @returns {{storageFee: number, processingFee: number}} + * @returns {{ + * storageFee: number, + * processingFee: number, + * feeRefunds: {identifier: Buffer, creditsPerEpoch: Object}[] + * }} */ function calculateOperationFees(operations) { let storageFee = 0; let processingFee = 0; + let feeResult; + operations.forEach((operation) => { + // TODO should use checked add when moved to Rust + storageFee += operation.getStorageCost(); processingFee += operation.getProcessingCost(); + + // Combine refunds which are currently present only in RS Drive's Fee Result + if (operation.feeResult && operation.feeResult.inner) { + if (!feeResult) { + feeResult = operation.feeResult; + } else { + feeResult.add(operation.feeResult); + } + } }); // TODO: Do we need to multiply pre calculated fees? @@ -26,6 +43,7 @@ function calculateOperationFees(operations) { return { storageFee, processingFee, + feeRefunds: feeResult ? feeResult.feeRefunds : [], }; } diff --git a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js index d9490966009..af0f2dc321e 100644 --- a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js +++ b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js @@ -7,16 +7,52 @@ const calculateOperationFees = require('./calculateOperationFees'); /** * @typedef {calculateStateTransitionFee} * @param {AbstractStateTransition} stateTransition + * @param {Object} options + * @param {boolean} [options.useCache=false] * @return {number} */ -function calculateStateTransitionFee(stateTransition) { +function calculateStateTransitionFee(stateTransition, options = {}) { const executionContext = stateTransition.getExecutionContext(); - const { storageFee, processingFee } = calculateOperationFees( - executionContext.getOperations(), - ); + if (options.useCache) { + const calculatedFeeDetails = executionContext.getLastCalculatedFeeDetails(); - return (storageFee + processingFee) + DEFAULT_USER_TIP; + if (!calculatedFeeDetails) { + throw new Error('State Transition Execution context doesn\'t contain cached fee calculation'); + } + + return calculatedFeeDetails.total; + } + + const calculatedFees = calculateOperationFees(executionContext.getOperations()); + + const { + storageFee, + processingFee, + feeRefunds, + } = calculatedFees; + + if (feeRefunds.length > 1) { + throw new Error('State Transition removed bytes from several identities that is not defined by protocol'); + } + + let feeRefundsSum = 0; + if (feeRefunds.length > 0) { + const stateTransitionIdentifier = stateTransition.getOwnerId(); + + if (!stateTransitionIdentifier.equals(feeRefunds[0].identifier)) { + throw new Error('State Transition removed bytes from different identity'); + } + + feeRefundsSum = feeRefunds[0].creditsPerEpoch.entries() + .reduce((sum, [, credits]) => sum + credits, 0); + } + + const total = (storageFee + processingFee - feeRefundsSum) + DEFAULT_USER_TIP; + + executionContext.setLastCalculatedFeeDetails({ ...calculatedFees, feeRefundsSum, total }); + + return total; } module.exports = calculateStateTransitionFee; diff --git a/packages/js-dpp/lib/stateTransition/fee/operations/AbstractOperation.js b/packages/js-dpp/lib/stateTransition/fee/operations/AbstractOperation.js index ed291c36a9d..58cc61790a8 100644 --- a/packages/js-dpp/lib/stateTransition/fee/operations/AbstractOperation.js +++ b/packages/js-dpp/lib/stateTransition/fee/operations/AbstractOperation.js @@ -18,6 +18,14 @@ class AbstractOperation { throw new Error('Not implemented'); } + /** + * @abstract + * @return {{identifier: Buffer, creditsPerEpoch: Object}[]} + */ + getRefunds() { + throw new Error('Not implemented'); + } + /** * @abstract * @returns {Object} diff --git a/packages/js-dpp/lib/stateTransition/fee/operations/PreCalculatedOperation.js b/packages/js-dpp/lib/stateTransition/fee/operations/PreCalculatedOperation.js index 205654e718d..f5ba14e47a3 100644 --- a/packages/js-dpp/lib/stateTransition/fee/operations/PreCalculatedOperation.js +++ b/packages/js-dpp/lib/stateTransition/fee/operations/PreCalculatedOperation.js @@ -30,6 +30,13 @@ class PreCalculatedOperation extends AbstractOperation { return this.feeResult.storageFee; } + /** + * @return {{identifier: Buffer, creditsPerEpoch: Object}[]} + */ + getRefunds() { + return this.feeResult.feeRefunds; + } + /** * @return {{processingCost: number, type: string, storageCost: number}} */ diff --git a/packages/js-dpp/lib/stateTransition/fee/operations/ReadOperation.js b/packages/js-dpp/lib/stateTransition/fee/operations/ReadOperation.js index e9462861c9a..010934a385a 100644 --- a/packages/js-dpp/lib/stateTransition/fee/operations/ReadOperation.js +++ b/packages/js-dpp/lib/stateTransition/fee/operations/ReadOperation.js @@ -33,6 +33,13 @@ class ReadOperation extends AbstractOperation { return 0; } + /** + * @return {{identifier: Buffer, creditsPerEpoch: Object}[]} + */ + getRefunds() { + return []; + } + /** * @return {{valueSize: number, type: string}} */ diff --git a/packages/js-dpp/lib/stateTransition/fee/operations/SignatureVerificationOperation.js b/packages/js-dpp/lib/stateTransition/fee/operations/SignatureVerificationOperation.js index 8f069ef0f2b..f2b18ab16bb 100644 --- a/packages/js-dpp/lib/stateTransition/fee/operations/SignatureVerificationOperation.js +++ b/packages/js-dpp/lib/stateTransition/fee/operations/SignatureVerificationOperation.js @@ -38,6 +38,13 @@ class SignatureVerificationOperation extends AbstractOperation { return 0; } + /** + * @return {{identifier: Buffer, creditsPerEpoch: Object}[]} + */ + getRefunds() { + return []; + } + /** * @return {{signatureType: number, type: string}} */ diff --git a/packages/js-drive/lib/abci/handlers/prepareProposalHandlerFactory.js b/packages/js-drive/lib/abci/handlers/prepareProposalHandlerFactory.js index d52dfe59b3e..c34c4b67510 100644 --- a/packages/js-drive/lib/abci/handlers/prepareProposalHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/prepareProposalHandlerFactory.js @@ -6,7 +6,7 @@ const { }, } = require('@dashevo/abci/types'); -const aggregateFees = require('./proposal/fees/aggregateFees'); +const addToFeeTxResults = require('./proposal/fees/addToFeeTxResults'); const txAction = { UNKNOWN: 0, // Unknown action @@ -80,7 +80,13 @@ function prepareProposalHandlerFactory( const txRecords = []; const txResults = []; - const feeResults = []; + const feeResults = { + storageFee: 0, + processingFee: 0, + feeRefunds: { }, + feeRefundsSum: 0, + }; + let validTxCount = 0; let invalidTxCount = 0; @@ -105,7 +111,7 @@ function prepareProposalHandlerFactory( if (code === 0) { validTxCount += 1; // TODO We probably should calculate fees for invalid transitions as well - feeResults.push(fees); + addToFeeTxResults(feeResults, fees); } else { invalidTxCount += 1; } @@ -135,7 +141,7 @@ function prepareProposalHandlerFactory( } = await endBlock({ height, round, - fees: aggregateFees(feeResults), + fees: feeResults, coreChainLockedHeight, }, consensusLogger); diff --git a/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js b/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js index 4fa28bf1f9b..282fe5f137a 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js @@ -5,9 +5,6 @@ const AbstractDocumentTransition = require( '@dashevo/dpp/lib/document/stateTransition/DocumentsBatchTransition/documentTransition/AbstractDocumentTransition', ); -const calculateOperationFees = require('@dashevo/dpp/lib/stateTransition/fee/calculateOperationFees'); -const aggregateOperationFees = require('./fees/aggregateOperationFees'); - const DPPValidationAbciError = require('../../errors/DPPValidationAbciError'); const DOCUMENT_ACTION_DESCRIPTIONS = { @@ -43,7 +40,14 @@ function deliverTxFactory( * @param {Buffer} stateTransitionByteArray * @param {number} round * @param {BaseLogger} consensusLogger - * @return {Promise<{ code: number, fees: FeeResult }>} + * @return {Promise<{ + * code: number, + * fees: { + * storageFee: number, + * processingFee: number, + * feeRefunds: Object, + * feeRefundsSum: number, + * }}>} */ async function deliverTx(stateTransitionByteArray, round, consensusLogger) { const blockHeight = proposalBlockExecutionContext.getHeight(); @@ -85,8 +89,9 @@ function deliverTxFactory( // Keep only actual operations const stateTransitionExecutionContext = stateTransition.getExecutionContext(); - const predictedStateTransitionFee = stateTransition.calculateFee(); const predictedStateTransitionOperations = stateTransitionExecutionContext.getOperations(); + const predictedStateTransitionFees = stateTransitionExecutionContext + .getLastCalculatedFeeDetails(); stateTransitionExecutionContext.clearDryOperations(); @@ -117,24 +122,24 @@ function deliverTxFactory( // Reduce an identity balance and accumulate fees for all STs in the block // in order to store them in credits distribution pool - const actualStateTransitionFee = stateTransition.calculateFee(); + const actualStateTransitionFees = stateTransitionExecutionContext + .getLastCalculatedFeeDetails(); + const actualStateTransitionOperations = stateTransition.getExecutionContext().getOperations(); - // TODO: enable once fee calculation is done - // if (actualStateTransitionFee > predictedStateTransitionFee) { - // throw new PredictedFeeLowerThanActualError( - // predictedStateTransitionFee, - // actualStateTransitionFee, - // stateTransition, - // ); - // } + if (actualStateTransitionFees.total > predictedStateTransitionFees.total) { + txConsensusLogger.warn({ + predictedFee: predictedStateTransitionFees.total, + actualFee: actualStateTransitionFees.total, + }, `Actual fees are greater than predicted for ${actualStateTransitionFees.total - predictedStateTransitionFees.total} credits`); + } const identity = await transactionalDpp.getStateRepository().fetchIdentity( stateTransition.getOwnerId(), ); - // TODO: We should increment identity balance debt in case if it goes negative - let updatedBalance = identity.getBalance() - actualStateTransitionFee; + let updatedBalance = identity.getBalance() - actualStateTransitionFees.total; + // TODO: We should increment identity balance debt in case if it goes negative if (updatedBalance < 0) { updatedBalance = 0; } @@ -219,18 +224,6 @@ function deliverTxFactory( const deliverTxTiming = executionTimer.stopTimer(TIMERS.DELIVER_TX.OVERALL); - const actualStateTransitionOperations = stateTransition.getExecutionContext().getOperations(); - - const { - storageFee: actualStorageFees, - processingFee: actualProcessingFees, - } = calculateOperationFees(actualStateTransitionOperations); - - const { - storageFee: predictedStorageFee, - processingFee: predictedProcessingFee, - } = calculateOperationFees(predictedStateTransitionOperations); - txConsensusLogger.trace( { timings: { @@ -243,26 +236,33 @@ function deliverTxFactory( }, fees: { predicted: { - storage: predictedStorageFee, - processing: predictedProcessingFee, - final: predictedStateTransitionFee, + storage: predictedStateTransitionFees.storageFee, + processing: predictedStateTransitionFees.processingFee, + refunds: predictedStateTransitionFees.feeRefundsSum, + final: predictedStateTransitionFees.total, operations: predictedStateTransitionOperations.map((operation) => operation.toJSON()), }, actual: { - storage: actualStorageFees, - processing: actualProcessingFees, - final: actualStateTransitionFee, + storage: actualStateTransitionFees.storageFee, + processing: actualStateTransitionFees.processingFee, + refunds: actualStateTransitionFees.feeRefundsSum, + final: actualStateTransitionFees.total, operations: actualStateTransitionOperations.map((operation) => operation.toJSON()), }, }, txType: stateTransition.getType(), }, - `${stateTransition.constructor.name} execution took ${deliverTxTiming} seconds and cost ${actualStateTransitionFee} credits`, + `${stateTransition.constructor.name} execution took ${deliverTxTiming} seconds and cost ${actualStateTransitionFees.total} credits`, ); return { code: 0, - fees: aggregateOperationFees(actualStateTransitionOperations), + fees: { + storageFee: actualStateTransitionFees.storageFee, + processingFee: actualStateTransitionFees.processingFee, + feeRefunds: actualStateTransitionFees.feeRefunds[0].creditsPerEpoch, + feeRefundsSum: actualStateTransitionFees.feeRefundsSum, + }, }; } diff --git a/packages/js-drive/lib/abci/handlers/proposal/endBlockFactory.js b/packages/js-drive/lib/abci/handlers/proposal/endBlockFactory.js index 4eb35d17e10..c529c302045 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/endBlockFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/endBlockFactory.js @@ -30,7 +30,12 @@ function endBlockFactory( * @param {Object} request * @param {number} request.height * @param {number} request.round - * @param {FeeResult} request.fees + * @param { + * storageFee: number, + * processingFee: number, + * feeRefunds: Object, + * feeRefundsSum: number + * } request.fees * @param {number} request.coreChainLockedHeight * @param {BaseLogger} consensusLogger * @return {Promise<{ @@ -46,14 +51,19 @@ function endBlockFactory( const { height, round, - fees, + fees: { + feeRefunds, + processingFee, + storageFee, + feeRefundsSum, + }, coreChainLockedHeight, } = request; // Call RS ABCI const rsRequest = { - fees, + feeRefunds, }; consensusLogger.debug(rsRequest, 'Request RS Drive\'s BlockEnd method'); @@ -64,17 +74,13 @@ function endBlockFactory( const { currentEpochIndex } = proposalBlockExecutionContext.getEpochInfo(); - const { - processingFee: processingFees, - storageFee: storageFees, - } = fees; - - if (processingFees > 0 || storageFees > 0) { + if (processingFee > 0 || storageFee > 0) { consensusLogger.debug({ currentEpochIndex, - processingFees, - storageFees, - }, `${processingFees} processing fees added to epoch #${currentEpochIndex}. ${storageFees} storage fees added to distribution pool`); + processingFee, + storageFee, + feeRefundsSum, + }, `${processingFee} processing fees added to epoch #${currentEpochIndex}. ${storageFee} storage fees added to distribution pool. ${feeRefundsSum} credits refunded to identities`); } if (rsResponse.proposersPaidCount) { diff --git a/packages/js-drive/lib/abci/handlers/proposal/fees/addToFeeTxResults.js b/packages/js-drive/lib/abci/handlers/proposal/fees/addToFeeTxResults.js new file mode 100644 index 00000000000..50c3307d034 --- /dev/null +++ b/packages/js-drive/lib/abci/handlers/proposal/fees/addToFeeTxResults.js @@ -0,0 +1,32 @@ +/** + * @param {{ + * storageFee: number, + * processingFee: number, + * feeRefunds: Object, + * feeRefundsSum: number, + * }} feeResults + * @param {{ + * storageFee: number, + * processingFee: number, + * feeRefunds: Object, + * feeRefundsSum: number, + * }} feeResult + */ +function addToFeeTxResults(feeResults, feeResult) { + /* eslint-disable no-param-reassign */ + feeResults.storageFee += feeResult.storageFee; + feeResults.processingFee += feeResult.processingFee; + feeResults.feeRefundsSum += feeResult.feeRefundsSum; + + for (const [epochIndex, credits] of feeResult.feeRefunds.entries()) { + if (!feeResults.feeRefunds[epochIndex]) { + feeResults.feeRefunds[epochIndex] = 0; + } + + feeResults.feeRefunds[epochIndex] += credits; + } + + return feeResults; +} + +module.exports = addToFeeTxResults; diff --git a/packages/js-drive/lib/abci/handlers/proposal/fees/aggregateFees.js b/packages/js-drive/lib/abci/handlers/proposal/fees/aggregateFees.js deleted file mode 100644 index 16568ba19ce..00000000000 --- a/packages/js-drive/lib/abci/handlers/proposal/fees/aggregateFees.js +++ /dev/null @@ -1,20 +0,0 @@ -const FeeResult = require('@dashevo/rs-drive/FeeResult'); - -/** - * Calculate processing and storage fees based on operations - * - * @param {FeeResult[]} feeResults - * - * @returns {FeeResult} - */ -function aggregateFees(feeResults) { - const rootFeeResult = FeeResult.create(); - - feeResults.forEach((feeResult) => { - rootFeeResult.add(feeResult); - }); - - return rootFeeResult; -} - -module.exports = aggregateFees; diff --git a/packages/js-drive/lib/abci/handlers/proposal/fees/aggregateOperationFees.js b/packages/js-drive/lib/abci/handlers/proposal/fees/aggregateOperationFees.js deleted file mode 100644 index 0d60ea7aba6..00000000000 --- a/packages/js-drive/lib/abci/handlers/proposal/fees/aggregateOperationFees.js +++ /dev/null @@ -1,42 +0,0 @@ -const { - FEE_MULTIPLIER, -} = require('@dashevo/dpp/lib/stateTransition/fee/constants'); - -const PreCalculatedOperation = require('@dashevo/dpp/lib/stateTransition/fee/operations/PreCalculatedOperation'); -const FeeResult = require('@dashevo/rs-drive/FeeResult'); - -/** - * Calculate processing and storage fees based on operations - * - * @param {AbstractOperation[]} operations - * - * @returns {FeeResult} - */ -function aggregateOperationFees(operations) { - let storageFee = 0; - let processingFee = 0; - - const rootFeeResult = FeeResult.create(); - - operations.forEach((operation) => { - if (operation instanceof PreCalculatedOperation && operation.feeResult instanceof FeeResult) { - rootFeeResult.add(operation.feeResult); - } else { - // Collect fees from operation which are not based on FeeResult - // and add to the root fee result later - storageFee += operation.getStorageCost(); - processingFee += operation.getProcessingCost(); - } - }); - - // TODO: Do we need to multiply pre calculated fees? - - storageFee *= FEE_MULTIPLIER; - processingFee *= FEE_MULTIPLIER; - - rootFeeResult.addFees(storageFee, processingFee); - - return rootFeeResult; -} - -module.exports = aggregateOperationFees; diff --git a/packages/js-drive/lib/abci/handlers/proposal/processProposalFactory.js b/packages/js-drive/lib/abci/handlers/proposal/processProposalFactory.js index 2ede0ba2b88..d80fb6da913 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/processProposalFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/processProposalFactory.js @@ -8,7 +8,7 @@ const { const statuses = require('./statuses'); -const aggregateFees = require('./fees/aggregateFees'); +const addToFeeTxResults = require('./fees/addToFeeTxResults'); /** * @@ -61,7 +61,12 @@ function processProposalFactory( ); const txResults = []; - const feeResults = []; + const feeResults = { + storageFee: 0, + processingFee: 0, + feeRefunds: { }, + feeRefundsSum: 0, + }; let validTxCount = 0; let invalidTxCount = 0; @@ -75,8 +80,8 @@ function processProposalFactory( if (code === 0) { validTxCount += 1; - // TODO We probably should calculate fees for invalid transitions as well - feeResults.push(fees); + // TODO We should calculate fees for invalid transitions as well + addToFeeTxResults(feeResults, fees); } else { invalidTxCount += 1; } @@ -100,7 +105,7 @@ function processProposalFactory( } = await endBlock({ height, round, - fees: aggregateFees(feeResults), + fees: feeResults, coreChainLockedHeight, }, consensusLogger); diff --git a/packages/js-drive/lib/abci/handlers/stateTransition/unserializeStateTransitionFactory.js b/packages/js-drive/lib/abci/handlers/stateTransition/unserializeStateTransitionFactory.js index aaabd3db10d..252fccf29d9 100644 --- a/packages/js-drive/lib/abci/handlers/stateTransition/unserializeStateTransitionFactory.js +++ b/packages/js-drive/lib/abci/handlers/stateTransition/unserializeStateTransitionFactory.js @@ -83,17 +83,16 @@ function unserializeStateTransitionFactory(dpp, noopLogger) { executionTimer.startTimer(TIMERS.DELIVER_TX.VALIDATE_FEE); - // const executionContext = stateTransition.getExecutionContext(); + const executionContext = stateTransition.getExecutionContext(); - // TODO: Enable fee validation when RS Drive is ready // Pre-calculate fee for validateState and state transition apply // with worst case costs to validate the whole state transition execution cost - // executionContext.enableDryRun(); - // - // await dpp.stateTransition.validateState(stateTransition); - // await dpp.stateTransition.apply(stateTransition); - // - // executionContext.disableDryRun(); + executionContext.enableDryRun(); + + await dpp.stateTransition.validateState(stateTransition); + await dpp.stateTransition.apply(stateTransition); + + executionContext.disableDryRun(); result = await dpp.stateTransition.validateFee(stateTransition); diff --git a/packages/rs-drive-nodejs/FeeResult.js b/packages/rs-drive-nodejs/FeeResult.js index 78905156c48..318c4f2a6b1 100644 --- a/packages/rs-drive-nodejs/FeeResult.js +++ b/packages/rs-drive-nodejs/FeeResult.js @@ -6,6 +6,7 @@ const { feeResultGetProcessingFee, feeResultAddFees, feeResultCreate, + feeResultGetRefunds, } = require('neon-load-or-build')({ dir: __dirname, }); @@ -17,6 +18,7 @@ const feeResultAddFeesWithStack = appendStack(feeResultAddFees); const feeResultGetStorageFeeWithStack = appendStack(feeResultGetStorageFee); const feeResultGetProcessingFeeWithStack = appendStack(feeResultGetProcessingFee); const feeResultCreateWithStack = appendStack(feeResultCreate); +const feeResultGetRefundsWithStack = appendStack(feeResultGetRefunds); class FeeResult { constructor(inner) { @@ -41,6 +43,15 @@ class FeeResult { return feeResultGetStorageFeeWithStack.call(this.inner); } + /** + * Credit refunds + * + * @return {{identifier: Buffer, creditsPerEpoch: Object}[]} + */ + get feeRefunds() { + return feeResultGetRefundsWithStack.call(this.inner); + } + /** * Adds and self assigns result between two Fee Results * diff --git a/packages/rs-drive-nodejs/src/fee_result.rs b/packages/rs-drive-nodejs/src/fee_result.rs index 6485dea6fda..28d8e69ea4d 100644 --- a/packages/rs-drive-nodejs/src/fee_result.rs +++ b/packages/rs-drive-nodejs/src/fee_result.rs @@ -76,6 +76,41 @@ impl FeeResultWrapper { Ok(cx.boxed(Self::new(fee_result_sum))) } + + pub fn get_fee_refunds(mut cx: FunctionContext) -> JsResult { + let fee_result_wrapper_self = cx + .this() + .downcast_or_throw::, _>(&mut cx)?; + + // Clone fee result because IntMap doesn't implement iterator for reference + let fee_result = fee_result_wrapper_self.deref().deref().deref().clone(); + + let js_fee_refunds: Handle = cx.empty_array(); + + for (index, (identifier, credits_per_epoch)) in + fee_result.fee_refunds.0.into_iter().enumerate() + { + let js_epoch_index_map = cx.empty_object(); + + for (epoch, credits) in credits_per_epoch { + // TODO: We could miss fees here + let js_credits = cx.number(credits as f64); + + js_epoch_index_map.set(&mut cx, epoch.to_string().as_str(), js_credits)?; + } + + let js_identity_to_epochs = cx.empty_object(); + + let js_identifier = JsBuffer::external(&mut cx, identifier); + + js_identity_to_epochs.set(&mut cx, "identifier", js_identifier)?; + js_identity_to_epochs.set(&mut cx, "creditsPerEpoch", js_epoch_index_map)?; + + js_fee_refunds.set(&mut cx, index as u32, js_identity_to_epochs)?; + } + + Ok(js_fee_refunds) + } } impl Finalize for FeeResultWrapper {} diff --git a/packages/rs-drive-nodejs/src/lib.rs b/packages/rs-drive-nodejs/src/lib.rs index fd9942d4e7e..68715405819 100644 --- a/packages/rs-drive-nodejs/src/lib.rs +++ b/packages/rs-drive-nodejs/src/lib.rs @@ -2349,6 +2349,7 @@ fn main(mut cx: ModuleContext) -> NeonResult<()> { cx.export_function("feeResultAdd", FeeResultWrapper::add)?; cx.export_function("feeResultAddFees", FeeResultWrapper::add_fees)?; cx.export_function("feeResultCreate", FeeResultWrapper::create)?; + cx.export_function("feeResultGetRefunds", FeeResultWrapper::get_fee_refunds)?; Ok(()) } diff --git a/packages/rs-drive/src/drive/document/delete.rs b/packages/rs-drive/src/drive/document/delete.rs index 2ac61ce3c83..da835204220 100644 --- a/packages/rs-drive/src/drive/document/delete.rs +++ b/packages/rs-drive/src/drive/document/delete.rs @@ -1405,7 +1405,7 @@ mod tests { .expect("expected to be able to delete the document"); let removed_bytes = fee_result - .removed_bytes_from_epochs_by_identities + .fee_refunds .get(&random_owner_id) .unwrap() .get(0) @@ -1476,10 +1476,7 @@ mod tests { ) .expect("expected to be able to delete the document"); - assert!(fee_result - .removed_bytes_from_epochs_by_identities - .0 - .is_empty()); + assert!(fee_result.fee_refunds.0.is_empty()); assert_eq!(fee_result.storage_fee, 0); assert_eq!(fee_result.processing_fee, 144746400); } diff --git a/packages/rs-drive/src/drive/document/insert.rs b/packages/rs-drive/src/drive/document/insert.rs index b4cc317bd49..9c19701af86 100644 --- a/packages/rs-drive/src/drive/document/insert.rs +++ b/packages/rs-drive/src/drive/document/insert.rs @@ -1176,7 +1176,7 @@ mod tests { let FeeResult { storage_fee, processing_fee, - removed_bytes_from_epochs_by_identities: _, + fee_refunds: _, removed_bytes_from_system: _, } = drive .add_serialized_document_for_contract( @@ -1225,7 +1225,7 @@ mod tests { let FeeResult { storage_fee, processing_fee, - removed_bytes_from_epochs_by_identities: _, + fee_refunds: _, removed_bytes_from_system: _, } = drive .add_serialized_document_for_contract( @@ -1274,7 +1274,7 @@ mod tests { let FeeResult { storage_fee, processing_fee, - removed_bytes_from_epochs_by_identities: _, + fee_refunds: _, removed_bytes_from_system: _, } = drive .add_serialized_document_for_contract( @@ -1480,7 +1480,7 @@ mod tests { let FeeResult { storage_fee, processing_fee, - removed_bytes_from_epochs_by_identities: _, + fee_refunds: _, removed_bytes_from_system: _, } = drive .add_document_for_contract( diff --git a/packages/rs-drive/src/drive/document/update.rs b/packages/rs-drive/src/drive/document/update.rs index ef55916f2f5..20972f569ea 100644 --- a/packages/rs-drive/src/drive/document/update.rs +++ b/packages/rs-drive/src/drive/document/update.rs @@ -1523,7 +1523,7 @@ mod tests { transaction.as_ref(), ); let removed_bytes = deletion_fees - .removed_bytes_from_epochs_by_identities + .fee_refunds .get(&owner_id) .unwrap() .get(0) @@ -1635,7 +1635,7 @@ mod tests { transaction.as_ref(), ); let removed_bytes = deletion_fees - .removed_bytes_from_epochs_by_identities + .fee_refunds .get(&owner_id) .unwrap() .get(0) @@ -1667,7 +1667,7 @@ mod tests { // this is because trees are added because of indexes, and also removed let added_bytes = update_fees.storage_fee / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; let removed_bytes = update_fees - .removed_bytes_from_epochs_by_identities + .fee_refunds .get(&owner_id) .unwrap() .get(0) diff --git a/packages/rs-drive/src/fee/mod.rs b/packages/rs-drive/src/fee/mod.rs index 51f03365d93..12c83e960e5 100644 --- a/packages/rs-drive/src/fee/mod.rs +++ b/packages/rs-drive/src/fee/mod.rs @@ -32,13 +32,13 @@ use enum_map::EnumMap; use crate::error::fee::FeeError; use crate::error::Error; use crate::fee::op::{BaseOp, DriveOperation}; -use crate::fee::removed_bytes_from_epochs_by_identities::RemovedBytesFromEpochsByIdentities; +use crate::fee::refunds::FeeRefunds; use crate::fee_pools::epochs::Epoch; /// Default costs module pub mod default_costs; pub mod op; -mod removed_bytes_from_epochs_by_identities; +mod refunds; /// Fee Result #[derive(Debug, Clone, Eq, PartialEq, Default)] @@ -47,8 +47,8 @@ pub struct FeeResult { pub storage_fee: u64, /// Processing fee pub processing_fee: u64, - /// Removed bytes from identities - pub removed_bytes_from_epochs_by_identities: RemovedBytesFromEpochsByIdentities, + /// Credits to refund to identities + pub fee_refunds: FeeRefunds, /// Removed bytes not needing to be refunded to identities pub removed_bytes_from_system: u32, } @@ -104,8 +104,7 @@ impl FeeResult { .ok_or(Error::Fee(FeeError::Overflow( "processing fee overflow error", )))?; - self.removed_bytes_from_epochs_by_identities - .checked_add_assign(rhs.removed_bytes_from_epochs_by_identities)?; + self.fee_refunds.checked_add_assign(rhs.fee_refunds)?; self.removed_bytes_from_system = self .removed_bytes_from_system .checked_add(rhs.removed_bytes_from_system) diff --git a/packages/rs-drive/src/fee/op.rs b/packages/rs-drive/src/fee/op.rs index 378c5697a7f..79722e47950 100644 --- a/packages/rs-drive/src/fee/op.rs +++ b/packages/rs-drive/src/fee/op.rs @@ -41,7 +41,9 @@ use enum_map::Enum; use grovedb::batch::key_info::KeyInfo; use grovedb::batch::KeyInfoPath; use grovedb::{batch::GroveDbOp, Element, PathQuery}; +use intmap::IntMap; use std::collections::BTreeMap; +use std::process::id; use crate::drive::flags::StorageFlags; use crate::error::drive::DriveError; @@ -54,7 +56,7 @@ use crate::fee::default_costs::{ use crate::fee::op::DriveOperation::{ CalculatedCostOperation, GroveOperation, PreCalculatedFeeResult, }; -use crate::fee::removed_bytes_from_epochs_by_identities::RemovedBytesFromEpochsByIdentities; +use crate::fee::refunds::FeeRefunds; use crate::fee::FeeResult; use crate::fee_pools::epochs::Epoch; @@ -398,26 +400,28 @@ impl DriveOperation { let cost = operation.operation_cost()?; let storage_fee = cost.storage_cost(epoch)?; let processing_fee = cost.ephemeral_cost(epoch)?; - let (removed_bytes_from_identities, removed_bytes_from_system) = + let (removed_bytes_from_epochs_by_identities, removed_bytes_from_system) = match cost.storage_cost.removed_bytes { - NoStorageRemoval => (BTreeMap::default(), 0), + NoStorageRemoval => (FeeRefunds::default(), 0), BasicStorageRemoval(amount) => { // this is not always considered an error - (BTreeMap::default(), amount) + (FeeRefunds::default(), amount) } - SectionedStorageRemoval(mut s) => { - let system_amount = s + SectionedStorageRemoval(mut removalPerEpochByIdentifier) => { + let system_amount = removalPerEpochByIdentifier .remove(&Identifier::default()) .map_or(0, |a| a.values().sum()); - (s, system_amount) + + ( + FeeRefunds::from_storage_removal(removalPerEpochByIdentifier)?, + system_amount, + ) } }; Ok(FeeResult { storage_fee, processing_fee, - removed_bytes_from_epochs_by_identities: RemovedBytesFromEpochsByIdentities( - removed_bytes_from_identities, - ), + fee_refunds: removed_bytes_from_epochs_by_identities, removed_bytes_from_system, }) } @@ -516,7 +520,7 @@ pub trait DriveCost { fn storage_cost(&self, epoch: &Epoch) -> Result; } -fn get_overflow_error(str: &'static str) -> Error { +pub(crate) fn get_overflow_error(str: &'static str) -> Error { Error::Fee(FeeError::Overflow(str)) } diff --git a/packages/rs-drive/src/fee/removed_bytes_from_epochs_by_identities.rs b/packages/rs-drive/src/fee/refunds.rs similarity index 62% rename from packages/rs-drive/src/fee/removed_bytes_from_epochs_by_identities.rs rename to packages/rs-drive/src/fee/refunds.rs index 3fd61aca621..63f68d94233 100644 --- a/packages/rs-drive/src/fee/removed_bytes_from_epochs_by_identities.rs +++ b/packages/rs-drive/src/fee/refunds.rs @@ -1,17 +1,46 @@ use crate::error::fee::FeeError; use crate::error::Error; +use crate::fee::default_costs::STORAGE_DISK_USAGE_CREDIT_PER_BYTE; +use crate::fee::op::get_overflow_error; use bincode::Options; -use costs::storage_cost::removal::Identifier; +use costs::storage_cost::removal::{Identifier, StorageRemovalPerEpochByIdentifier}; use intmap::IntMap; use serde::{Deserialize, Serialize}; use std::collections::btree_map::{IntoIter, Iter}; use std::collections::BTreeMap; -/// Fee Result +type CreditsPerEpoch = IntMap; +type CreditsPerEpochByIdentifier = BTreeMap; + +/// Fee refunds to identities based on removed data from specific epochs #[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)] -pub struct RemovedBytesFromEpochsByIdentities(pub BTreeMap>); +pub struct FeeRefunds(pub CreditsPerEpochByIdentifier); + +impl FeeRefunds { + pub fn from_storage_removal( + storage_removal: StorageRemovalPerEpochByIdentifier, + ) -> Result { + let refunds_per_epoch_by_identifier = storage_removal + .into_iter() + .map(|(identifier, bytes_per_epochs)| { + bytes_per_epochs + .into_iter() + .map(|(epoch_index, bytes)| { + (bytes as u64) + .checked_mul(STORAGE_DISK_USAGE_CREDIT_PER_BYTE) + .ok_or_else(|| { + get_overflow_error("storage written bytes cost overflow") + }) + .map(|credits| (epoch_index, credits)) + }) + .collect::>() + .map(|credits_per_epochs| (identifier, credits_per_epochs)) + }) + .collect::>()?; + + Ok(Self(refunds_per_epoch_by_identifier)) + } -impl RemovedBytesFromEpochsByIdentities { /// Adds and self assigns result between two Fee Results pub fn checked_add_assign(&mut self, rhs: Self) -> Result<(), Error> { for (identifier, mut int_map_b) in rhs.0.into_iter() { @@ -28,7 +57,7 @@ impl RemovedBytesFromEpochsByIdentities { }; combined.map(|c| (k, c)) }) - .collect::, Error>>()?; + .collect::>()?; intersection.into_iter().chain(int_map_b).collect() } else { int_map_b @@ -40,17 +69,17 @@ impl RemovedBytesFromEpochsByIdentities { } /// Passthrough method for get - pub fn get(&self, key: &Identifier) -> Option<&IntMap> { + pub fn get(&self, key: &Identifier) -> Option<&CreditsPerEpoch> { self.0.get(key) } /// Passthrough method for iteration - pub fn iter(&self) -> Iter> { + pub fn iter(&self) -> Iter { self.0.iter() } /// Passthrough method for into interation - pub fn into_iter(self) -> IntoIter> { + pub fn into_iter(self) -> IntoIter { self.0.into_iter() } @@ -80,7 +109,7 @@ impl RemovedBytesFromEpochsByIdentities { } pub fn deserialize(bytes: &[u8]) -> Result { - Ok(RemovedBytesFromEpochsByIdentities( + Ok(FeeRefunds( bincode::DefaultOptions::default() .with_varint_encoding() .reject_trailing_bytes() From 3ebd39c7e0b601aec3b8a2187c9000def2f08436 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Tue, 13 Dec 2022 19:49:58 +0800 Subject: [PATCH 02/54] feat: pass refunds to RS Drive --- Cargo.lock | 28 +++++---- .../handlers/proposal/deliverTxFactory.js | 7 +-- .../abci/handlers/proposal/endBlockFactory.js | 15 ++--- .../proposal/fees/addToFeeTxResults.js | 2 +- .../test/integration/fee/pools.spec.js | 10 +++- .../prepareProposalHandlerFactory.spec.js | 25 +++++--- .../proposal/deliverTxFactory.spec.js | 39 ++++++++++-- .../handlers/proposal/endBlockFactory.spec.js | 16 ++--- .../proposal/processProposalFactory.spec.js | 29 ++++++--- packages/rs-drive-abci/src/abci/handlers.rs | 9 ++- packages/rs-drive-abci/src/abci/messages.rs | 47 +++++++++++---- .../rs-drive-abci/src/error/serialization.rs | 4 +- .../execution/fee_pools/fee_distribution.rs | 8 ++- .../execution/fee_pools/process_block_fees.rs | 9 ++- packages/rs-drive-nodejs/Cargo.toml | 1 + packages/rs-drive-nodejs/Drive.js | 20 +++---- packages/rs-drive-nodejs/src/converter.rs | 1 + packages/rs-drive-nodejs/src/lib.rs | 59 ++++++++++++++----- packages/rs-drive-nodejs/test/Drive.spec.js | 15 ++++- packages/rs-drive/src/fee/mod.rs | 4 +- packages/rs-drive/src/fee/refunds.rs | 10 +++- 21 files changed, 247 insertions(+), 111 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c829e243f3..25cbbe9c1f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -135,6 +135,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" + [[package]] name = "bech32" version = "0.8.1" @@ -856,7 +862,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.13.1", "bls-signatures", "bs58", "byteorder", @@ -894,7 +900,7 @@ name = "drive" version = "0.24.0-dev.1" dependencies = [ "array_tool", - "base64", + "base64 0.13.1", "bincode", "bs58", "byteorder", @@ -926,7 +932,7 @@ dependencies = [ name = "drive-abci" version = "0.1.0" dependencies = [ - "base64", + "base64 0.13.1", "bs58", "chrono", "ciborium", @@ -948,6 +954,7 @@ version = "0.24.0-dev.1" dependencies = [ "drive", "drive-abci", + "intmap", "neon", "num 0.4.0", ] @@ -1530,7 +1537,7 @@ source = "git+https://github.com/qrayven/jsonschema-rs?branch=feat-unknown-forma dependencies = [ "ahash 0.7.6", "anyhow", - "base64", + "base64 0.20.0", "bytecount", "fancy-regex", "fraction", @@ -2441,11 +2448,10 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e060280438193c554f654141c9ea9417886713b7acd75974c85b18a69a88e0b" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" dependencies = [ - "crossbeam-deque", "either", "rayon-core", ] @@ -2698,9 +2704,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" +checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91" dependencies = [ "serde_derive", ] @@ -2737,9 +2743,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4" +checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e" dependencies = [ "proc-macro2", "quote", diff --git a/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js b/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js index 282fe5f137a..b0951b48c1f 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js @@ -42,12 +42,7 @@ function deliverTxFactory( * @param {BaseLogger} consensusLogger * @return {Promise<{ * code: number, - * fees: { - * storageFee: number, - * processingFee: number, - * feeRefunds: Object, - * feeRefundsSum: number, - * }}>} + * fees: BlockFeeResult}>} */ async function deliverTx(stateTransitionByteArray, round, consensusLogger) { const blockHeight = proposalBlockExecutionContext.getHeight(); diff --git a/packages/js-drive/lib/abci/handlers/proposal/endBlockFactory.js b/packages/js-drive/lib/abci/handlers/proposal/endBlockFactory.js index c529c302045..a121a3d0536 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/endBlockFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/endBlockFactory.js @@ -51,19 +51,14 @@ function endBlockFactory( const { height, round, - fees: { - feeRefunds, - processingFee, - storageFee, - feeRefundsSum, - }, + fees, coreChainLockedHeight, } = request; // Call RS ABCI const rsRequest = { - feeRefunds, + fees, }; consensusLogger.debug(rsRequest, 'Request RS Drive\'s BlockEnd method'); @@ -74,6 +69,12 @@ function endBlockFactory( const { currentEpochIndex } = proposalBlockExecutionContext.getEpochInfo(); + const { + processingFee, + storageFee, + feeRefundsSum, + } = fees; + if (processingFee > 0 || storageFee > 0) { consensusLogger.debug({ currentEpochIndex, diff --git a/packages/js-drive/lib/abci/handlers/proposal/fees/addToFeeTxResults.js b/packages/js-drive/lib/abci/handlers/proposal/fees/addToFeeTxResults.js index 50c3307d034..2798f63cfd5 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/fees/addToFeeTxResults.js +++ b/packages/js-drive/lib/abci/handlers/proposal/fees/addToFeeTxResults.js @@ -18,7 +18,7 @@ function addToFeeTxResults(feeResults, feeResult) { feeResults.processingFee += feeResult.processingFee; feeResults.feeRefundsSum += feeResult.feeRefundsSum; - for (const [epochIndex, credits] of feeResult.feeRefunds.entries()) { + for (const [epochIndex, credits] of Object.entries(feeResult.feeRefunds)) { if (!feeResults.feeRefunds[epochIndex]) { feeResults.feeRefunds[epochIndex] = 0; } diff --git a/packages/js-drive/test/integration/fee/pools.spec.js b/packages/js-drive/test/integration/fee/pools.spec.js index 11e2e060272..7d7ce9ab004 100644 --- a/packages/js-drive/test/integration/fee/pools.spec.js +++ b/packages/js-drive/test/integration/fee/pools.spec.js @@ -8,7 +8,6 @@ const getMasternodeRewardShareDocumentsFixture = require('@dashevo/dpp/lib/test/ const getIdentityFixture = require('@dashevo/dpp/lib/test/fixtures/getIdentityFixture'); const Identifier = require('@dashevo/dpp/lib/identifier/Identifier'); -const FeeResult = require('@dashevo/rs-drive/FeeResult'); const generateRandomIdentifier = require('@dashevo/dpp/lib/test/utils/generateRandomIdentifier'); const createTestDIContainer = require('../../../lib/test/createTestDIContainer'); const BlockInfo = require('../../../lib/blockExecution/BlockInfo'); @@ -104,7 +103,14 @@ describe('Fee Pools', () => { await rsDrive.getAbci().blockBegin(blockBeginRequest); const blockEndRequest = { - fees: FeeResult.create(10000, 1000), + fees: { + processingFee: 1000, + storageFee: 10000 + 15, + feeRefunds: { + 1: 15, + }, + feeRefundsSum: 15, + }, }; await rsDrive.getAbci().blockEnd(blockEndRequest); diff --git a/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js index 27d70c5e9fd..c52e03cb2b0 100644 --- a/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js @@ -13,8 +13,6 @@ const { const Long = require('long'); -const FeeResult = require('@dashevo/rs-drive/FeeResult'); - const prepareProposalHandlerFactory = require('../../../../lib/abci/handlers/prepareProposalHandlerFactory'); const LoggerMock = require('../../../../lib/test/mock/LoggerMock'); const BlockExecutionContextMock = require('../../../../lib/test/mock/BlockExecutionContextMock'); @@ -76,7 +74,14 @@ describe('prepareProposalHandlerFactory', () => { deliverTxMock = this.sinon.stub().resolves({ code: 0, - fees: FeeResult.create(1, 2), + fees: { + processingFee: 10, + storageFee: 100, + feeRefunds: { + 1: 15, + }, + feeRefundsSum: 15, + }, }); endBlockMock = this.sinon.stub().resolves( @@ -174,17 +179,19 @@ describe('prepareProposalHandlerFactory', () => { { height: request.height, round, - fees: FeeResult.create(), + fees: { + processingFee: 10 * 3, + storageFee: 100 * 3, + feeRefunds: { + 1: 15 * 3, + }, + feeRefundsSum: 15 * 3, + }, coreChainLockedHeight: request.coreChainLockedHeight, }, loggerMock, ); - const { fees } = endBlockMock.getCall(0).args[0]; - - expect(fees.storageFee).to.equal(3); - expect(fees.processingFee).to.equal(6); - expect(proposalBlockExecutionContextMock.setPrepareProposalResult).to.be.calledOnceWithExactly({ appHash, txResults: new Array(3).fill({ code: 0 }), diff --git a/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js index 15f33d5d1ee..a925a9025d7 100644 --- a/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js @@ -9,14 +9,14 @@ const createStateRepositoryMock = require('@dashevo/dpp/lib/test/mocks/createSta const getDataContractFixture = require('@dashevo/dpp/lib/test/fixtures/getDataContractFixture'); const getDocumentFixture = require('@dashevo/dpp/lib/test/fixtures/getDocumentsFixture'); const GrpcErrorCodes = require('@dashevo/grpc-common/lib/server/error/GrpcErrorCodes'); +const StateTransitionExecutionContext = require('@dashevo/dpp/lib/stateTransition/StateTransitionExecutionContext'); + const SomeConsensusError = require('@dashevo/dpp/lib/test/mocks/SomeConsensusError'); -const FeeResult = require('@dashevo/rs-drive/FeeResult'); const BlockExecutionContextMock = require('../../../../../lib/test/mock/BlockExecutionContextMock'); - const deliverTxFactory = require('../../../../../lib/abci/handlers/proposal/deliverTxFactory'); -const LoggerMock = require('../../../../../lib/test/mock/LoggerMock'); +const LoggerMock = require('../../../../../lib/test/mock/LoggerMock'); const DPPValidationAbciError = require('../../../../../lib/abci/errors/DPPValidationAbciError'); const InvalidArgumentAbciError = require('../../../../../lib/abci/errors/InvalidArgumentAbciError'); const PredictedFeeLowerThanActualError = require('../../../../../lib/abci/handlers/errors/PredictedFeeLowerThanActualError'); @@ -38,6 +38,7 @@ describe('deliverTxFactory', () => { let loggerMock; let round; let proposalBlockExecutionContextMock; + let stateTransitionExecutionContextMock; beforeEach(async function beforeEach() { round = 42; @@ -49,13 +50,27 @@ describe('deliverTxFactory', () => { dpp = new DashPlatformProtocol(); await dpp.initialize(); + stateTransitionExecutionContextMock = new StateTransitionExecutionContext(); + + stateTransitionExecutionContextMock.setLastCalculatedFeeDetails({ + storageFee: 100, + processingFee: 10, + feeRefunds: [{ identifier: Buffer.alloc(32), creditsPerEpoch: { 1: 15 } }], + feeRefundsSum: 15, + total: 95, + }); + documentsBatchTransitionFixture = dpp.document.createStateTransition({ create: documentFixture, }); + documentsBatchTransitionFixture.setExecutionContext(stateTransitionExecutionContextMock); + dataContractCreateTransitionFixture = dpp .dataContract.createDataContractCreateTransition(dataContractFixture); + dataContractCreateTransitionFixture.setExecutionContext(stateTransitionExecutionContextMock); + documentTx = documentsBatchTransitionFixture.toBuffer(); dataContractTx = dataContractCreateTransitionFixture.toBuffer(); @@ -105,7 +120,14 @@ describe('deliverTxFactory', () => { expect(response).to.deep.equal({ code: 0, - fees: FeeResult.create(), + fees: { + processingFee: 10, + storageFee: 100, + feeRefunds: { + 1: 15, + }, + feeRefundsSum: 15, + }, }); expect(unserializeStateTransitionMock).to.be.calledOnceWith( @@ -139,7 +161,14 @@ describe('deliverTxFactory', () => { expect(response).to.deep.equal({ code: 0, - fees: FeeResult.create(), + fees: { + processingFee: 10, + storageFee: 100, + feeRefunds: { + 1: 15, + }, + feeRefundsSum: 15, + }, }); expect(unserializeStateTransitionMock).to.be.calledOnceWith( diff --git a/packages/js-drive/test/unit/abci/handlers/proposal/endBlockFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/proposal/endBlockFactory.spec.js index 5137f715499..cbb0ca7a2cc 100644 --- a/packages/js-drive/test/unit/abci/handlers/proposal/endBlockFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/proposal/endBlockFactory.spec.js @@ -1,7 +1,5 @@ const Long = require('long'); -const FeeResult = require('@dashevo/rs-drive/FeeResult'); - const endBlockFactory = require('../../../../../lib/abci/handlers/proposal/endBlockFactory'); const BlockExecutionContextMock = require('../../../../../lib/test/mock/BlockExecutionContextMock'); const LoggerMock = require('../../../../../lib/test/mock/LoggerMock'); @@ -34,7 +32,14 @@ describe('endBlockFactory', () => { round = 42; coreChainLockedHeight = 41; time = Date.now(); - fees = FeeResult.create(1, 2); + fees = { + processingFee: 10, + storageFee: 100, + feeRefunds: { + 1: 15, + }, + feeRefundsSum: 15, + }; executionTimerMock = { clearTimer: this.sinon.stub(), @@ -124,11 +129,6 @@ describe('endBlockFactory', () => { expect(rsAbciMock.blockEnd).to.be.calledOnceWithExactly({ fees }, true); - const { fees: actualFees } = rsAbciMock.blockEnd.getCall(0).args[0]; - - expect(actualFees.storageFee).to.equal(1); - expect(actualFees.processingFee).to.equal(2); - expect(executionTimerMock.stopTimer).to.be.calledOnceWithExactly('roundExecution', true); }); }); diff --git a/packages/js-drive/test/unit/abci/handlers/proposal/processProposalFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/proposal/processProposalFactory.spec.js index 2f83c429b3d..cda64a71068 100644 --- a/packages/js-drive/test/unit/abci/handlers/proposal/processProposalFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/proposal/processProposalFactory.spec.js @@ -12,8 +12,6 @@ const { const Long = require('long'); -const FeeResult = require('@dashevo/rs-drive/FeeResult'); - const processProposalHandlerFactory = require('../../../../../lib/abci/handlers/proposal/processProposalFactory'); const LoggerMock = require('../../../../../lib/test/mock/LoggerMock'); const BlockExecutionContextMock = require('../../../../../lib/test/mock/BlockExecutionContextMock'); @@ -55,18 +53,29 @@ describe('processProposalFactory', () => { appVersion: 1, }, }); + validatorSetUpdate = new ValidatorSetUpdate(); beginBlockMock = this.sinon.stub(); + endBlockMock = this.sinon.stub().resolves({ consensusParamUpdates, appHash, validatorSetUpdate, }); + deliverTxMock = this.sinon.stub().resolves({ code: 0, - fees: FeeResult.create(1, 2), + fees: { + processingFee: 10, + storageFee: 100, + feeRefunds: { + 1: 15, + }, + feeRefundsSum: 15, + }, }); + beginBlockMock = this.sinon.stub(); executionTimerMock = { @@ -142,14 +151,16 @@ describe('processProposalFactory', () => { expect(endBlockMock).to.be.calledOnceWithExactly({ height: request.height, round, - fees: FeeResult.create(), + fees: { + processingFee: 10 * 3, + storageFee: 100 * 3, + feeRefunds: { + 1: 15 * 3, + }, + feeRefundsSum: 15 * 3, + }, coreChainLockedHeight: request.coreChainLockedHeight, }, loggerMock); - - const { fees } = endBlockMock.getCall(0).args[0]; - - expect(fees.storageFee).to.equal(3); - expect(fees.processingFee).to.equal(6); }); }); diff --git a/packages/rs-drive-abci/src/abci/handlers.rs b/packages/rs-drive-abci/src/abci/handlers.rs index 9f33c75c839..d5db4fe0a29 100644 --- a/packages/rs-drive-abci/src/abci/handlers.rs +++ b/packages/rs-drive-abci/src/abci/handlers.rs @@ -173,6 +173,8 @@ impl TenderdashAbci for Platform { drive_cache.cached_contracts.clear_block_cache(); + // Super slow operation of preparing chunks + Ok(AfterFinalizeBlockResponse {}) } } @@ -190,7 +192,8 @@ mod tests { use std::ops::Div; use crate::abci::messages::{ - AfterFinalizeBlockRequest, BlockBeginRequest, BlockEndRequest, InitChainRequest, + AfterFinalizeBlockRequest, BlockBeginRequest, BlockEndRequest, BlockFeeResult, + InitChainRequest, }; use crate::common::helpers::setup::setup_platform; @@ -357,7 +360,7 @@ mod tests { } let block_end_request = BlockEndRequest { - fees: FeeResult::from_fees(storage_fees_per_block, 1600), + fees: BlockFeeResult::from_fees(storage_fees_per_block, 1600), }; let block_end_response = platform @@ -509,7 +512,7 @@ mod tests { ); let block_end_request = BlockEndRequest { - fees: FeeResult::from_fees(storage_fees_per_block, 1600), + fees: BlockFeeResult::from_fees(storage_fees_per_block, 1600), }; let block_end_response = platform diff --git a/packages/rs-drive-abci/src/abci/messages.rs b/packages/rs-drive-abci/src/abci/messages.rs index 9149a083b4c..1689c58f0b4 100644 --- a/packages/rs-drive-abci/src/abci/messages.rs +++ b/packages/rs-drive-abci/src/abci/messages.rs @@ -37,8 +37,9 @@ use crate::error::serialization::SerializationError; use crate::error::Error; use crate::execution::fee_pools::epoch::EpochInfo; use crate::execution::fee_pools::process_block_fees::ProcessedBlockFeesResult; -use drive::fee::FeeResult; -use serde::{Deserialize, Serialize}; +use drive::fee::refunds::CreditsPerEpoch; +use serde::{Deserialize, Deserializer, Serialize}; +use std::ops::Deref; /// A struct for handling chain initialization requests #[derive(Serialize, Deserialize)] @@ -82,8 +83,30 @@ pub struct BlockBeginResponse { pub struct BlockEndRequest { /// The fees for the block /// Avoid of serialization to optimize transfer through Node.JS binding - #[serde(skip)] - pub fees: FeeResult, + pub fees: BlockFeeResult, +} + +/// Aggregated fees after block execution +#[derive(Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct BlockFeeResult { + /// Processing fee + pub processing_fee: u64, + /// Storage fee + pub storage_fee: u64, + /// Fee refunds + pub fee_refunds: CreditsPerEpoch, +} + +impl BlockFeeResult { + /// Create block fee result from fees + pub fn from_fees(storage_fee: u64, processing_fee: u64) -> Self { + Self { + storage_fee, + processing_fee, + ..Default::default() + } + } } /// A struct for handling block end responses @@ -146,10 +169,10 @@ pub trait Serializable<'a>: Serialize + Deserialize<'a> { fn to_bytes(&self) -> Result, Error> { let mut bytes = vec![]; - ciborium::ser::into_writer(&self, &mut bytes).map_err(|_| { - Error::Serialization(SerializationError::CorruptedSerialization( - "can't serialize ABCI message", - )) + ciborium::ser::into_writer(&self, &mut bytes).map_err(|e| { + let message = format!("can't deserialize ABCI message: {}", e); + + Error::Serialization(SerializationError::CorruptedSerialization(message)) })?; Ok(bytes) @@ -157,10 +180,10 @@ pub trait Serializable<'a>: Serialize + Deserialize<'a> { /// Deserialize ABCI message fn from_bytes(bytes: &[u8]) -> Result { - ciborium::de::from_reader(bytes).map_err(|_| { - Error::Serialization(SerializationError::CorruptedDeserialization( - "can't deserialize ABCI message", - )) + ciborium::de::from_reader(bytes).map_err(|e| { + let message = format!("can't deserialize ABCI message: {}", e); + + Error::Serialization(SerializationError::CorruptedDeserialization(message)) }) } } diff --git a/packages/rs-drive-abci/src/error/serialization.rs b/packages/rs-drive-abci/src/error/serialization.rs index 5de531f0cca..af13f1f8f99 100644 --- a/packages/rs-drive-abci/src/error/serialization.rs +++ b/packages/rs-drive-abci/src/error/serialization.rs @@ -3,9 +3,9 @@ pub enum SerializationError { /// Error #[error("corrupted serialization error key: {0}")] - CorruptedSerialization(&'static str), + CorruptedSerialization(String), /// Error #[error("corrupted deserialization error key: {0}")] - CorruptedDeserialization(&'static str), + CorruptedDeserialization(String), } diff --git a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs index 847bf5b5f4c..fe544a24171 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs @@ -34,6 +34,7 @@ use crate::error::execution::ExecutionError; +use crate::abci::messages::BlockFeeResult; use crate::error::Error; use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; @@ -383,7 +384,7 @@ impl Platform { pub fn add_distribute_block_fees_into_pools_operations( &self, current_epoch: &Epoch, - block_fees: &FeeResult, + block_fees: &BlockFeeResult, cached_aggregated_storage_fees: Option, transaction: TransactionArg, batch: &mut GroveDbOpBatch, @@ -1303,6 +1304,7 @@ mod tests { } mod add_distribute_block_fees_into_pools_operations { + use crate::abci::messages::BlockFeeResult; use crate::common::helpers::setup::setup_platform_with_initial_state_structure; use drive::drive::batch::GroveDbOpBatch; use drive::fee::FeeResult; @@ -1325,7 +1327,7 @@ mod tests { platform .add_distribute_block_fees_into_pools_operations( ¤t_epoch_tree, - &FeeResult::from_fees(storage_fees, processing_fees), + &BlockFeeResult::from_fees(storage_fees, processing_fees), None, Some(&transaction), &mut batch, @@ -1379,7 +1381,7 @@ mod tests { platform .add_distribute_block_fees_into_pools_operations( ¤t_epoch_tree, - &FeeResult::from_fees(storage_fees, processing_fees), + &BlockFeeResult::from_fees(storage_fees, processing_fees), None, Some(&transaction), &mut batch, diff --git a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs index 1caed2510e9..ffc990dc3ae 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs @@ -33,6 +33,7 @@ //! epoch changes. //! +use crate::abci::messages::BlockFeeResult; use crate::block::BlockInfo; use crate::error::Error; use crate::execution::fee_pools::constants::DEFAULT_ORIGINAL_FEE_MULTIPLIER; @@ -125,7 +126,7 @@ impl Platform { &self, block_info: &BlockInfo, epoch_info: &EpochInfo, - block_fees: &FeeResult, + block_fees: &BlockFeeResult, transaction: TransactionArg, ) -> Result { let current_epoch = Epoch::new(epoch_info.current_epoch_index); @@ -203,6 +204,7 @@ mod tests { use rust_decimal::prelude::ToPrimitive; mod helpers { + use crate::abci::messages::BlockFeeResult; use crate::block::BlockInfo; use crate::execution::fee_pools::epoch::{EpochInfo, EPOCH_CHANGE_TIME_MS}; use crate::platform::Platform; @@ -226,7 +228,7 @@ mod tests { // Add some storage fees to distribute next time if should_distribute { - let block_fees = FeeResult::from_fees(1000000000, 1000); + let block_fees = BlockFeeResult::from_fees(1000000000, 1000); let mut batch = GroveDbOpBatch::new(); @@ -395,6 +397,7 @@ mod tests { use rust_decimal::prelude::ToPrimitive; mod helpers { + use crate::abci::messages::BlockFeeResult; use crate::block::BlockInfo; use crate::execution::fee_pools::epoch::{EpochInfo, EPOCH_CHANGE_TIME_MS}; use crate::platform::Platform; @@ -429,7 +432,7 @@ mod tests { EpochInfo::from_genesis_time_and_block_info(genesis_time_ms, &block_info) .expect("should calculate epoch info"); - let block_fees = FeeResult::from_fees(1000, 10000); + let block_fees = BlockFeeResult::from_fees(1000, 10000); let distribute_storage_pool_result = platform .process_block_fees(&block_info, &epoch_info, &block_fees, transaction) diff --git a/packages/rs-drive-nodejs/Cargo.toml b/packages/rs-drive-nodejs/Cargo.toml index b9a3a70ae5b..826c62d2653 100644 --- a/packages/rs-drive-nodejs/Cargo.toml +++ b/packages/rs-drive-nodejs/Cargo.toml @@ -13,6 +13,7 @@ crate-type = ["cdylib"] drive = { path = "../rs-drive" } drive-abci = { path = "../rs-drive-abci" } num = "0.4.0" +intmap = { version="2.0.0", features=["serde"] } [dependencies.neon] version = "0.10.1" diff --git a/packages/rs-drive-nodejs/Drive.js b/packages/rs-drive-nodejs/Drive.js index 5a1041ee21c..d612048bc90 100644 --- a/packages/rs-drive-nodejs/Drive.js +++ b/packages/rs-drive-nodejs/Drive.js @@ -430,17 +430,9 @@ class Drive { * @returns {Promise} */ async blockEnd(request, useTransaction = false) { - // Pass instance of FeeResult separately to avoid of unnecessary serialization - const feeResultInner = request.fees.inner; - - delete request.fees; - - const requestBytes = cbor.encode(request); - const responseBytes = await abciBlockEndAsync.call( drive, - feeResultInner, - requestBytes, + request, useTransaction, ); @@ -512,7 +504,15 @@ class Drive { /** * @typedef BlockEndRequest - * @property {FeeResult} fees + * @property {BlockFeeResult} fees + */ + +/** + * @typedef BlockFeeResult + * @property {number} storageFee + * @property {number} processingFee + * @property {Object} feeRefunds + * @property {number} feeRefundsSum */ /** diff --git a/packages/rs-drive-nodejs/src/converter.rs b/packages/rs-drive-nodejs/src/converter.rs index bab1e7f5f76..d55ab4395a3 100644 --- a/packages/rs-drive-nodejs/src/converter.rs +++ b/packages/rs-drive-nodejs/src/converter.rs @@ -4,6 +4,7 @@ use drive::fee::FeeResult; use drive::fee_pools::epochs::Epoch; use drive::grovedb::reference_path::ReferencePathType; use drive::grovedb::{Element, PathQuery, Query, SizedQuery}; +use intmap::IntMap; use neon::prelude::*; use neon::types::buffer::TypedArray; use num::FromPrimitive; diff --git a/packages/rs-drive-nodejs/src/lib.rs b/packages/rs-drive-nodejs/src/lib.rs index 68715405819..c99ddb03be7 100644 --- a/packages/rs-drive-nodejs/src/lib.rs +++ b/packages/rs-drive-nodejs/src/lib.rs @@ -2,6 +2,7 @@ mod converter; mod fee_result; use neon::object::PropertyKey; +use std::num::ParseIntError; use std::ops::Deref; use std::{option::Option::None, path::Path, sync::mpsc, thread}; @@ -11,12 +12,14 @@ use drive::drive::batch::GroveDbOpBatch; use drive::drive::config::DriveConfig; use drive::drive::flags::StorageFlags; use drive::error::Error; +use drive::fee::refunds::CreditsPerEpoch; use drive::fee_pools::epochs::Epoch; use drive::grovedb::{PathQuery, Transaction}; use drive::query::TransactionArg; use drive_abci::abci::handlers::TenderdashAbci; use drive_abci::abci::messages::{ - AfterFinalizeBlockRequest, BlockBeginRequest, BlockEndRequest, InitChainRequest, Serializable, + AfterFinalizeBlockRequest, BlockBeginRequest, BlockEndRequest, BlockFeeResult, + InitChainRequest, Serializable, }; use drive_abci::platform::Platform; use neon::prelude::*; @@ -2015,11 +2018,11 @@ impl PlatformWrapper { } fn js_abci_block_end(mut cx: FunctionContext) -> JsResult { - let js_fee_result = cx.argument::>(0)?; - let js_request = cx.argument::(1)?; - let js_using_transaction = cx.argument::(2)?; + let js_request = cx.argument::(0)?; - let js_callback = cx.argument::(3)?.root(&mut cx); + let js_using_transaction = cx.argument::(1)?; + + let js_callback = cx.argument::(2)?.root(&mut cx); let using_transaction = js_using_transaction.value(&mut cx); @@ -2027,9 +2030,34 @@ impl PlatformWrapper { .this() .downcast_or_throw::, _>(&mut cx)?; - let fee_result = js_fee_result.deref().deref().deref().clone(); + let js_fees: Handle = js_request.get(&mut cx, "fees")?; - let request_bytes = converter::js_buffer_to_vec_u8(js_request, &mut cx); + let js_processing_fee: Handle = js_fees.get(&mut cx, "processingFee")?; + let processing_fee = js_processing_fee.value(&mut cx) as u64; + + let js_storage_fee: Handle = js_fees.get(&mut cx, "storageFee")?; + let storage_fee = js_storage_fee.value(&mut cx) as u64; + + let js_fee_refunds: Handle = js_fees.get(&mut cx, "feeRefunds")?; + + let mut fee_refunds: CreditsPerEpoch = Default::default(); + + for js_epoch_index_value in js_fee_refunds + .get_own_property_names(&mut cx)? + .to_vec(&mut cx)? + { + let js_epoch_index = js_epoch_index_value.downcast_or_throw::(&mut cx)?; + + let epoch_index = js_epoch_index + .value(&mut cx) + .parse() + .or_else(|e: ParseIntError| cx.throw_error(e.to_string()))?; + + let js_credits: Handle = js_fee_refunds.get(&mut cx, js_epoch_index)?; + let credits = js_credits.value(&mut cx) as u64; + + fee_refunds.insert(epoch_index, credits); + } db.send_to_drive_thread(move |platform: &Platform, transaction, channel| { let transaction_result = if using_transaction { @@ -2043,15 +2071,16 @@ impl PlatformWrapper { }; let result = transaction_result.and_then(|transaction_arg| { - BlockEndRequest::from_bytes(&request_bytes) - .and_then(|request| { - let full_request = BlockEndRequest { - fees: fee_result, - ..request - }; + let request = BlockEndRequest { + fees: BlockFeeResult { + processing_fee, + storage_fee, + fee_refunds, + }, + }; - platform.block_end(full_request, transaction_arg) - }) + platform + .block_end(request, transaction_arg) .and_then(|response| response.to_bytes()) .map_err(|e| e.to_string()) }); diff --git a/packages/rs-drive-nodejs/test/Drive.spec.js b/packages/rs-drive-nodejs/test/Drive.spec.js index f00a4f58775..84c661fb7dc 100644 --- a/packages/rs-drive-nodejs/test/Drive.spec.js +++ b/packages/rs-drive-nodejs/test/Drive.spec.js @@ -565,7 +565,14 @@ describe('Drive', () => { it('should process a block', async () => { const request = { - fees: FeeResult.create(100, 10), + fees: { + storageFee: 100, + processingFee: 100, + feeRefunds: { + 1: 15, + 2: 16, + }, + }, }; const response = await drive.getAbci().blockEnd(request); @@ -590,7 +597,11 @@ describe('Drive', () => { validatorSetQuorumHash: Buffer.alloc(32, 2), }); await drive.getAbci().blockEnd({ - fees: FeeResult.create(100, 10), + fees: { + storageFee: 100, + processingFee: 100, + feeRefunds: {}, + }, }); }); diff --git a/packages/rs-drive/src/fee/mod.rs b/packages/rs-drive/src/fee/mod.rs index 12c83e960e5..ee4c09a0df2 100644 --- a/packages/rs-drive/src/fee/mod.rs +++ b/packages/rs-drive/src/fee/mod.rs @@ -38,7 +38,9 @@ use crate::fee_pools::epochs::Epoch; /// Default costs module pub mod default_costs; pub mod op; -mod refunds; + +/// Fee result refunds +pub mod refunds; /// Fee Result #[derive(Debug, Clone, Eq, PartialEq, Default)] diff --git a/packages/rs-drive/src/fee/refunds.rs b/packages/rs-drive/src/fee/refunds.rs index 63f68d94233..35ddf57c85f 100644 --- a/packages/rs-drive/src/fee/refunds.rs +++ b/packages/rs-drive/src/fee/refunds.rs @@ -9,14 +9,18 @@ use serde::{Deserialize, Serialize}; use std::collections::btree_map::{IntoIter, Iter}; use std::collections::BTreeMap; -type CreditsPerEpoch = IntMap; -type CreditsPerEpochByIdentifier = BTreeMap; +/// Credits per Epoch +pub type CreditsPerEpoch = IntMap; + +/// Credits per Epoch by Identifier +pub type CreditsPerEpochByIdentifier = BTreeMap; /// Fee refunds to identities based on removed data from specific epochs #[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)] pub struct FeeRefunds(pub CreditsPerEpochByIdentifier); impl FeeRefunds { + /// Create fee refunds from GroveDB's StorageRemovalPerEpochByIdentifier pub fn from_storage_removal( storage_removal: StorageRemovalPerEpochByIdentifier, ) -> Result { @@ -96,6 +100,7 @@ impl FeeRefunds { }) } + /// Returns serialized size pub fn serialized_size(&self) -> Result { bincode::DefaultOptions::default() .with_varint_encoding() @@ -108,6 +113,7 @@ impl FeeRefunds { }) } + /// Deserialized struct from bytes pub fn deserialize(bytes: &[u8]) -> Result { Ok(FeeRefunds( bincode::DefaultOptions::default() From c73581697e084c5b1737fd539cd7b43f261e274a Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 15 Dec 2022 00:01:49 +0800 Subject: [PATCH 03/54] chore: manage pending updates --- Cargo.lock | 1 - packages/js-drive/Dockerfile | 6 +- packages/rs-drive-abci/src/abci/handlers.rs | 2 +- .../execution/fee_pools/process_block_fees.rs | 28 +- packages/rs-drive-nodejs/Cargo.toml | 1 - packages/rs-drive-nodejs/src/converter.rs | 2 - packages/rs-drive-nodejs/test/Drive.spec.js | 4 + .../src/drive/batch/grovedb_op_batch.rs | 2 +- packages/rs-drive/src/drive/fee_pools/mod.rs | 13 +- .../src/drive/fee_pools/pending_updates.rs | 304 ++++++++++++++++++ .../epochs_root_tree_key_constants.rs | 2 + packages/rs-drive/src/fee_pools/mod.rs | 9 +- 12 files changed, 361 insertions(+), 13 deletions(-) create mode 100644 packages/rs-drive/src/drive/fee_pools/pending_updates.rs diff --git a/Cargo.lock b/Cargo.lock index 25cbbe9c1f9..1d9a7ecb4a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -954,7 +954,6 @@ version = "0.24.0-dev.1" dependencies = [ "drive", "drive-abci", - "intmap", "neon", "num 0.4.0", ] diff --git a/packages/js-drive/Dockerfile b/packages/js-drive/Dockerfile index 794e4696364..bb9ceaf3bcf 100644 --- a/packages/js-drive/Dockerfile +++ b/packages/js-drive/Dockerfile @@ -1,5 +1,5 @@ -# syntax = docker/dockerfile:1.3 -FROM node:16-alpine as builder +# syntax = docker/dockerfile:1.4 +FROM dashpay/node:16-alpine as builder ARG NODE_ENV=production ENV NODE_ENV ${NODE_ENV} @@ -88,7 +88,7 @@ RUN --mount=type=cache,target=/tmp/unplugged \ yarn workspaces focus --production @dashevo/drive && \ cp -R /platform/.yarn/unplugged /tmp/ -FROM node:16-alpine +FROM dashpay/node:16-alpine ARG NODE_ENV=production ENV NODE_ENV ${NODE_ENV} diff --git a/packages/rs-drive-abci/src/abci/handlers.rs b/packages/rs-drive-abci/src/abci/handlers.rs index d5db4fe0a29..cff05da4ecd 100644 --- a/packages/rs-drive-abci/src/abci/handlers.rs +++ b/packages/rs-drive-abci/src/abci/handlers.rs @@ -156,7 +156,7 @@ impl TenderdashAbci for Platform { let process_block_fees_result = self.process_block_fees( &block_execution_context.block_info, &block_execution_context.epoch_info, - &request.fees, + request.fees, transaction, )?; diff --git a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs index ffc990dc3ae..579c5622474 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs @@ -43,6 +43,7 @@ use crate::execution::fee_pools::fee_distribution::{FeesInPools, ProposersPayout use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; use drive::drive::fee_pools::epochs::constants::{GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; +use drive::drive::fee_pools::pending_updates::add_update_pending_epoch_pool_update_operations; use drive::fee::FeeResult; use drive::fee_pools::epochs::Epoch; use drive::grovedb::TransactionArg; @@ -79,6 +80,7 @@ impl Platform { &self, block_info: &BlockInfo, epoch_info: &EpochInfo, + block_fees: &BlockFeeResult, transaction: TransactionArg, batch: &mut GroveDbOpBatch, ) -> Result, Error> { @@ -107,6 +109,8 @@ impl Platform { return Ok(None); } + // TODO: Distribute pending updates + // Distribute storage fees accumulated during previous epoch let storage_distribution_leftover_credits = self .add_distribute_storage_fee_distribution_pool_to_epochs_operations( @@ -115,6 +119,13 @@ impl Platform { batch, )?; + self.drive + .add_delete_pending_epoch_pool_updates_except_specified_operations( + batch, + &block_fees.fee_refunds, + transaction, + )?; + Ok(Some(storage_distribution_leftover_credits)) } @@ -126,7 +137,7 @@ impl Platform { &self, block_info: &BlockInfo, epoch_info: &EpochInfo, - block_fees: &BlockFeeResult, + block_fees: BlockFeeResult, transaction: TransactionArg, ) -> Result { let current_epoch = Epoch::new(epoch_info.current_epoch_index); @@ -137,6 +148,7 @@ impl Platform { self.add_process_epoch_change_operations( block_info, epoch_info, + &block_fees, transaction, &mut batch, )? @@ -179,13 +191,25 @@ impl Platform { let fees_in_pools = self.add_distribute_block_fees_into_pools_operations( ¤t_epoch, - block_fees, + &block_fees, // Add leftovers after storage fee pool distribution to the current block storage fees storage_distribution_leftover_credits, transaction, &mut batch, )?; + let pending_epoch_pool_updates = if !epoch_info.is_epoch_change { + self.drive + .fetch_and_merge_with_existing_pending_epoch_pool_updates( + block_fees.fee_refunds, + transaction, + )? + } else { + block_fees.fee_refunds + }; + + add_update_pending_epoch_pool_update_operations(&mut batch, pending_epoch_pool_updates)?; + self.drive.grove_apply_batch(batch, false, transaction)?; Ok(ProcessedBlockFeesResult { diff --git a/packages/rs-drive-nodejs/Cargo.toml b/packages/rs-drive-nodejs/Cargo.toml index 826c62d2653..b9a3a70ae5b 100644 --- a/packages/rs-drive-nodejs/Cargo.toml +++ b/packages/rs-drive-nodejs/Cargo.toml @@ -13,7 +13,6 @@ crate-type = ["cdylib"] drive = { path = "../rs-drive" } drive-abci = { path = "../rs-drive-abci" } num = "0.4.0" -intmap = { version="2.0.0", features=["serde"] } [dependencies.neon] version = "0.10.1" diff --git a/packages/rs-drive-nodejs/src/converter.rs b/packages/rs-drive-nodejs/src/converter.rs index d55ab4395a3..6e9165c5362 100644 --- a/packages/rs-drive-nodejs/src/converter.rs +++ b/packages/rs-drive-nodejs/src/converter.rs @@ -1,10 +1,8 @@ use drive::drive::block_info::BlockInfo; use drive::drive::flags::StorageFlags; -use drive::fee::FeeResult; use drive::fee_pools::epochs::Epoch; use drive::grovedb::reference_path::ReferencePathType; use drive::grovedb::{Element, PathQuery, Query, SizedQuery}; -use intmap::IntMap; use neon::prelude::*; use neon::types::buffer::TypedArray; use num::FromPrimitive; diff --git a/packages/rs-drive-nodejs/test/Drive.spec.js b/packages/rs-drive-nodejs/test/Drive.spec.js index 84c661fb7dc..5610c0e623e 100644 --- a/packages/rs-drive-nodejs/test/Drive.spec.js +++ b/packages/rs-drive-nodejs/test/Drive.spec.js @@ -555,6 +555,7 @@ describe('Drive', () => { describe('BlockEnd', () => { beforeEach(async () => { await drive.getAbci().initChain({}); + await drive.getAbci().blockBegin({ blockHeight: 1, blockTimeMs: (new Date()).getTime(), @@ -587,15 +588,18 @@ describe('Drive', () => { this.timeout(10000); await drive.createInitialStateStructure(); + await drive.createContract(dataContract, blockInfo); await drive.getAbci().initChain({}); + await drive.getAbci().blockBegin({ blockHeight: 1, blockTimeMs: (new Date()).getTime(), proposerProTxHash: Buffer.alloc(32, 1), validatorSetQuorumHash: Buffer.alloc(32, 2), }); + await drive.getAbci().blockEnd({ fees: { storageFee: 100, diff --git a/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs b/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs index a68e8a1da92..9e09b4d1240 100644 --- a/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs +++ b/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs @@ -38,7 +38,7 @@ use grovedb::Element; /// A batch of GroveDB operations as a vector. // TODO move to GroveDB -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct GroveDbOpBatch { /// Operations pub(crate) operations: Vec, diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index ceda5a63682..e6b462f86f4 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -28,10 +28,13 @@ // use crate::drive::RootTree; -use crate::fee_pools::epochs_root_tree_key_constants::KEY_STORAGE_FEE_POOL; +use crate::fee_pools::epochs_root_tree_key_constants::{ + KEY_PENDING_POOL_UPDATES, KEY_STORAGE_FEE_POOL, +}; /// Epochs module pub mod epochs; +pub mod pending_updates; pub mod storage_fee_distribution_pool; pub mod unpaid_epoch; @@ -45,6 +48,14 @@ pub fn pools_vec_path() -> Vec> { vec![vec![RootTree::Pools as u8]] } +/// Returns the path to pending pool updates +pub fn pools_pending_updates_path() -> Vec> { + vec![ + vec![RootTree::Pools as u8], + KEY_PENDING_POOL_UPDATES.to_vec(), + ] +} + /// Returns the path to the aggregate storage fee distribution pool. pub fn aggregate_storage_fees_distribution_pool_path() -> [&'static [u8]; 2] { [ diff --git a/packages/rs-drive/src/drive/fee_pools/pending_updates.rs b/packages/rs-drive/src/drive/fee_pools/pending_updates.rs new file mode 100644 index 00000000000..e0eceb664c3 --- /dev/null +++ b/packages/rs-drive/src/drive/fee_pools/pending_updates.rs @@ -0,0 +1,304 @@ +// MIT LICENSE +// +// Copyright (c) 2021 Dash Core Group +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +//! Pending epoch pool updates +//! + +use crate::drive::batch::GroveDbOpBatch; +use crate::drive::fee_pools::pools_pending_updates_path; +use crate::drive::Drive; +use crate::error::drive::DriveError; +use crate::error::Error; +use crate::fee::refunds::CreditsPerEpoch; +use grovedb::query_result_type::QueryResultType; +use grovedb::{Element, PathQuery, Query, TransactionArg}; + +type PendingUpdatesCount = usize; + +impl Drive { + /// Fetches existing pending epoch pool updates using specified epochs + /// and returns merged result + pub fn fetch_and_merge_with_existing_pending_epoch_pool_updates( + &self, + mut credits_per_epoch: CreditsPerEpoch, + transaction: TransactionArg, + ) -> Result { + let mut query = Query::new(); + + for (epoch_index_key, _) in credits_per_epoch.iter() { + let epoch_index: u64 = epoch_index_key.to_owned(); + let encoded_epoch_index = epoch_index.to_be_bytes().to_vec(); + + query.insert_key(encoded_epoch_index); + } + + // Query existing pending updates + let (query_result, _) = self + .grove + .query_raw( + &PathQuery::new_unsized(pools_pending_updates_path(), query), + QueryResultType::QueryKeyElementPairResultType, + transaction, + ) + .unwrap() + .map_err(Error::GroveDB)?; + + // Merge with existing pending updates + for (encoded_epoch_index, element) in query_result.to_key_elements() { + let epoch_index = + u16::from_be_bytes(encoded_epoch_index.as_slice().try_into().map_err(|_| { + Error::Drive(DriveError::CorruptedSerialization( + "epoch index for pending pool updates must be u64", + )) + })?); + + let epoch_index_key = epoch_index as u64; + + let Some(credits_to_update) = credits_per_epoch.get(epoch_index_key) else { + return Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates should contain fetched epochs"))); + }; + + let Element::Item(encoded_existing_signed_credits, _) = element else { + return Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates should contain only items"))); + }; + + let existing_signed_credits = i64::from_be_bytes( + encoded_existing_signed_credits + .as_slice() + .try_into() + .map_err(|_| { + Error::Drive(DriveError::CorruptedSerialization( + "credits for pending pool updates must be i64", + )) + })?, + ); + + let existing_credits = existing_signed_credits as u64; + + credits_per_epoch.insert(epoch_index_key, credits_to_update + existing_credits); + } + + Ok(credits_per_epoch) + } + + /// Adds operations to delete pending epoch pool updates except specified epochs + pub fn add_delete_pending_epoch_pool_updates_except_specified_operations( + &self, + batch: &mut GroveDbOpBatch, + credits_per_epoch: &CreditsPerEpoch, + transaction: TransactionArg, + ) -> Result<(), Error> { + // TODO: Replace with key iterator + let mut query = Query::new(); + + query.insert_all(); + + let (query_result, _) = self + .grove + .query_raw( + &PathQuery::new_unsized(pools_pending_updates_path(), query), + QueryResultType::QueryKeyElementPairResultType, + transaction, + ) + .unwrap() + .map_err(Error::GroveDB)?; + + for (encoded_epoch_index, _) in query_result.to_key_elements() { + let epoch_index = + u16::from_be_bytes(encoded_epoch_index.as_slice().try_into().map_err(|_| { + Error::Drive(DriveError::CorruptedSerialization( + "epoch index for pending pool updates must be i64", + )) + })?); + + let epoch_index_key = epoch_index as u64; + + if credits_per_epoch.contains_key(epoch_index_key) { + continue; + } + + batch.add_delete(pools_pending_updates_path(), encoded_epoch_index); + } + + Ok(()) + } +} + +/// Returns the index of the unpaid Epoch. +pub fn add_update_pending_epoch_pool_update_operations( + batch: &mut GroveDbOpBatch, + credits_per_epoch: CreditsPerEpoch, +) -> Result { + let credits_per_epoch_count = credits_per_epoch.len(); + + for (epoch_index_key, credits) in credits_per_epoch { + let epoch_index = epoch_index_key as u16; + let encoded_epoch_index = epoch_index.to_be_bytes().to_vec(); + + let signed_credits = -(credits as i64); + + let element = Element::new_item(signed_credits.to_be_bytes().to_vec()); + + batch.add_insert(pools_pending_updates_path(), encoded_epoch_index, element); + } + + Ok(credits_per_epoch_count) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::common::helpers::setup::setup_drive_with_initial_state_structure; + + mod fetch_and_merge_with_existing_pending_epoch_pool_updates { + use super::*; + } + + mod add_delete_pending_epoch_pool_updates_except_specified_operations { + use super::*; + } + + mod add_update_pending_epoch_pool_update_operations { + use super::*; + + mod helpers { + use super::*; + use grovedb::batch::Op; + + pub fn process_and_assert_pending_updates( + drive: &Drive, + pending_updates: CreditsPerEpoch, + encoded_pending_updates: &HashMap, i64>, + transaction: TransactionArg, + ) { + let pending_updates_count = pending_updates.len(); + + let mut batch = GroveDbOpBatch::new(); + + let updates_count = drive + .add_update_pending_epoch_pool_update_operations( + &mut batch, + pending_updates, + transaction, + ) + .expect("should update pending pool updates"); + + drive + .grove_apply_batch(batch.clone(), false, transaction) + .expect("should apply batch"); + + assert_eq!(batch.len(), updates_count); + assert_eq!(batch.len(), pending_updates_count); + + for operation in batch.operations { + assert_eq!(operation.path.to_path(), pools_pending_updates_path()); + + let encoded_key = operation.key.get_key(); + + assert!(encoded_pending_updates.contains_key(&encoded_key)); + + let expected_credits = encoded_pending_updates[&encoded_key]; + + let expected_encoded_credits = expected_credits.to_be_bytes().to_vec(); + + let Op::Insert { + element: Element::Item(encoded_credits, None) + } = operation.op else { + panic!("pending pool update should be stored as an item"); + }; + + assert_eq!(encoded_credits, expected_encoded_credits); + } + } + } + + #[test] + fn should_add_new_pending_pool_updates() { + let drive = setup_drive_with_initial_state_structure(); + + let transaction = drive.grove.start_transaction(); + + // Add initial set of pending updates + + let initial_pending_updates = + CreditsPerEpoch::from_iter(vec![(1, 15), (3, 25), (7, 95), (9, 100), (12, 120)]); + + let initial_encoded_epoch_indices_and_signed_credits = initial_pending_updates + .clone() + .into_iter() + .map(|(epoch_index, credits)| { + let epoch_index_key = epoch_index as u16; + let epoch_index_key = epoch_index_key.to_be_bytes().to_vec(); + + // Credits must be stored as negative to support total credits balance + // verification with sum trees + (epoch_index_key, -(credits as i64)) + }) + .collect::>(); + + helpers::process_and_assert_pending_updates( + &drive, + initial_pending_updates, + &initial_encoded_epoch_indices_and_signed_credits, + Some(&transaction), + ); + + // Add new and update existing + + let additional_pending_updates = + CreditsPerEpoch::from_iter(vec![(1, 15), (3, 25), (15, 95), (16, 100)]); + + let additional_encoded_epoch_indices_and_signed_credits = additional_pending_updates + .clone() + .into_iter() + .map(|(epoch_index, credits)| { + let epoch_index_key = epoch_index as u16; + let epoch_index_key = epoch_index_key.to_be_bytes().to_vec(); + let mut signed_credits = -(credits as i64); + + if let Some(initial_signed_credits) = + initial_encoded_epoch_indices_and_signed_credits.get(&epoch_index_key) + { + signed_credits += initial_signed_credits; + } + + // Credits must be stored as negative to support total credits balance + // verification with sum trees + (epoch_index_key, signed_credits) + }) + .collect::>(); + + helpers::process_and_assert_pending_updates( + &drive, + additional_pending_updates, + &additional_encoded_epoch_indices_and_signed_credits, + Some(&transaction), + ); + } + } +} diff --git a/packages/rs-drive/src/fee_pools/epochs_root_tree_key_constants.rs b/packages/rs-drive/src/fee_pools/epochs_root_tree_key_constants.rs index f13c725983f..1d68e7d80b3 100644 --- a/packages/rs-drive/src/fee_pools/epochs_root_tree_key_constants.rs +++ b/packages/rs-drive/src/fee_pools/epochs_root_tree_key_constants.rs @@ -2,3 +2,5 @@ pub const KEY_STORAGE_FEE_POOL: &[u8; 1] = b"s"; /// Unpaid epoch index key pub const KEY_UNPAID_EPOCH_INDEX: &[u8; 1] = b"u"; +/// Pending updates for epoch pool storage fees +pub const KEY_PENDING_POOL_UPDATES: &[u8; 1] = b"p"; diff --git a/packages/rs-drive/src/fee_pools/mod.rs b/packages/rs-drive/src/fee_pools/mod.rs index e10535e840d..a86b87b9fc3 100644 --- a/packages/rs-drive/src/fee_pools/mod.rs +++ b/packages/rs-drive/src/fee_pools/mod.rs @@ -32,7 +32,7 @@ use crate::drive::fee_pools::epochs::constants::{GENESIS_EPOCH_INDEX, PERPETUAL_ use crate::drive::fee_pools::pools_vec_path; use crate::fee_pools::epochs::Epoch; use crate::fee_pools::epochs_root_tree_key_constants::{ - KEY_STORAGE_FEE_POOL, KEY_UNPAID_EPOCH_INDEX, + KEY_PENDING_POOL_UPDATES, KEY_STORAGE_FEE_POOL, KEY_UNPAID_EPOCH_INDEX, }; use grovedb::batch::GroveDbOp; use grovedb::Element; @@ -50,6 +50,8 @@ pub fn add_create_fee_pool_trees_operations(batch: &mut GroveDbOpBatch) { // Init next epoch to pay batch.push(update_unpaid_epoch_index_operation(GENESIS_EPOCH_INDEX)); + add_create_pending_pool_updates_tree_operations(batch); + // We need to insert 50 years worth of epochs, // with 20 epochs per year that's 1000 epochs for i in GENESIS_EPOCH_INDEX..PERPETUAL_STORAGE_EPOCHS { @@ -58,6 +60,11 @@ pub fn add_create_fee_pool_trees_operations(batch: &mut GroveDbOpBatch) { } } +/// Adds operations to batch to create pending pool updates tree +pub fn add_create_pending_pool_updates_tree_operations(batch: &mut GroveDbOpBatch) { + batch.add_insert_empty_tree(pools_vec_path(), KEY_PENDING_POOL_UPDATES.to_vec()); +} + /// Updates the storage fee distribution pool with a new storage fee pub fn update_storage_fee_distribution_pool_operation(storage_fee: u64) -> GroveDbOp { GroveDbOp::insert_op( From 5f6049681d3da6e8202eac04fde52d0b901d93ae Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 15 Dec 2022 15:27:58 +0800 Subject: [PATCH 04/54] tests: pending updates management --- packages/rs-drive-abci/src/abci/handlers.rs | 6 +- packages/rs-drive-abci/src/abci/messages.rs | 8 +- .../execution/fee_pools/fee_distribution.rs | 10 +- .../execution/fee_pools/process_block_fees.rs | 69 +++---- packages/rs-drive-nodejs/src/lib.rs | 6 +- .../src/drive/fee_pools/pending_updates.rs | 192 ++++++++---------- 6 files changed, 129 insertions(+), 162 deletions(-) diff --git a/packages/rs-drive-abci/src/abci/handlers.rs b/packages/rs-drive-abci/src/abci/handlers.rs index cff05da4ecd..03adc29aef0 100644 --- a/packages/rs-drive-abci/src/abci/handlers.rs +++ b/packages/rs-drive-abci/src/abci/handlers.rs @@ -192,7 +192,7 @@ mod tests { use std::ops::Div; use crate::abci::messages::{ - AfterFinalizeBlockRequest, BlockBeginRequest, BlockEndRequest, BlockFeeResult, + AfterFinalizeBlockRequest, BlockBeginRequest, BlockEndRequest, BlockFees, InitChainRequest, }; use crate::common::helpers::setup::setup_platform; @@ -360,7 +360,7 @@ mod tests { } let block_end_request = BlockEndRequest { - fees: BlockFeeResult::from_fees(storage_fees_per_block, 1600), + fees: BlockFees::from_fees(storage_fees_per_block, 1600), }; let block_end_response = platform @@ -512,7 +512,7 @@ mod tests { ); let block_end_request = BlockEndRequest { - fees: BlockFeeResult::from_fees(storage_fees_per_block, 1600), + fees: BlockFees::from_fees(storage_fees_per_block, 1600), }; let block_end_response = platform diff --git a/packages/rs-drive-abci/src/abci/messages.rs b/packages/rs-drive-abci/src/abci/messages.rs index 1689c58f0b4..50478d6c10f 100644 --- a/packages/rs-drive-abci/src/abci/messages.rs +++ b/packages/rs-drive-abci/src/abci/messages.rs @@ -83,13 +83,13 @@ pub struct BlockBeginResponse { pub struct BlockEndRequest { /// The fees for the block /// Avoid of serialization to optimize transfer through Node.JS binding - pub fees: BlockFeeResult, + pub fees: BlockFees, } /// Aggregated fees after block execution -#[derive(Serialize, Deserialize, Default)] +#[derive(Serialize, Deserialize, Default, Clone)] #[serde(rename_all = "camelCase")] -pub struct BlockFeeResult { +pub struct BlockFees { /// Processing fee pub processing_fee: u64, /// Storage fee @@ -98,7 +98,7 @@ pub struct BlockFeeResult { pub fee_refunds: CreditsPerEpoch, } -impl BlockFeeResult { +impl BlockFees { /// Create block fee result from fees pub fn from_fees(storage_fee: u64, processing_fee: u64) -> Self { Self { diff --git a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs index fe544a24171..41244972e92 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs @@ -34,7 +34,7 @@ use crate::error::execution::ExecutionError; -use crate::abci::messages::BlockFeeResult; +use crate::abci::messages::BlockFees; use crate::error::Error; use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; @@ -384,7 +384,7 @@ impl Platform { pub fn add_distribute_block_fees_into_pools_operations( &self, current_epoch: &Epoch, - block_fees: &BlockFeeResult, + block_fees: &BlockFees, cached_aggregated_storage_fees: Option, transaction: TransactionArg, batch: &mut GroveDbOpBatch, @@ -1304,7 +1304,7 @@ mod tests { } mod add_distribute_block_fees_into_pools_operations { - use crate::abci::messages::BlockFeeResult; + use crate::abci::messages::BlockFees; use crate::common::helpers::setup::setup_platform_with_initial_state_structure; use drive::drive::batch::GroveDbOpBatch; use drive::fee::FeeResult; @@ -1327,7 +1327,7 @@ mod tests { platform .add_distribute_block_fees_into_pools_operations( ¤t_epoch_tree, - &BlockFeeResult::from_fees(storage_fees, processing_fees), + &BlockFees::from_fees(storage_fees, processing_fees), None, Some(&transaction), &mut batch, @@ -1381,7 +1381,7 @@ mod tests { platform .add_distribute_block_fees_into_pools_operations( ¤t_epoch_tree, - &BlockFeeResult::from_fees(storage_fees, processing_fees), + &BlockFees::from_fees(storage_fees, processing_fees), None, Some(&transaction), &mut batch, diff --git a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs index 579c5622474..72badb60990 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs @@ -33,7 +33,15 @@ //! epoch changes. //! -use crate::abci::messages::BlockFeeResult; +use std::option::Option::None; + +use drive::drive::batch::GroveDbOpBatch; +use drive::drive::fee_pools::epochs::constants::{GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; +use drive::drive::fee_pools::pending_updates::add_update_pending_epoch_pool_update_operations; +use drive::fee_pools::epochs::Epoch; +use drive::grovedb::TransactionArg; + +use crate::abci::messages::BlockFees; use crate::block::BlockInfo; use crate::error::Error; use crate::execution::fee_pools::constants::DEFAULT_ORIGINAL_FEE_MULTIPLIER; @@ -41,13 +49,6 @@ use crate::execution::fee_pools::distribute_storage_pool::StorageDistributionLef use crate::execution::fee_pools::epoch::EpochInfo; use crate::execution::fee_pools::fee_distribution::{FeesInPools, ProposersPayouts}; use crate::platform::Platform; -use drive::drive::batch::GroveDbOpBatch; -use drive::drive::fee_pools::epochs::constants::{GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; -use drive::drive::fee_pools::pending_updates::add_update_pending_epoch_pool_update_operations; -use drive::fee::FeeResult; -use drive::fee_pools::epochs::Epoch; -use drive::grovedb::TransactionArg; -use std::option::Option::None; /// From the Dash Improvement Proposal: @@ -80,7 +81,7 @@ impl Platform { &self, block_info: &BlockInfo, epoch_info: &EpochInfo, - block_fees: &BlockFeeResult, + block_fees: &BlockFees, transaction: TransactionArg, batch: &mut GroveDbOpBatch, ) -> Result, Error> { @@ -137,7 +138,7 @@ impl Platform { &self, block_info: &BlockInfo, epoch_info: &EpochInfo, - block_fees: BlockFeeResult, + block_fees: BlockFees, transaction: TransactionArg, ) -> Result { let current_epoch = Epoch::new(epoch_info.current_epoch_index); @@ -208,7 +209,7 @@ impl Platform { block_fees.fee_refunds }; - add_update_pending_epoch_pool_update_operations(&mut batch, pending_epoch_pool_updates)?; + add_update_pending_epoch_pool_update_operations(&mut batch, pending_epoch_pool_updates); self.drive.grove_apply_batch(batch, false, transaction)?; @@ -221,22 +222,18 @@ impl Platform { #[cfg(test)] mod tests { + use super::*; + + use crate::common::helpers::setup::setup_platform_with_initial_state_structure; + use crate::execution::fee_pools::epoch::EPOCH_CHANGE_TIME_MS; + use chrono::Utc; + use rust_decimal::prelude::ToPrimitive; + mod add_process_epoch_change_operations { - use crate::common::helpers::setup::setup_platform_with_initial_state_structure; - use chrono::Utc; - use drive::drive::fee_pools::epochs::constants::GENESIS_EPOCH_INDEX; - use rust_decimal::prelude::ToPrimitive; + use super::*; mod helpers { - use crate::abci::messages::BlockFeeResult; - use crate::block::BlockInfo; - use crate::execution::fee_pools::epoch::{EpochInfo, EPOCH_CHANGE_TIME_MS}; - use crate::platform::Platform; - use drive::drive::batch::GroveDbOpBatch; - use drive::drive::fee_pools::epochs::constants::PERPETUAL_STORAGE_EPOCHS; - use drive::fee::FeeResult; - use drive::fee_pools::epochs::Epoch; - use drive::grovedb::TransactionArg; + use super::*; /// Process and validate an epoch change pub fn process_and_validate_epoch_change( @@ -252,7 +249,7 @@ mod tests { // Add some storage fees to distribute next time if should_distribute { - let block_fees = BlockFeeResult::from_fees(1000000000, 1000); + let block_fees = BlockFees::from_fees(1000000000, 1000); let mut batch = GroveDbOpBatch::new(); @@ -290,12 +287,15 @@ mod tests { EpochInfo::from_genesis_time_and_block_info(genesis_time_ms, &block_info) .expect("should calculate epoch info"); + let block_fees = BlockFees::default(); + let mut batch = GroveDbOpBatch::new(); let distribute_storage_pool_result = platform .add_process_epoch_change_operations( &block_info, &epoch_info, + &block_fees, transaction, &mut batch, ) @@ -414,21 +414,12 @@ mod tests { } mod process_block_fees { - use crate::common::helpers::setup::setup_platform_with_initial_state_structure; - use chrono::Utc; + use super::*; + use drive::common::helpers::identities::create_test_masternode_identities; - use drive::drive::fee_pools::epochs::constants::GENESIS_EPOCH_INDEX; - use rust_decimal::prelude::ToPrimitive; mod helpers { - use crate::abci::messages::BlockFeeResult; - use crate::block::BlockInfo; - use crate::execution::fee_pools::epoch::{EpochInfo, EPOCH_CHANGE_TIME_MS}; - use crate::platform::Platform; - use drive::drive::fee_pools::epochs::constants::GENESIS_EPOCH_INDEX; - use drive::fee::FeeResult; - use drive::fee_pools::epochs::Epoch; - use drive::grovedb::TransactionArg; + use super::*; /// Process and validate block fees pub fn process_and_validate_block_fees( @@ -456,10 +447,10 @@ mod tests { EpochInfo::from_genesis_time_and_block_info(genesis_time_ms, &block_info) .expect("should calculate epoch info"); - let block_fees = BlockFeeResult::from_fees(1000, 10000); + let block_fees = BlockFees::from_fees(1000, 10000); let distribute_storage_pool_result = platform - .process_block_fees(&block_info, &epoch_info, &block_fees, transaction) + .process_block_fees(&block_info, &epoch_info, block_fees.clone(), transaction) .expect("should process block fees"); // Should process epoch change diff --git a/packages/rs-drive-nodejs/src/lib.rs b/packages/rs-drive-nodejs/src/lib.rs index c99ddb03be7..793cf8a6c0f 100644 --- a/packages/rs-drive-nodejs/src/lib.rs +++ b/packages/rs-drive-nodejs/src/lib.rs @@ -18,8 +18,8 @@ use drive::grovedb::{PathQuery, Transaction}; use drive::query::TransactionArg; use drive_abci::abci::handlers::TenderdashAbci; use drive_abci::abci::messages::{ - AfterFinalizeBlockRequest, BlockBeginRequest, BlockEndRequest, BlockFeeResult, - InitChainRequest, Serializable, + AfterFinalizeBlockRequest, BlockBeginRequest, BlockEndRequest, BlockFees, InitChainRequest, + Serializable, }; use drive_abci::platform::Platform; use neon::prelude::*; @@ -2072,7 +2072,7 @@ impl PlatformWrapper { let result = transaction_result.and_then(|transaction_arg| { let request = BlockEndRequest { - fees: BlockFeeResult { + fees: BlockFees { processing_fee, storage_fee, fee_refunds, diff --git a/packages/rs-drive/src/drive/fee_pools/pending_updates.rs b/packages/rs-drive/src/drive/fee_pools/pending_updates.rs index e0eceb664c3..6556d0b1ef8 100644 --- a/packages/rs-drive/src/drive/fee_pools/pending_updates.rs +++ b/packages/rs-drive/src/drive/fee_pools/pending_updates.rs @@ -33,13 +33,12 @@ use crate::drive::batch::GroveDbOpBatch; use crate::drive::fee_pools::pools_pending_updates_path; use crate::drive::Drive; use crate::error::drive::DriveError; +use crate::error::fee::FeeError; use crate::error::Error; use crate::fee::refunds::CreditsPerEpoch; use grovedb::query_result_type::QueryResultType; use grovedb::{Element, PathQuery, Query, TransactionArg}; -type PendingUpdatesCount = usize; - impl Drive { /// Fetches existing pending epoch pool updates using specified epochs /// and returns merged result @@ -51,7 +50,7 @@ impl Drive { let mut query = Query::new(); for (epoch_index_key, _) in credits_per_epoch.iter() { - let epoch_index: u64 = epoch_index_key.to_owned(); + let epoch_index = epoch_index_key.to_owned() as u16; let encoded_epoch_index = epoch_index.to_be_bytes().to_vec(); query.insert_key(encoded_epoch_index); @@ -98,9 +97,16 @@ impl Drive { })?, ); - let existing_credits = existing_signed_credits as u64; + let existing_credits = existing_signed_credits.unsigned_abs(); + + let result_credits = + credits_to_update + .checked_add(existing_credits) + .ok_or(Error::Fee(FeeError::Overflow( + "pending updates overflow error", + )))?; - credits_per_epoch.insert(epoch_index_key, credits_to_update + existing_credits); + credits_per_epoch.insert(epoch_index_key, result_credits); } Ok(credits_per_epoch) @@ -153,9 +159,7 @@ impl Drive { pub fn add_update_pending_epoch_pool_update_operations( batch: &mut GroveDbOpBatch, credits_per_epoch: CreditsPerEpoch, -) -> Result { - let credits_per_epoch_count = credits_per_epoch.len(); - +) { for (epoch_index_key, credits) in credits_per_epoch { let epoch_index = epoch_index_key as u16; let encoded_epoch_index = epoch_index.to_be_bytes().to_vec(); @@ -166,8 +170,6 @@ pub fn add_update_pending_epoch_pool_update_operations( batch.add_insert(pools_pending_updates_path(), encoded_epoch_index, element); } - - Ok(credits_per_epoch_count) } #[cfg(test)] @@ -177,128 +179,102 @@ mod tests { mod fetch_and_merge_with_existing_pending_epoch_pool_updates { use super::*; - } - - mod add_delete_pending_epoch_pool_updates_except_specified_operations { - use super::*; - } - - mod add_update_pending_epoch_pool_update_operations { - use super::*; - - mod helpers { - use super::*; - use grovedb::batch::Op; - pub fn process_and_assert_pending_updates( - drive: &Drive, - pending_updates: CreditsPerEpoch, - encoded_pending_updates: &HashMap, i64>, - transaction: TransactionArg, - ) { - let pending_updates_count = pending_updates.len(); + #[test] + fn should_fetch_and_merge_pending_updates() { + let drive = setup_drive_with_initial_state_structure(); - let mut batch = GroveDbOpBatch::new(); + let transaction = drive.grove.start_transaction(); - let updates_count = drive - .add_update_pending_epoch_pool_update_operations( - &mut batch, - pending_updates, - transaction, - ) - .expect("should update pending pool updates"); + // Store initial set of pending updates - drive - .grove_apply_batch(batch.clone(), false, transaction) - .expect("should apply batch"); + let initial_pending_updates = + CreditsPerEpoch::from_iter(vec![(1, 15), (3, 25), (7, 95), (9, 100), (12, 120)]); - assert_eq!(batch.len(), updates_count); - assert_eq!(batch.len(), pending_updates_count); + let mut batch = GroveDbOpBatch::new(); - for operation in batch.operations { - assert_eq!(operation.path.to_path(), pools_pending_updates_path()); + add_update_pending_epoch_pool_update_operations(&mut batch, initial_pending_updates); - let encoded_key = operation.key.get_key(); + drive + .grove_apply_batch(batch, false, Some(&transaction)) + .expect("should apply batch"); - assert!(encoded_pending_updates.contains_key(&encoded_key)); + // Fetch and merge - let expected_credits = encoded_pending_updates[&encoded_key]; + let new_pending_updates = + CreditsPerEpoch::from_iter(vec![(1, 15), (3, 25), (30, 195), (41, 150)]); - let expected_encoded_credits = expected_credits.to_be_bytes().to_vec(); + let updated_pending_updates = drive + .fetch_and_merge_with_existing_pending_epoch_pool_updates( + new_pending_updates, + Some(&transaction), + ) + .expect("should fetch and merge pending updates"); - let Op::Insert { - element: Element::Item(encoded_credits, None) - } = operation.op else { - panic!("pending pool update should be stored as an item"); - }; + let expected_pending_updates = + CreditsPerEpoch::from_iter(vec![(1, 30), (3, 50), (30, 195), (41, 150)]); - assert_eq!(encoded_credits, expected_encoded_credits); - } - } + assert_eq!(updated_pending_updates, expected_pending_updates); } + } + + mod add_delete_pending_epoch_pool_updates_except_specified_operations { + use super::*; + use grovedb::batch::Op; #[test] - fn should_add_new_pending_pool_updates() { + fn should_add_delete_operations() { let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - // Add initial set of pending updates + // Store initial set of pending updates let initial_pending_updates = CreditsPerEpoch::from_iter(vec![(1, 15), (3, 25), (7, 95), (9, 100), (12, 120)]); - let initial_encoded_epoch_indices_and_signed_credits = initial_pending_updates - .clone() - .into_iter() - .map(|(epoch_index, credits)| { - let epoch_index_key = epoch_index as u16; - let epoch_index_key = epoch_index_key.to_be_bytes().to_vec(); - - // Credits must be stored as negative to support total credits balance - // verification with sum trees - (epoch_index_key, -(credits as i64)) - }) - .collect::>(); - - helpers::process_and_assert_pending_updates( - &drive, - initial_pending_updates, - &initial_encoded_epoch_indices_and_signed_credits, - Some(&transaction), - ); + let mut batch = GroveDbOpBatch::new(); - // Add new and update existing - - let additional_pending_updates = - CreditsPerEpoch::from_iter(vec![(1, 15), (3, 25), (15, 95), (16, 100)]); - - let additional_encoded_epoch_indices_and_signed_credits = additional_pending_updates - .clone() - .into_iter() - .map(|(epoch_index, credits)| { - let epoch_index_key = epoch_index as u16; - let epoch_index_key = epoch_index_key.to_be_bytes().to_vec(); - let mut signed_credits = -(credits as i64); - - if let Some(initial_signed_credits) = - initial_encoded_epoch_indices_and_signed_credits.get(&epoch_index_key) - { - signed_credits += initial_signed_credits; - } - - // Credits must be stored as negative to support total credits balance - // verification with sum trees - (epoch_index_key, signed_credits) - }) - .collect::>(); - - helpers::process_and_assert_pending_updates( - &drive, - additional_pending_updates, - &additional_encoded_epoch_indices_and_signed_credits, - Some(&transaction), - ); + add_update_pending_epoch_pool_update_operations(&mut batch, initial_pending_updates); + + drive + .grove_apply_batch(batch, false, Some(&transaction)) + .expect("should apply batch"); + + // Delete existing pending updates expect specified pending updates + + let new_pending_updates = CreditsPerEpoch::from_iter(vec![(1, 15), (3, 25)]); + + let mut batch = GroveDbOpBatch::new(); + + drive + .add_delete_pending_epoch_pool_updates_except_specified_operations( + &mut batch, + &new_pending_updates, + Some(&transaction), + ) + .expect("should fetch and merge pending updates"); + + let expected_pending_updates = + CreditsPerEpoch::from_iter(vec![(7, 95), (9, 100), (12, 120)]); + + assert_eq!(batch.len(), expected_pending_updates.len()); + + for operation in batch.operations { + assert!(matches!(operation.op, Op::Delete)); + + assert_eq!(operation.path.to_path(), pools_pending_updates_path()); + + let encoded_epoch_index = operation.key.get_key(); + let epoch_index = u16::from_be_bytes( + encoded_epoch_index + .try_into() + .expect("should convert to u16 bytes"), + ); + let epoch_index_key = epoch_index as u64; + + assert!(expected_pending_updates.contains_key(epoch_index_key)); + } } } } From 03d8e2a8f6fc5985402aeece9753280703213b64 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 15 Dec 2022 15:37:39 +0800 Subject: [PATCH 05/54] tests: fixed removed bytes assertions --- .../src/drive/batch/grovedb_op_batch.rs | 2 +- .../rs-drive/src/drive/document/delete.rs | 7 ++-- .../rs-drive/src/drive/document/update.rs | 32 ++++++++++++++----- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs b/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs index 9e09b4d1240..a68e8a1da92 100644 --- a/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs +++ b/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs @@ -38,7 +38,7 @@ use grovedb::Element; /// A batch of GroveDB operations as a vector. // TODO move to GroveDB -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default)] pub struct GroveDbOpBatch { /// Operations pub(crate) operations: Vec, diff --git a/packages/rs-drive/src/drive/document/delete.rs b/packages/rs-drive/src/drive/document/delete.rs index da835204220..a329ab3bc0b 100644 --- a/packages/rs-drive/src/drive/document/delete.rs +++ b/packages/rs-drive/src/drive/document/delete.rs @@ -1404,13 +1404,16 @@ mod tests { ) .expect("expected to be able to delete the document"); - let removed_bytes = fee_result + let removed_credits = fee_result .fee_refunds .get(&random_owner_id) .unwrap() .get(0) .unwrap(); - assert_eq!(added_bytes, *removed_bytes as u64); + + let removed_bytes = removed_credits / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; + + assert_eq!(added_bytes, removed_bytes); } #[test] diff --git a/packages/rs-drive/src/drive/document/update.rs b/packages/rs-drive/src/drive/document/update.rs index 20972f569ea..cf9fb574d27 100644 --- a/packages/rs-drive/src/drive/document/update.rs +++ b/packages/rs-drive/src/drive/document/update.rs @@ -1522,13 +1522,18 @@ mod tests { &person_0_original, transaction.as_ref(), ); - let removed_bytes = deletion_fees + + let removed_credits = deletion_fees .fee_refunds .get(&owner_id) .unwrap() .get(0) .unwrap(); - assert_eq!(original_bytes, *removed_bytes as u64); + + let removed_bytes = removed_credits / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; + + assert_eq!(original_bytes, removed_bytes); + // let's re-add it again let original_fees = apply_person( &drive, @@ -1538,7 +1543,9 @@ mod tests { true, transaction.as_ref(), ); + let original_bytes = original_fees.storage_fee / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; + assert_eq!(original_bytes, expected_added_bytes); } @@ -1634,13 +1641,18 @@ mod tests { &person_0_original, transaction.as_ref(), ); - let removed_bytes = deletion_fees + + let removed_credits = deletion_fees .fee_refunds .get(&owner_id) .unwrap() .get(0) .unwrap(); - assert_eq!(original_bytes, *removed_bytes as u64); + + let removed_bytes = removed_credits / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; + + assert_eq!(original_bytes, removed_bytes); + // let's re-add it again let original_fees = apply_person( &drive, @@ -1650,11 +1662,13 @@ mod tests { true, transaction.as_ref(), ); + let original_bytes = original_fees.storage_fee / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; + assert_eq!(original_bytes, expected_added_bytes); } - // now let's update it + // now let's update it let update_fees = apply_person( &drive, &contract, @@ -1666,20 +1680,22 @@ mod tests { // we both add and remove bytes // this is because trees are added because of indexes, and also removed let added_bytes = update_fees.storage_fee / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; - let removed_bytes = update_fees + + let removed_credits = update_fees .fee_refunds .get(&owner_id) .unwrap() .get(0) .unwrap(); + let removed_bytes = removed_credits / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; + // We added one byte, and since it is an index, and keys are doubled it's 2 extra bytes let expected_added_bytes = if using_history { 601 } else { 599 }; assert_eq!(added_bytes, expected_added_bytes); let expected_removed_bytes = if using_history { 598 } else { 596 }; - - assert_eq!(*removed_bytes, expected_removed_bytes); + assert_eq!(removed_bytes, expected_removed_bytes); } #[test] From 4952f5ae37c20450e062d77779b7246969674009 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 16 Dec 2022 02:15:36 +0800 Subject: [PATCH 06/54] chore: distribution function and other wip stuff --- Cargo.lock | 4 +- packages/rs-drive-abci/Cargo.toml | 2 - .../fee_pools/distribute_storage_pool.rs | 10 +-- .../execution/fee_pools/fee_distribution.rs | 77 +++++------------ .../src/execution/fee_pools/mod.rs | 1 - .../execution/fee_pools/process_block_fees.rs | 15 ++-- packages/rs-drive/Cargo.toml | 2 + .../src/drive/batch/drive_op_batch.rs | 3 +- packages/rs-drive/src/drive/contract/mod.rs | 3 +- .../rs-drive/src/drive/document/delete.rs | 3 +- .../rs-drive/src/drive/document/insert.rs | 3 +- .../rs-drive/src/drive/document/update.rs | 3 +- .../epochs/credit_distribution_pools.rs | 9 +- .../src/drive/fee_pools/epochs/mod.rs | 13 ++- packages/rs-drive/src/drive/fee_pools/mod.rs | 73 +++++++++++++++- ...ng_updates.rs => pending_epoch_updates.rs} | 41 ++++++--- packages/rs-drive/src/drive/identity/mod.rs | 3 +- packages/rs-drive/src/drive/mod.rs | 4 +- packages/rs-drive/src/fee/default_costs.rs | 34 ++++++++ .../src/fee/epoch/distribution.rs} | 64 +++++++++++++- .../epochs/constants.rs => fee/epoch/mod.rs} | 9 ++ packages/rs-drive/src/fee/mod.rs | 55 ++---------- packages/rs-drive/src/fee/op.rs | 8 +- packages/rs-drive/src/fee/result/mod.rs | 86 +++++++++++++++++++ .../rs-drive/src/fee/{ => result}/refunds.rs | 46 ++++++++-- .../fee_pools/epochs/operations_factory.rs | 17 ++-- packages/rs-drive/src/fee_pools/mod.rs | 7 +- 27 files changed, 408 insertions(+), 187 deletions(-) rename packages/rs-drive/src/drive/fee_pools/{pending_updates.rs => pending_epoch_updates.rs} (87%) rename packages/{rs-drive-abci/src/execution/fee_pools/constants.rs => rs-drive/src/fee/epoch/distribution.rs} (56%) rename packages/rs-drive/src/{drive/fee_pools/epochs/constants.rs => fee/epoch/mod.rs} (70%) create mode 100644 packages/rs-drive/src/fee/result/mod.rs rename packages/rs-drive/src/fee/{ => result}/refunds.rs (76%) diff --git a/Cargo.lock b/Cargo.lock index 98bbaec1c41..858646824f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -920,6 +920,8 @@ dependencies = [ "moka", "rand 0.8.5", "rand_distr", + "rust_decimal", + "rust_decimal_macros", "serde", "serde_json", "sqlparser", @@ -940,8 +942,6 @@ dependencies = [ "drive", "hex", "rand 0.8.5", - "rust_decimal", - "rust_decimal_macros", "serde", "serde_json", "tempfile", diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml index b3678fb8d37..282f3fefd59 100644 --- a/packages/rs-drive-abci/Cargo.toml +++ b/packages/rs-drive-abci/Cargo.toml @@ -7,9 +7,7 @@ license = "MIT" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -rust_decimal = "1.2.5" ciborium = "0.2.0" -rust_decimal_macros = "1.25.0" chrono = "0.4.20" serde = { version = "1.0.132", features = ["derive"] } serde_json = { version="1.0", features=["preserve_order"] } diff --git a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs index 9cf3b147d52..561f2de8e3a 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs @@ -35,10 +35,10 @@ use crate::error::execution::ExecutionError; use crate::error::Error; -use crate::execution::fee_pools::constants; use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; use drive::drive::fee_pools::epochs::constants::{EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS}; +use drive::fee::constants; use drive::fee_pools::epochs::Epoch; use drive::grovedb::TransactionArg; use drive::{error, grovedb}; @@ -111,11 +111,9 @@ impl Platform { // TODO: It's not convenient and confusing when in one case you should push operation to batch // and sometimes you pass batch inside to add operations. Also, in future a single operation function // could become a multiple operations function so you need to change many code. Also, you can't use helpers which batch provides - batch.push( - epoch_tree.update_storage_credits_for_distribution_operation( - current_epoch_pool_storage_credits + epoch_fee_share, - ), - ); + batch.push(epoch_tree.update_storage_fee_pool_operation( + current_epoch_pool_storage_credits + epoch_fee_share, + )); storage_distribution_leftover_credits = storage_distribution_leftover_credits .checked_sub(epoch_fee_share) diff --git a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs index 41244972e92..607a2f3c972 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs @@ -40,7 +40,7 @@ use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; use drive::drive::fee_pools::epochs::constants::GENESIS_EPOCH_INDEX; use drive::error::fee::FeeError; -use drive::fee::FeeResult; +use drive::fee::epoch::GENESIS_EPOCH_INDEX; use drive::fee_pools::epochs::Epoch; use drive::fee_pools::{ update_storage_fee_distribution_pool_operation, update_unpaid_epoch_index_operation, @@ -401,10 +401,7 @@ impl Platform { let total_processing_fees = epoch_processing_fees + block_fees.processing_fee; - batch.push( - current_epoch - .update_processing_credits_for_distribution_operation(total_processing_fees), - ); + batch.push(current_epoch.update_processing_fee_pool_operation(total_processing_fees)); // update storage fee pool let storage_distribution_credits_in_fee_pool = match cached_aggregated_storage_fees { @@ -429,14 +426,13 @@ impl Platform { #[cfg(test)] mod tests { + use super::*; + + use crate::common::helpers::setup::setup_platform_with_initial_state_structure; + use drive::common::helpers::identities::create_test_masternode_identities_and_add_them_as_epoch_block_proposers; + mod add_distribute_fees_from_oldest_unpaid_epoch_pool_to_proposers_operations { - use crate::common::helpers::setup::setup_platform_with_initial_state_structure; - use drive::common::helpers::identities::create_test_masternode_identities_and_add_them_as_epoch_block_proposers; - use drive::drive::batch::GroveDbOpBatch; - use drive::drive::fee_pools::epochs::constants::GENESIS_EPOCH_INDEX; - use drive::error::Error; - use drive::fee_pools::epochs::Epoch; - use drive::grovedb; + use super::*; #[test] fn test_nothing_to_distribute_if_there_is_no_epochs_needing_payment() { @@ -479,9 +475,7 @@ mod tests { unpaid_epoch_tree_0.add_init_current_operations(1.0, 1, 1, &mut batch); - batch.push( - unpaid_epoch_tree_0.update_processing_credits_for_distribution_operation(10000), - ); + batch.push(unpaid_epoch_tree_0.update_processing_fee_pool_operation(10000)); let proposers_count = 100u16; @@ -550,9 +544,7 @@ mod tests { unpaid_epoch_tree_0.add_init_current_operations(1.0, 1, 1, &mut batch); - batch.push( - unpaid_epoch_tree_0.update_processing_credits_for_distribution_operation(10000), - ); + batch.push(unpaid_epoch_tree_0.update_processing_fee_pool_operation(10000)); let proposers_count = 100u16; @@ -636,9 +628,7 @@ mod tests { unpaid_epoch_tree_0.add_init_current_operations(1.0, 1, 1, &mut batch); - batch.push( - unpaid_epoch_tree_0.update_processing_credits_for_distribution_operation(10000), - ); + batch.push(unpaid_epoch_tree_0.update_processing_fee_pool_operation(10000)); let proposers_count = 200u16; @@ -733,12 +723,9 @@ mod tests { unpaid_epoch.add_init_current_operations(1.0, 1, 1, &mut batch); - batch.push( - unpaid_epoch.update_processing_credits_for_distribution_operation(processing_fees), - ); + batch.push(unpaid_epoch.update_processing_fee_pool_operation(processing_fees)); - batch - .push(unpaid_epoch.update_storage_credits_for_distribution_operation(storage_fees)); + batch.push(unpaid_epoch.update_storage_fee_pool_operation(storage_fees)); current_epoch.add_init_current_operations(1.0, 2, 2, &mut batch); @@ -819,12 +806,9 @@ mod tests { unpaid_epoch.add_init_current_operations(1.0, 1, 1, &mut batch); - batch.push( - unpaid_epoch.update_processing_credits_for_distribution_operation(processing_fees), - ); + batch.push(unpaid_epoch.update_processing_fee_pool_operation(processing_fees)); - batch - .push(unpaid_epoch.update_storage_credits_for_distribution_operation(storage_fees)); + batch.push(unpaid_epoch.update_storage_fee_pool_operation(storage_fees)); current_epoch.add_init_current_operations(1.0, 2, 2, &mut batch); @@ -913,13 +897,7 @@ mod tests { } mod find_oldest_epoch_needing_payment { - use crate::common::helpers::setup::setup_platform_with_initial_state_structure; - use crate::error::execution::ExecutionError; - use crate::error::Error; - use drive::drive::batch::GroveDbOpBatch; - use drive::drive::fee_pools::epochs::constants::GENESIS_EPOCH_INDEX; - use drive::fee_pools::epochs::Epoch; - use drive::fee_pools::update_unpaid_epoch_index_operation; + use super::*; #[test] fn test_no_epoch_to_pay_on_genesis_epoch() { @@ -1166,16 +1144,10 @@ mod tests { } mod add_epoch_pool_to_proposers_payout_operations { + use super::*; use crate::common::helpers::fee_pools::{ create_test_masternode_share_identities_and_documents, refetch_identities, }; - use crate::common::helpers::setup::setup_platform_with_initial_state_structure; - use crate::execution::fee_pools::fee_distribution::UnpaidEpoch; - use drive::common::helpers::identities::create_test_masternode_identities_and_add_them_as_epoch_block_proposers; - use drive::drive::batch::GroveDbOpBatch; - use drive::fee_pools::epochs::Epoch; - use rust_decimal::Decimal; - use rust_decimal_macros::dec; #[test] fn test_payout_to_proposers() { @@ -1196,14 +1168,9 @@ mod tests { unpaid_epoch_tree.add_init_current_operations(1.0, 1, 1, &mut batch); - batch.push( - unpaid_epoch_tree - .update_processing_credits_for_distribution_operation(processing_fees), - ); + batch.push(unpaid_epoch_tree.update_processing_fee_pool_operation(processing_fees)); - batch.push( - unpaid_epoch_tree.update_storage_credits_for_distribution_operation(storage_fees), - ); + batch.push(unpaid_epoch_tree.update_storage_fee_pool_operation(storage_fees)); next_epoch_tree.add_init_current_operations( 1.0, @@ -1304,11 +1271,7 @@ mod tests { } mod add_distribute_block_fees_into_pools_operations { - use crate::abci::messages::BlockFees; - use crate::common::helpers::setup::setup_platform_with_initial_state_structure; - use drive::drive::batch::GroveDbOpBatch; - use drive::fee::FeeResult; - use drive::fee_pools::epochs::Epoch; + use super::*; #[test] fn test_distribute_block_fees_into_uncommitted_epoch_on_epoch_change() { diff --git a/packages/rs-drive-abci/src/execution/fee_pools/mod.rs b/packages/rs-drive-abci/src/execution/fee_pools/mod.rs index 84064819c31..3006174835c 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/mod.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/mod.rs @@ -1,4 +1,3 @@ -pub mod constants; pub mod distribute_storage_pool; pub mod epoch; pub mod fee_distribution; diff --git a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs index 72badb60990..55d7ecfc447 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs @@ -37,18 +37,20 @@ use std::option::Option::None; use drive::drive::batch::GroveDbOpBatch; use drive::drive::fee_pools::epochs::constants::{GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; -use drive::drive::fee_pools::pending_updates::add_update_pending_epoch_pool_update_operations; +use drive::drive::fee_pools::pending_epoch_updates::add_update_pending_epoch_storage_pool_update_operations; use drive::fee_pools::epochs::Epoch; use drive::grovedb::TransactionArg; use crate::abci::messages::BlockFees; use crate::block::BlockInfo; use crate::error::Error; -use crate::execution::fee_pools::constants::DEFAULT_ORIGINAL_FEE_MULTIPLIER; use crate::execution::fee_pools::distribute_storage_pool::StorageDistributionLeftoverCredits; use crate::execution::fee_pools::epoch::EpochInfo; use crate::execution::fee_pools::fee_distribution::{FeesInPools, ProposersPayouts}; use crate::platform::Platform; +use drive::fee::constants::DEFAULT_ORIGINAL_FEE_MULTIPLIER; +use drive::fee::epoch::{GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; +use drive::fee::DEFAULT_ORIGINAL_FEE_MULTIPLIER; /// From the Dash Improvement Proposal: @@ -121,7 +123,7 @@ impl Platform { )?; self.drive - .add_delete_pending_epoch_pool_updates_except_specified_operations( + .add_delete_pending_epoch_storage_pool_updates_except_specified_operations( batch, &block_fees.fee_refunds, transaction, @@ -201,7 +203,7 @@ impl Platform { let pending_epoch_pool_updates = if !epoch_info.is_epoch_change { self.drive - .fetch_and_merge_with_existing_pending_epoch_pool_updates( + .fetch_and_merge_with_existing_pending_epoch_storage_pool_updates( block_fees.fee_refunds, transaction, )? @@ -209,7 +211,10 @@ impl Platform { block_fees.fee_refunds }; - add_update_pending_epoch_pool_update_operations(&mut batch, pending_epoch_pool_updates); + add_update_pending_epoch_storage_pool_update_operations( + &mut batch, + pending_epoch_pool_updates, + )?; self.drive.grove_apply_batch(batch, false, transaction)?; diff --git a/packages/rs-drive/Cargo.toml b/packages/rs-drive/Cargo.toml index 92d40e4002b..af61e3dd4d2 100644 --- a/packages/rs-drive/Cargo.toml +++ b/packages/rs-drive/Cargo.toml @@ -30,6 +30,8 @@ bincode = "1.3.3" dpp = { path = "../rs-dpp" } itertools = { version = "0.10.5" } dashcore = { git="https://github.com/dashpay/rust-dashcore", features=["no-std", "secp-recovery", "rand", "signer"], default-features = false, branch="master" } +rust_decimal = "1.2.5" +rust_decimal_macros = "1.25.0" [dependencies.grovedb] git = "https://github.com/dashpay/grovedb" diff --git a/packages/rs-drive/src/drive/batch/drive_op_batch.rs b/packages/rs-drive/src/drive/batch/drive_op_batch.rs index c90ace9d059..e7d5a46817a 100644 --- a/packages/rs-drive/src/drive/batch/drive_op_batch.rs +++ b/packages/rs-drive/src/drive/batch/drive_op_batch.rs @@ -35,8 +35,9 @@ use crate::drive::object_size_info::DocumentAndContractInfo; use crate::drive::object_size_info::DocumentInfo::DocumentRefAndSerialization; use crate::drive::Drive; use crate::error::Error; +use crate::fee::calculate_fee; use crate::fee::op::DriveOperation; -use crate::fee::{calculate_fee, FeeResult}; +use crate::fee::result::FeeResult; use dpp::data_contract::extra::DriveContractExt; use grovedb::batch::KeyInfoPath; use grovedb::{EstimatedLayerInformation, TransactionArg}; diff --git a/packages/rs-drive/src/drive/contract/mod.rs b/packages/rs-drive/src/drive/contract/mod.rs index a6623f5a2bc..4a1a24fd2c4 100644 --- a/packages/rs-drive/src/drive/contract/mod.rs +++ b/packages/rs-drive/src/drive/contract/mod.rs @@ -64,9 +64,10 @@ use crate::drive::object_size_info::PathKeyInfo::PathFixedSizeKeyRef; use crate::drive::{contract_documents_path, Drive, RootTree}; use crate::error::drive::DriveError; use crate::error::Error; +use crate::fee::calculate_fee; use crate::fee::op::DriveOperation; use crate::fee::op::DriveOperation::{CalculatedCostOperation, PreCalculatedFeeResult}; -use crate::fee::{calculate_fee, FeeResult}; +use crate::fee::result::FeeResult; use crate::fee_pools::epochs::Epoch; /// The global root path for all contracts diff --git a/packages/rs-drive/src/drive/document/delete.rs b/packages/rs-drive/src/drive/document/delete.rs index 4cfb015f517..5d27514deab 100644 --- a/packages/rs-drive/src/drive/document/delete.rs +++ b/packages/rs-drive/src/drive/document/delete.rs @@ -70,8 +70,9 @@ use crate::error::document::DocumentError; use crate::error::drive::DriveError; use crate::error::fee::FeeError; use crate::error::Error; +use crate::fee::calculate_fee; use crate::fee::op::DriveOperation; -use crate::fee::{calculate_fee, FeeResult}; +use crate::fee::result::FeeResult; use dpp::data_contract::extra::{DocumentType, DriveContractExt, IndexLevel}; impl Drive { diff --git a/packages/rs-drive/src/drive/document/insert.rs b/packages/rs-drive/src/drive/document/insert.rs index 9b5eebb26d9..ae02dba4304 100644 --- a/packages/rs-drive/src/drive/document/insert.rs +++ b/packages/rs-drive/src/drive/document/insert.rs @@ -68,8 +68,8 @@ use crate::drive::object_size_info::{DocumentAndContractInfo, PathInfo, PathKeyE use crate::drive::Drive; use crate::error::drive::DriveError; use crate::error::Error; +use crate::fee::calculate_fee; use crate::fee::op::DriveOperation; -use crate::fee::{calculate_fee, FeeResult}; use crate::common::encode::encode_unsigned_integer; use crate::contract::document::Document; @@ -79,6 +79,7 @@ use crate::drive::grove_operations::QueryTarget::QueryTargetValue; use crate::drive::grove_operations::{BatchInsertApplyType, BatchInsertTreeApplyType}; use crate::error::document::DocumentError; use crate::error::fee::FeeError; +use crate::fee::result::FeeResult; use dpp::data_contract::extra::{DriveContractExt, IndexLevel}; impl Drive { diff --git a/packages/rs-drive/src/drive/document/update.rs b/packages/rs-drive/src/drive/document/update.rs index 793bbee066b..50f54289b52 100644 --- a/packages/rs-drive/src/drive/document/update.rs +++ b/packages/rs-drive/src/drive/document/update.rs @@ -57,8 +57,8 @@ use crate::drive::object_size_info::{DocumentAndContractInfo, DriveKeyInfo, Path use crate::drive::Drive; use crate::error::drive::DriveError; use crate::error::Error; +use crate::fee::calculate_fee; use crate::fee::op::DriveOperation; -use crate::fee::{calculate_fee, FeeResult}; use crate::drive::block_info::BlockInfo; use crate::drive::object_size_info::DriveKeyInfo::{Key, KeyRef, KeySize}; @@ -68,6 +68,7 @@ use crate::drive::grove_operations::{ BatchDeleteUpTreeApplyType, BatchInsertApplyType, BatchInsertTreeApplyType, DirectQueryType, QueryType, }; +use crate::fee::result::FeeResult; use dpp::data_contract::extra::DriveContractExt; impl Drive { diff --git a/packages/rs-drive/src/drive/fee_pools/epochs/credit_distribution_pools.rs b/packages/rs-drive/src/drive/fee_pools/epochs/credit_distribution_pools.rs index 5e4d47bbbf2..00f8954512c 100644 --- a/packages/rs-drive/src/drive/fee_pools/epochs/credit_distribution_pools.rs +++ b/packages/rs-drive/src/drive/fee_pools/epochs/credit_distribution_pools.rs @@ -37,6 +37,7 @@ use grovedb::{Element, TransactionArg}; use crate::drive::Drive; use crate::error::fee::FeeError; use crate::error::Error; +use crate::fee::get_overflow_error; use crate::fee_pools::epochs::Epoch; use crate::fee_pools::epochs::epoch_key_constants; @@ -149,9 +150,7 @@ impl Drive { storage_pool_credits .checked_add(processing_pool_credits) - .ok_or(Error::Fee(FeeError::Overflow( - "overflow getting total credits for distribution", - ))) + .ok_or_else(|| get_overflow_error("overflow getting total credits for distribution")) } } @@ -268,9 +267,9 @@ mod tests { let mut batch = GroveDbOpBatch::new(); - batch.push(epoch.update_processing_credits_for_distribution_operation(processing_fee)); + batch.push(epoch.update_processing_fee_pool_operation(processing_fee)); - batch.push(epoch.update_storage_credits_for_distribution_operation(storage_fee)); + batch.push(epoch.update_storage_fee_pool_operation(storage_fee)); drive .grove_apply_batch(batch, false, Some(&transaction)) diff --git a/packages/rs-drive/src/drive/fee_pools/epochs/mod.rs b/packages/rs-drive/src/drive/fee_pools/epochs/mod.rs index 6451e3f5393..ff07c475330 100644 --- a/packages/rs-drive/src/drive/fee_pools/epochs/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/epochs/mod.rs @@ -36,8 +36,6 @@ use crate::error::Error; use crate::fee_pools::epochs::Epoch; use grovedb::TransactionArg; -/// Constants module -pub mod constants; pub mod credit_distribution_pools; pub mod proposers; pub mod start_block; @@ -59,13 +57,14 @@ impl Drive { #[cfg(test)] mod tests { + use super::*; + + use crate::common::helpers::setup::setup_drive_with_initial_state_structure; mod is_epoch_tree_exists { - use crate::common::helpers::setup::setup_drive_with_initial_state_structure; - use crate::drive::fee_pools::epochs::constants::{ - GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS, - }; - use crate::fee_pools::epochs::Epoch; + use super::*; + + use crate::fee::epoch::{GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; #[test] fn test_return_true_if_tree_exists() { diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index e6b462f86f4..bfa0bf17f5a 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -27,14 +27,23 @@ // DEALINGS IN THE SOFTWARE. // -use crate::drive::RootTree; +use crate::drive::batch::GroveDbOpBatch; +use crate::drive::{Drive, RootTree}; +use crate::error::drive::DriveError; +use crate::error::Error; +use crate::fee::epoch::CreditsPerEpoch; +use crate::fee_pools::epochs::epoch_key_constants::KEY_POOL_STORAGE_FEES; +use crate::fee_pools::epochs::{paths, Epoch}; use crate::fee_pools::epochs_root_tree_key_constants::{ KEY_PENDING_POOL_UPDATES, KEY_STORAGE_FEE_POOL, }; +use crate::query::QueryItem; +use grovedb::{PathQuery, Query, TransactionArg}; +use itertools::Itertools; /// Epochs module pub mod epochs; -pub mod pending_updates; +pub mod pending_epoch_updates; pub mod storage_fee_distribution_pool; pub mod unpaid_epoch; @@ -69,6 +78,66 @@ pub fn aggregate_storage_fees_distribution_pool_vec_path() -> Vec> { vec![vec![RootTree::Pools as u8], KEY_STORAGE_FEE_POOL.to_vec()] } +impl Drive { + pub fn add_update_epoch_storage_fee_pools_operations( + &self, + batch: &mut GroveDbOpBatch, + credits_per_epochs: CreditsPerEpoch, + transaction: TransactionArg, + ) -> Result<(), Error> { + if credits_per_epochs.len() == 0 { + return Ok(()); + } + + let min_epoch_index_key = credits_per_epochs.keys().min().ok_or(Error::Drive( + DriveError::CorruptedCodeExecution("can't find min epoch index"), + ))?; + let min_epoch_index = min_epoch_index_key.to_owned() as u16; + let min_encoded_epoch_index = paths::encode_epoch_index_key(min_epoch_index)?.to_vec(); + + let max_epoch_index_key = credits_per_epochs.keys().max().ok_or(Error::Drive( + DriveError::CorruptedCodeExecution("can't find max epoch index"), + ))?; + let max_epoch_index = max_epoch_index_key.to_owned() as u16; + let max_encoded_epoch_index = paths::encode_epoch_index_key(max_epoch_index)?.to_vec(); + + let mut storage_fee_pool_query = Query::new(); + storage_fee_pool_query.insert_key(KEY_POOL_STORAGE_FEES.to_vec()); + + let mut epochs_query = Query::new(); + + epochs_query.insert_range_inclusive(min_encoded_epoch_index..=max_encoded_epoch_index); + + epochs_query.set_subquery(storage_fee_pool_query); + + let (storage_fee_pools, _) = self + .grove + .query( + &PathQuery::new_unsized(pools_vec_path(), epochs_query), + transaction, + ) + .unwrap() + .map_err(Error::GroveDB)?; + + if storage_fee_pools.len() != credits_per_epochs.len() { + return Err(Error::Drive(DriveError::CorruptedCodeExecution( + "number of fetched epoch storage fee pools must be equal to requested for update", + ))); + } + + for (i, (epoch_index_key, credits)) in credits_per_epochs + .into_iter() + .sorted_by_key(|x| x.0) + .enumerate() + { + let encoded_epoch_index_key = + paths::encode_epoch_index_key(epoch_index_key as u16)?.to_vec(); + } + + Ok(()) + } +} + #[cfg(test)] mod tests { use crate::common::helpers::setup::setup_drive_with_initial_state_structure; diff --git a/packages/rs-drive/src/drive/fee_pools/pending_updates.rs b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs similarity index 87% rename from packages/rs-drive/src/drive/fee_pools/pending_updates.rs rename to packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs index 6556d0b1ef8..c7a6d9e77d3 100644 --- a/packages/rs-drive/src/drive/fee_pools/pending_updates.rs +++ b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs @@ -35,14 +35,15 @@ use crate::drive::Drive; use crate::error::drive::DriveError; use crate::error::fee::FeeError; use crate::error::Error; -use crate::fee::refunds::CreditsPerEpoch; +use crate::fee::epoch::CreditsPerEpoch; +use crate::fee::get_overflow_error; use grovedb::query_result_type::QueryResultType; use grovedb::{Element, PathQuery, Query, TransactionArg}; impl Drive { /// Fetches existing pending epoch pool updates using specified epochs /// and returns merged result - pub fn fetch_and_merge_with_existing_pending_epoch_pool_updates( + pub fn fetch_and_merge_with_existing_pending_epoch_storage_pool_updates( &self, mut credits_per_epoch: CreditsPerEpoch, transaction: TransactionArg, @@ -113,7 +114,7 @@ impl Drive { } /// Adds operations to delete pending epoch pool updates except specified epochs - pub fn add_delete_pending_epoch_pool_updates_except_specified_operations( + pub fn add_delete_pending_epoch_storage_pool_updates_except_specified_operations( &self, batch: &mut GroveDbOpBatch, credits_per_epoch: &CreditsPerEpoch, @@ -138,7 +139,7 @@ impl Drive { let epoch_index = u16::from_be_bytes(encoded_epoch_index.as_slice().try_into().map_err(|_| { Error::Drive(DriveError::CorruptedSerialization( - "epoch index for pending pool updates must be i64", + "epoch index for pending pool updates must be u16", )) })?); @@ -155,21 +156,25 @@ impl Drive { } } -/// Returns the index of the unpaid Epoch. -pub fn add_update_pending_epoch_pool_update_operations( +/// Adds GroveDB batch operations to update pending epoch storage pool updates +pub fn add_update_pending_epoch_storage_pool_update_operations( batch: &mut GroveDbOpBatch, credits_per_epoch: CreditsPerEpoch, -) { +) -> Result<(), Error> { for (epoch_index_key, credits) in credits_per_epoch { let epoch_index = epoch_index_key as u16; let encoded_epoch_index = epoch_index.to_be_bytes().to_vec(); - let signed_credits = -(credits as i64); + let signed_credits = -i64::try_from(credits).map_err(|_| { + get_overflow_error("can't convert credits to negative amount for pending updates") + })?; let element = Element::new_item(signed_credits.to_be_bytes().to_vec()); batch.add_insert(pools_pending_updates_path(), encoded_epoch_index, element); } + + Ok(()) } #[cfg(test)] @@ -177,7 +182,7 @@ mod tests { use super::*; use crate::common::helpers::setup::setup_drive_with_initial_state_structure; - mod fetch_and_merge_with_existing_pending_epoch_pool_updates { + mod fetch_and_merge_with_existing_pending_epoch_storage_pool_updates { use super::*; #[test] @@ -193,7 +198,11 @@ mod tests { let mut batch = GroveDbOpBatch::new(); - add_update_pending_epoch_pool_update_operations(&mut batch, initial_pending_updates); + add_update_pending_epoch_storage_pool_update_operations( + &mut batch, + initial_pending_updates, + ) + .expect("should update pending epoch updates"); drive .grove_apply_batch(batch, false, Some(&transaction)) @@ -205,7 +214,7 @@ mod tests { CreditsPerEpoch::from_iter(vec![(1, 15), (3, 25), (30, 195), (41, 150)]); let updated_pending_updates = drive - .fetch_and_merge_with_existing_pending_epoch_pool_updates( + .fetch_and_merge_with_existing_pending_epoch_storage_pool_updates( new_pending_updates, Some(&transaction), ) @@ -218,7 +227,7 @@ mod tests { } } - mod add_delete_pending_epoch_pool_updates_except_specified_operations { + mod add_delete_pending_epoch_storage_pool_updates_except_specified_operations { use super::*; use grovedb::batch::Op; @@ -235,7 +244,11 @@ mod tests { let mut batch = GroveDbOpBatch::new(); - add_update_pending_epoch_pool_update_operations(&mut batch, initial_pending_updates); + add_update_pending_epoch_storage_pool_update_operations( + &mut batch, + initial_pending_updates, + ) + .expect("should update pending epoch updates"); drive .grove_apply_batch(batch, false, Some(&transaction)) @@ -248,7 +261,7 @@ mod tests { let mut batch = GroveDbOpBatch::new(); drive - .add_delete_pending_epoch_pool_updates_except_specified_operations( + .add_delete_pending_epoch_storage_pool_updates_except_specified_operations( &mut batch, &new_pending_updates, Some(&transaction), diff --git a/packages/rs-drive/src/drive/identity/mod.rs b/packages/rs-drive/src/drive/identity/mod.rs index 8c28b9ee53b..31e38e1baa5 100644 --- a/packages/rs-drive/src/drive/identity/mod.rs +++ b/packages/rs-drive/src/drive/identity/mod.rs @@ -47,8 +47,9 @@ use crate::drive::{Drive, RootTree}; use crate::error::drive::DriveError; use crate::error::identity::IdentityError; use crate::error::Error; +use crate::fee::calculate_fee; use crate::fee::op::DriveOperation; -use crate::fee::{calculate_fee, FeeResult}; +use crate::fee::result::FeeResult; pub mod withdrawal_queue; diff --git a/packages/rs-drive/src/drive/mod.rs b/packages/rs-drive/src/drive/mod.rs index 56bb09e1b22..386c260d84a 100644 --- a/packages/rs-drive/src/drive/mod.rs +++ b/packages/rs-drive/src/drive/mod.rs @@ -73,12 +73,10 @@ mod test_utils; use crate::drive::block_info::BlockInfo; use crate::drive::cache::{DataContractCache, DriveCache}; -use crate::fee::FeeResult; +use crate::fee::result::FeeResult; use crate::fee_pools::epochs::Epoch; use dpp::data_contract::extra::DriveContractExt; -type TransactionPointerAddress = usize; - /// Drive struct pub struct Drive { /// GroveDB diff --git a/packages/rs-drive/src/fee/default_costs.rs b/packages/rs-drive/src/fee/default_costs.rs index fb774147897..1cb6c6ca051 100644 --- a/packages/rs-drive/src/fee/default_costs.rs +++ b/packages/rs-drive/src/fee/default_costs.rs @@ -1,3 +1,37 @@ +// MIT LICENSE +// +// Copyright (c) 2021 Dash Core Group +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +//! Fee pool constants. +//! +//! This module defines constants related to fee distribution pools. +//! + /// Storage disk usage credit per byte pub(crate) const STORAGE_DISK_USAGE_CREDIT_PER_BYTE: u64 = 27000; /// Storage processing credit per byte diff --git a/packages/rs-drive-abci/src/execution/fee_pools/constants.rs b/packages/rs-drive/src/fee/epoch/distribution.rs similarity index 56% rename from packages/rs-drive-abci/src/execution/fee_pools/constants.rs rename to packages/rs-drive/src/fee/epoch/distribution.rs index 1472b31d097..184e518e13f 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/constants.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -32,13 +32,14 @@ //! This module defines constants related to fee distribution pools. //! -use drive::drive::fee_pools::epochs::constants::PERPETUAL_STORAGE_YEARS; +use crate::error::fee::FeeError; +use crate::error::Error; +use crate::fee::epoch::{CreditsPerEpoch, EpochIndex, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS}; +use crate::fee::{get_overflow_error, Credits}; +use rust_decimal::prelude::*; use rust_decimal::Decimal; use rust_decimal_macros::dec; -/// Default original fee multiplier -pub const DEFAULT_ORIGINAL_FEE_MULTIPLIER: f64 = 2.0; - // TODO: Should be updated from the doc /// The amount of the perpetual storage fee to be paid out to masternodes per year. Adds up to 1. @@ -56,6 +57,61 @@ pub const FEE_DISTRIBUTION_TABLE: [Decimal; PERPETUAL_STORAGE_YEARS as usize] = dec!(0.00325), dec!(0.00275), dec!(0.00225), dec!(0.00175), dec!(0.00125), ]; +pub type DistributionLeftoverCredits = Credits; + +pub fn distribute_storage_fee_to_epochs( + storage_fee: u64, + start_epoch_index: EpochIndex, + credits_per_epochs: Option, +) -> Result { + let storage_fee_dec = Decimal::from_u64(storage_fee).ok_or(Error::Fee( + FeeError::CorruptedCodeExecution("storage fees are not fitting in a Decimal"), + ))?; + + let mut distribution_leftover_credits = storage_fee; + + let mut credits_per_epochs = credits_per_epochs.unwrap_or_default(); + + let epochs_per_year = Decimal::from(EPOCHS_PER_YEAR); + + for year in 0..PERPETUAL_STORAGE_YEARS { + let distribution_for_that_year_ratio = FEE_DISTRIBUTION_TABLE[year as usize]; + + let year_fee_share = storage_fee_dec * distribution_for_that_year_ratio; + + let epoch_fee_share_dec = year_fee_share / epochs_per_year; + + let epoch_fee_share = epoch_fee_share_dec + .floor() + .to_u64() + .ok_or_else(|| get_overflow_error("storage fees are not fitting in a u64"))?; + + let year_start_epoch_index = start_epoch_index + EPOCHS_PER_YEAR * year; + + for epoch_index in year_start_epoch_index..year_start_epoch_index + EPOCHS_PER_YEAR { + let epoch_index_key = epoch_index as u64; + + let current_epoch_credits = credits_per_epochs + .get(epoch_index_key) + .map_or(0, |i| i.to_owned()); + + let result_storage_fee = current_epoch_credits + .checked_add(epoch_fee_share) + .ok_or_else(|| get_overflow_error("storage fees are not fitting in a u64"))?; + + credits_per_epochs.insert(epoch_index_key, result_storage_fee); + + distribution_leftover_credits = distribution_leftover_credits + .checked_sub(epoch_fee_share) + .ok_or(Error::Fee(FeeError::CorruptedCodeExecution( + "leftovers bigger than initial value", + )))?; + } + } + + Ok(distribution_leftover_credits) +} + #[cfg(test)] mod tests { use rust_decimal::Decimal; diff --git a/packages/rs-drive/src/drive/fee_pools/epochs/constants.rs b/packages/rs-drive/src/fee/epoch/mod.rs similarity index 70% rename from packages/rs-drive/src/drive/fee_pools/epochs/constants.rs rename to packages/rs-drive/src/fee/epoch/mod.rs index 20337de3e63..d5e26ff2d3f 100644 --- a/packages/rs-drive/src/drive/fee_pools/epochs/constants.rs +++ b/packages/rs-drive/src/fee/epoch/mod.rs @@ -1,3 +1,8 @@ +use crate::fee::Credits; +use intmap::IntMap; + +mod distribution; + /// Genesis epoch index pub const GENESIS_EPOCH_INDEX: u16 = 0; @@ -9,3 +14,7 @@ pub const PERPETUAL_STORAGE_YEARS: u16 = 50; /// Perpetual storage epochs pub const PERPETUAL_STORAGE_EPOCHS: u16 = PERPETUAL_STORAGE_YEARS * EPOCHS_PER_YEAR; + +pub type CreditsPerEpoch = IntMap; + +pub type EpochIndex = u16; diff --git a/packages/rs-drive/src/fee/mod.rs b/packages/rs-drive/src/fee/mod.rs index ee4c09a0df2..b75c0972c1d 100644 --- a/packages/rs-drive/src/fee/mod.rs +++ b/packages/rs-drive/src/fee/mod.rs @@ -32,28 +32,18 @@ use enum_map::EnumMap; use crate::error::fee::FeeError; use crate::error::Error; use crate::fee::op::{BaseOp, DriveOperation}; -use crate::fee::refunds::FeeRefunds; +use crate::fee::result::FeeResult; use crate::fee_pools::epochs::Epoch; -/// Default costs module pub mod default_costs; +pub mod epoch; pub mod op; +pub mod result; -/// Fee result refunds -pub mod refunds; +/// Default original fee multiplier +pub const DEFAULT_ORIGINAL_FEE_MULTIPLIER: f64 = 2.0; -/// Fee Result -#[derive(Debug, Clone, Eq, PartialEq, Default)] -pub struct FeeResult { - /// Storage fee - pub storage_fee: u64, - /// Processing fee - pub processing_fee: u64, - /// Credits to refund to identities - pub fee_refunds: FeeRefunds, - /// Removed bytes not needing to be refunded to identities - pub removed_bytes_from_system: u32, -} +pub type Credits = u64; /// Calculates fees for the given operations. Returns the storage and processing costs. pub fn calculate_fee( @@ -84,35 +74,6 @@ pub fn calculate_fee( Ok(aggregate_fee_result) } -impl FeeResult { - /// Creates a FeeResult instance with specified storage and processing fees - pub fn from_fees(storage_fee: u64, processing_fee: u64) -> Self { - FeeResult { - storage_fee, - processing_fee, - ..Default::default() - } - } - - /// Adds and self assigns result between two Fee Results - pub fn checked_add_assign(&mut self, rhs: Self) -> Result<(), Error> { - self.storage_fee = self - .storage_fee - .checked_add(rhs.storage_fee) - .ok_or(Error::Fee(FeeError::Overflow("storage fee overflow error")))?; - self.processing_fee = - self.processing_fee - .checked_add(rhs.processing_fee) - .ok_or(Error::Fee(FeeError::Overflow( - "processing fee overflow error", - )))?; - self.fee_refunds.checked_add_assign(rhs.fee_refunds)?; - self.removed_bytes_from_system = self - .removed_bytes_from_system - .checked_add(rhs.removed_bytes_from_system) - .ok_or(Error::Fee(FeeError::Overflow( - "removed_bytes_from_system overflow error", - )))?; - Ok(()) - } +pub(crate) fn get_overflow_error(str: &'static str) -> Error { + Error::Fee(FeeError::Overflow(str)) } diff --git a/packages/rs-drive/src/fee/op.rs b/packages/rs-drive/src/fee/op.rs index 804b771b7b9..648751bee7a 100644 --- a/packages/rs-drive/src/fee/op.rs +++ b/packages/rs-drive/src/fee/op.rs @@ -54,8 +54,8 @@ use crate::fee::default_costs::{ use crate::fee::op::DriveOperation::{ CalculatedCostOperation, GroveOperation, PreCalculatedFeeResult, }; -use crate::fee::refunds::FeeRefunds; -use crate::fee::FeeResult; +use crate::fee::result::refunds::FeeRefunds; +use crate::fee::{get_overflow_error, FeeResult}; use crate::fee_pools::epochs::Epoch; /// Base ops @@ -304,10 +304,6 @@ pub trait DriveCost { fn storage_cost(&self, epoch: &Epoch) -> Result; } -pub(crate) fn get_overflow_error(str: &'static str) -> Error { - Error::Fee(FeeError::Overflow(str)) -} - impl DriveCost for OperationCost { /// Return the ephemeral cost from the operation fn ephemeral_cost(&self, epoch: &Epoch) -> Result { diff --git a/packages/rs-drive/src/fee/result/mod.rs b/packages/rs-drive/src/fee/result/mod.rs new file mode 100644 index 00000000000..fc2b4fedee0 --- /dev/null +++ b/packages/rs-drive/src/fee/result/mod.rs @@ -0,0 +1,86 @@ +// MIT LICENSE +// +// Copyright (c) 2021 Dash Core Group +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +//! Fee pool constants. +//! +//! This module defines constants related to fee distribution pools. +//! + +use crate::error::fee::FeeError; +use crate::error::Error; +use crate::fee::result::refunds::FeeRefunds; +use crate::fee::Credits; + +pub mod refunds; + +/// Fee Result +#[derive(Debug, Clone, Eq, PartialEq, Default)] +pub struct FeeResult { + /// Storage fee + pub storage_fee: Credits, + /// Processing fee + pub processing_fee: Credits, + /// Credits to refund to identities + pub fee_refunds: FeeRefunds, + /// Removed bytes not needing to be refunded to identities + pub removed_bytes_from_system: u32, +} + +impl FeeResult { + /// Creates a FeeResult instance with specified storage and processing fees + pub fn from_fees(storage_fee: Credits, processing_fee: Credits) -> Self { + FeeResult { + storage_fee, + processing_fee, + ..Default::default() + } + } + + /// Adds and self assigns result between two Fee Results + pub fn checked_add_assign(&mut self, rhs: Self) -> Result<(), Error> { + self.storage_fee = self + .storage_fee + .checked_add(rhs.storage_fee) + .ok_or(Error::Fee(FeeError::Overflow("storage fee overflow error")))?; + self.processing_fee = + self.processing_fee + .checked_add(rhs.processing_fee) + .ok_or(Error::Fee(FeeError::Overflow( + "processing fee overflow error", + )))?; + self.fee_refunds.checked_add_assign(rhs.fee_refunds)?; + self.removed_bytes_from_system = self + .removed_bytes_from_system + .checked_add(rhs.removed_bytes_from_system) + .ok_or(Error::Fee(FeeError::Overflow( + "removed_bytes_from_system overflow error", + )))?; + Ok(()) + } +} diff --git a/packages/rs-drive/src/fee/refunds.rs b/packages/rs-drive/src/fee/result/refunds.rs similarity index 76% rename from packages/rs-drive/src/fee/refunds.rs rename to packages/rs-drive/src/fee/result/refunds.rs index 35ddf57c85f..064ca2f15a1 100644 --- a/packages/rs-drive/src/fee/refunds.rs +++ b/packages/rs-drive/src/fee/result/refunds.rs @@ -1,17 +1,48 @@ +// MIT LICENSE +// +// Copyright (c) 2021 Dash Core Group +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +//! Fee pool constants. +//! +//! This module defines constants related to fee distribution pools. +//! + use crate::error::fee::FeeError; use crate::error::Error; use crate::fee::default_costs::STORAGE_DISK_USAGE_CREDIT_PER_BYTE; -use crate::fee::op::get_overflow_error; +use crate::fee::epoch::CreditsPerEpoch; +use crate::fee::get_overflow_error; use bincode::Options; use costs::storage_cost::removal::{Identifier, StorageRemovalPerEpochByIdentifier}; -use intmap::IntMap; use serde::{Deserialize, Serialize}; use std::collections::btree_map::{IntoIter, Iter}; use std::collections::BTreeMap; -/// Credits per Epoch -pub type CreditsPerEpoch = IntMap; - /// Credits per Epoch by Identifier pub type CreditsPerEpochByIdentifier = BTreeMap; @@ -30,6 +61,7 @@ impl FeeRefunds { bytes_per_epochs .into_iter() .map(|(epoch_index, bytes)| { + // TODO We should use multipliers (bytes as u64) .checked_mul(STORAGE_DISK_USAGE_CREDIT_PER_BYTE) .ok_or_else(|| { @@ -66,7 +98,7 @@ impl FeeRefunds { } else { int_map_b }; - // reinsert the now combined intmap + // reinsert the now combined IntMap self.0.insert(identifier, to_insert_int_map); } Ok(()) @@ -82,7 +114,7 @@ impl FeeRefunds { self.0.iter() } - /// Passthrough method for into interation + /// Passthrough method for into iteration pub fn into_iter(self) -> IntoIter { self.0.into_iter() } diff --git a/packages/rs-drive/src/fee_pools/epochs/operations_factory.rs b/packages/rs-drive/src/fee_pools/epochs/operations_factory.rs index 60bd0008d31..a0515141e67 100644 --- a/packages/rs-drive/src/fee_pools/epochs/operations_factory.rs +++ b/packages/rs-drive/src/fee_pools/epochs/operations_factory.rs @@ -79,7 +79,7 @@ impl Epoch { self.add_init_empty_without_storage_operations(batch); // init storage fee item to 0 - batch.push(self.update_storage_credits_for_distribution_operation(0)); + batch.push(self.update_storage_fee_pool_operation(0)); } /// Adds to the groveDB op batch initialization operations for the epoch. @@ -136,10 +136,7 @@ impl Epoch { } /// Returns a groveDB op which updates the epoch processing credits for distribution. - pub fn update_processing_credits_for_distribution_operation( - &self, - processing_fee: u64, - ) -> GroveDbOp { + pub fn update_processing_fee_pool_operation(&self, processing_fee: u64) -> GroveDbOp { GroveDbOp::insert_op( self.get_vec_path(), KEY_POOL_PROCESSING_FEES.to_vec(), @@ -153,7 +150,7 @@ impl Epoch { } /// Returns a groveDB op which updates the epoch storage credits for distribution. - pub fn update_storage_credits_for_distribution_operation(&self, storage_fee: u64) -> GroveDbOp { + pub fn update_storage_fee_pool_operation(&self, storage_fee: u64) -> GroveDbOp { GroveDbOp::insert_op( self.get_vec_path(), KEY_POOL_STORAGE_FEES.to_vec(), @@ -562,7 +559,7 @@ mod tests { let epoch = super::Epoch::new(7000); - let op = epoch.update_processing_credits_for_distribution_operation(42); + let op = epoch.update_processing_fee_pool_operation(42); match drive.grove_apply_operation(op, false, Some(&transaction)) { Ok(_) => assert!( @@ -587,7 +584,7 @@ mod tests { let processing_fee: u64 = 42; - let op = epoch.update_processing_credits_for_distribution_operation(42); + let op = epoch.update_processing_fee_pool_operation(42); drive .grove_apply_operation(op, false, Some(&transaction)) @@ -611,7 +608,7 @@ mod tests { let epoch = super::Epoch::new(7000); - let op = epoch.update_storage_credits_for_distribution_operation(42); + let op = epoch.update_storage_fee_pool_operation(42); match drive.grove_apply_operation(op, false, Some(&transaction)) { Ok(_) => assert!( @@ -636,7 +633,7 @@ mod tests { let storage_fee = 42; - let op = epoch.update_storage_credits_for_distribution_operation(storage_fee); + let op = epoch.update_storage_fee_pool_operation(storage_fee); drive .grove_apply_operation(op, false, Some(&transaction)) diff --git a/packages/rs-drive/src/fee_pools/mod.rs b/packages/rs-drive/src/fee_pools/mod.rs index a86b87b9fc3..537b742164f 100644 --- a/packages/rs-drive/src/fee_pools/mod.rs +++ b/packages/rs-drive/src/fee_pools/mod.rs @@ -28,8 +28,9 @@ // use crate::drive::batch::GroveDbOpBatch; -use crate::drive::fee_pools::epochs::constants::{GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; use crate::drive::fee_pools::pools_vec_path; +use crate::fee::epoch::{EpochIndex, GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; +use crate::fee::Credits; use crate::fee_pools::epochs::Epoch; use crate::fee_pools::epochs_root_tree_key_constants::{ KEY_PENDING_POOL_UPDATES, KEY_STORAGE_FEE_POOL, KEY_UNPAID_EPOCH_INDEX, @@ -66,7 +67,7 @@ pub fn add_create_pending_pool_updates_tree_operations(batch: &mut GroveDbOpBatc } /// Updates the storage fee distribution pool with a new storage fee -pub fn update_storage_fee_distribution_pool_operation(storage_fee: u64) -> GroveDbOp { +pub fn update_storage_fee_distribution_pool_operation(storage_fee: Credits) -> GroveDbOp { GroveDbOp::insert_op( pools_vec_path(), KEY_STORAGE_FEE_POOL.to_vec(), @@ -75,7 +76,7 @@ pub fn update_storage_fee_distribution_pool_operation(storage_fee: u64) -> Grove } /// Updates the unpaid epoch index -pub fn update_unpaid_epoch_index_operation(epoch_index: u16) -> GroveDbOp { +pub fn update_unpaid_epoch_index_operation(epoch_index: EpochIndex) -> GroveDbOp { GroveDbOp::insert_op( pools_vec_path(), KEY_UNPAID_EPOCH_INDEX.to_vec(), From 2c29e6f7096280b5455f2c9780f74cdf59e8092b Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 16 Dec 2022 02:27:08 +0800 Subject: [PATCH 07/54] fix: non deterministic updates --- packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs index c7a6d9e77d3..5186a6b9e35 100644 --- a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs +++ b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs @@ -39,6 +39,7 @@ use crate::fee::epoch::CreditsPerEpoch; use crate::fee::get_overflow_error; use grovedb::query_result_type::QueryResultType; use grovedb::{Element, PathQuery, Query, TransactionArg}; +use itertools::Itertools; impl Drive { /// Fetches existing pending epoch pool updates using specified epochs @@ -161,7 +162,7 @@ pub fn add_update_pending_epoch_storage_pool_update_operations( batch: &mut GroveDbOpBatch, credits_per_epoch: CreditsPerEpoch, ) -> Result<(), Error> { - for (epoch_index_key, credits) in credits_per_epoch { + for (epoch_index_key, credits) in credits_per_epoch.into_iter().sorted_by_key(|x| x.0) { let epoch_index = epoch_index_key as u16; let encoded_epoch_index = epoch_index.to_be_bytes().to_vec(); From f0cb2f5b818fb020de74951a18ce72f447cf3eb7 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 16 Dec 2022 19:02:38 +0800 Subject: [PATCH 08/54] refactor: change to HashMap with nohasher --- Cargo.lock | 5 +---- .../fee_pools/distribute_storage_pool.rs | 11 ++++++----- .../src/execution/fee_pools/process_block_fees.rs | 2 +- packages/rs-drive/Cargo.toml | 2 +- .../src/drive/fee_pools/pending_epoch_updates.rs | 15 +++++---------- packages/rs-drive/src/fee/epoch/distribution.rs | 6 ++---- packages/rs-drive/src/fee/epoch/mod.rs | 6 +++--- packages/rs-drive/src/fee/result/refunds.rs | 7 +++++-- 8 files changed, 24 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 858646824f2..2d5dc63d65e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -915,9 +915,9 @@ dependencies = [ "hex", "indexmap", "integer-encoding", - "intmap", "itertools", "moka", + "nohash-hasher", "rand 0.8.5", "rand_distr", "rust_decimal", @@ -1433,9 +1433,6 @@ name = "intmap" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee87fd093563344074bacf24faa0bb0227fb6969fb223e922db798516de924d6" -dependencies = [ - "serde", -] [[package]] name = "iso8601" diff --git a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs index 561f2de8e3a..b65d3420711 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs @@ -39,6 +39,7 @@ use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; use drive::drive::fee_pools::epochs::constants::{EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS}; use drive::fee::constants; +use drive::fee::epoch::{EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS}; use drive::fee_pools::epochs::Epoch; use drive::grovedb::TransactionArg; use drive::{error, grovedb}; @@ -51,7 +52,7 @@ pub type StorageDistributionLeftoverCredits = u64; impl Platform { /// Adds operations to the GroveDB op batch which calculate and distribute storage fees /// from the distribution pool to the epoch pools and returns the leftovers. - pub fn add_distribute_storage_fee_distribution_pool_to_epochs_operations( + pub fn add_distribute_storage_fee_to_epochs_operations( &self, current_epoch_index: u16, transaction: TransactionArg, @@ -150,7 +151,7 @@ mod tests { let mut batch = GroveDbOpBatch::new(); platform - .add_distribute_storage_fee_distribution_pool_to_epochs_operations( + .add_distribute_storage_fee_to_epochs_operations( epoch_index, Some(&transaction), &mut batch, @@ -200,7 +201,7 @@ mod tests { let mut batch = GroveDbOpBatch::new(); let leftovers = platform - .add_distribute_storage_fee_distribution_pool_to_epochs_operations( + .add_distribute_storage_fee_to_epochs_operations( epoch_index, Some(&transaction), &mut batch, @@ -243,7 +244,7 @@ mod tests { let mut batch = GroveDbOpBatch::new(); let leftovers = platform - .add_distribute_storage_fee_distribution_pool_to_epochs_operations( + .add_distribute_storage_fee_to_epochs_operations( epoch_index, Some(&transaction), &mut batch, @@ -363,7 +364,7 @@ mod tests { // distribute fees once more platform - .add_distribute_storage_fee_distribution_pool_to_epochs_operations( + .add_distribute_storage_fee_to_epochs_operations( epoch_index, Some(&transaction), &mut batch, diff --git a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs index 55d7ecfc447..8f87672c5f6 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs @@ -116,7 +116,7 @@ impl Platform { // Distribute storage fees accumulated during previous epoch let storage_distribution_leftover_credits = self - .add_distribute_storage_fee_distribution_pool_to_epochs_operations( + .add_distribute_storage_fee_to_epochs_operations( current_epoch.index, transaction, batch, diff --git a/packages/rs-drive/Cargo.toml b/packages/rs-drive/Cargo.toml index af61e3dd4d2..e688515eb0b 100644 --- a/packages/rs-drive/Cargo.toml +++ b/packages/rs-drive/Cargo.toml @@ -24,7 +24,7 @@ sqlparser = "0.13.0" enum-map = "2.0.3" thiserror = "1.0.30" moka = "0.8.1" -intmap = { version="2.0.0", features=["serde"] } +nohash-hasher = "0.2.0" chrono = "0.4.20" bincode = "1.3.3" dpp = { path = "../rs-dpp" } diff --git a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs index 5186a6b9e35..8963802426b 100644 --- a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs +++ b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs @@ -74,13 +74,11 @@ impl Drive { let epoch_index = u16::from_be_bytes(encoded_epoch_index.as_slice().try_into().map_err(|_| { Error::Drive(DriveError::CorruptedSerialization( - "epoch index for pending pool updates must be u64", + "epoch index for pending pool updates must be u16", )) })?); - let epoch_index_key = epoch_index as u64; - - let Some(credits_to_update) = credits_per_epoch.get(epoch_index_key) else { + let Some(credits_to_update) = credits_per_epoch.get(&epoch_index) else { return Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates should contain fetched epochs"))); }; @@ -108,7 +106,7 @@ impl Drive { "pending updates overflow error", )))?; - credits_per_epoch.insert(epoch_index_key, result_credits); + credits_per_epoch.insert(epoch_index, result_credits); } Ok(credits_per_epoch) @@ -144,9 +142,7 @@ impl Drive { )) })?); - let epoch_index_key = epoch_index as u64; - - if credits_per_epoch.contains_key(epoch_index_key) { + if credits_per_epoch.contains_key(&epoch_index) { continue; } @@ -285,9 +281,8 @@ mod tests { .try_into() .expect("should convert to u16 bytes"), ); - let epoch_index_key = epoch_index as u64; - assert!(expected_pending_updates.contains_key(epoch_index_key)); + assert!(expected_pending_updates.contains_key(&epoch_index)); } } } diff --git a/packages/rs-drive/src/fee/epoch/distribution.rs b/packages/rs-drive/src/fee/epoch/distribution.rs index 184e518e13f..b63f97494e6 100644 --- a/packages/rs-drive/src/fee/epoch/distribution.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -89,17 +89,15 @@ pub fn distribute_storage_fee_to_epochs( let year_start_epoch_index = start_epoch_index + EPOCHS_PER_YEAR * year; for epoch_index in year_start_epoch_index..year_start_epoch_index + EPOCHS_PER_YEAR { - let epoch_index_key = epoch_index as u64; - let current_epoch_credits = credits_per_epochs - .get(epoch_index_key) + .get(&epoch_index) .map_or(0, |i| i.to_owned()); let result_storage_fee = current_epoch_credits .checked_add(epoch_fee_share) .ok_or_else(|| get_overflow_error("storage fees are not fitting in a u64"))?; - credits_per_epochs.insert(epoch_index_key, result_storage_fee); + credits_per_epochs.insert(epoch_index, result_storage_fee); distribution_leftover_credits = distribution_leftover_credits .checked_sub(epoch_fee_share) diff --git a/packages/rs-drive/src/fee/epoch/mod.rs b/packages/rs-drive/src/fee/epoch/mod.rs index d5e26ff2d3f..1fa5df5b4e8 100644 --- a/packages/rs-drive/src/fee/epoch/mod.rs +++ b/packages/rs-drive/src/fee/epoch/mod.rs @@ -1,5 +1,5 @@ use crate::fee::Credits; -use intmap::IntMap; +use nohash_hasher::IntMap; mod distribution; @@ -15,6 +15,6 @@ pub const PERPETUAL_STORAGE_YEARS: u16 = 50; /// Perpetual storage epochs pub const PERPETUAL_STORAGE_EPOCHS: u16 = PERPETUAL_STORAGE_YEARS * EPOCHS_PER_YEAR; -pub type CreditsPerEpoch = IntMap; - pub type EpochIndex = u16; + +pub type CreditsPerEpoch = IntMap; diff --git a/packages/rs-drive/src/fee/result/refunds.rs b/packages/rs-drive/src/fee/result/refunds.rs index 064ca2f15a1..aeee97359e4 100644 --- a/packages/rs-drive/src/fee/result/refunds.rs +++ b/packages/rs-drive/src/fee/result/refunds.rs @@ -60,8 +60,11 @@ impl FeeRefunds { .map(|(identifier, bytes_per_epochs)| { bytes_per_epochs .into_iter() - .map(|(epoch_index, bytes)| { + .map(|(key, bytes)| { + let epoch_index = u16::try_from(key).map_err(|_| get_overflow_error("can't fit u64 epoch index from StorageRemovalPerEpochByIdentifier to u16 EpochIndex"))?; + // TODO We should use multipliers + (bytes as u64) .checked_mul(STORAGE_DISK_USAGE_CREDIT_PER_BYTE) .ok_or_else(|| { @@ -85,7 +88,7 @@ impl FeeRefunds { let intersection = sint_map_a .into_iter() .map(|(k, v)| { - let combined = if let Some(value_b) = int_map_b.remove(k) { + let combined = if let Some(value_b) = int_map_b.remove(&k) { v.checked_add(value_b) .ok_or(Error::Fee(FeeError::Overflow("storage fee overflow error"))) } else { From 8c121bf6051b0a9c532074377d4180f9eb01560b Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 16 Dec 2022 20:52:42 +0800 Subject: [PATCH 09/54] chore: continue distribution --- .../fee_pools/distribute_storage_pool.rs | 83 +++-------- .../execution/fee_pools/process_block_fees.rs | 2 - .../drive/fee_pools/pending_epoch_updates.rs | 131 ++++++++++-------- .../storage_fee_distribution_pool.rs | 3 +- packages/rs-drive/src/fee/credits.rs | 65 +++++++++ .../rs-drive/src/fee/epoch/distribution.rs | 31 +++-- packages/rs-drive/src/fee/epoch/mod.rs | 5 +- packages/rs-drive/src/fee/mod.rs | 4 +- packages/rs-drive/src/fee/result/refunds.rs | 22 +-- 9 files changed, 197 insertions(+), 149 deletions(-) create mode 100644 packages/rs-drive/src/fee/credits.rs diff --git a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs index b65d3420711..0be0a3488bf 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs @@ -39,7 +39,11 @@ use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; use drive::drive::fee_pools::epochs::constants::{EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS}; use drive::fee::constants; -use drive::fee::epoch::{EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS}; +use drive::fee::credits::{Creditable, SignedCredits}; +use drive::fee::epoch::distribution::distribute_storage_fee_to_epochs; +use drive::fee::epoch::{ + EpochIndex, SignedCreditsPerEpoch, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS, +}; use drive::fee_pools::epochs::Epoch; use drive::grovedb::TransactionArg; use drive::{error, grovedb}; @@ -51,10 +55,10 @@ pub type StorageDistributionLeftoverCredits = u64; impl Platform { /// Adds operations to the GroveDB op batch which calculate and distribute storage fees - /// from the distribution pool to the epoch pools and returns the leftovers. + /// from the distribution pool and pending updates to the epoch pools and returns the leftovers. pub fn add_distribute_storage_fee_to_epochs_operations( &self, - current_epoch_index: u16, + current_epoch_index: EpochIndex, transaction: TransactionArg, batch: &mut GroveDbOpBatch, ) -> Result { @@ -62,66 +66,19 @@ impl Platform { .drive .get_aggregate_storage_fees_from_distribution_pool(transaction)?; - if storage_distribution_fees == 0 { - return Ok(0); - } + let credits_per_epochs = SignedCreditsPerEpoch::new(); - // a separate buffer from which we withdraw to correctly calculate fee share - let mut storage_distribution_leftover_credits = storage_distribution_fees; - - let storage_distribution_fees = - Decimal::from_u64(storage_distribution_fees).ok_or(Error::Execution( - ExecutionError::Overflow("storage distribution fees are not fitting in a u64"), - ))?; - - let epochs_per_year = Decimal::from(EPOCHS_PER_YEAR); - - for year in 0..PERPETUAL_STORAGE_YEARS { - let distribution_for_that_year_ratio = constants::FEE_DISTRIBUTION_TABLE[year as usize]; - - let year_fee_share = storage_distribution_fees * distribution_for_that_year_ratio; - - let epoch_fee_share_dec = year_fee_share / epochs_per_year; - - let epoch_fee_share = epoch_fee_share_dec - .floor() - .to_u64() - .ok_or(Error::Execution(ExecutionError::Overflow( - "storage distribution fees are not fitting in a u64", - )))?; - - let year_start_epoch_index = current_epoch_index + EPOCHS_PER_YEAR * year; - - for index in year_start_epoch_index..year_start_epoch_index + EPOCHS_PER_YEAR { - let epoch_tree = Epoch::new(index); - - let current_epoch_pool_storage_credits = self - .drive - .get_epoch_storage_credits_for_distribution(&epoch_tree, transaction) - .or_else(|e| match e { - // In case if we have a gap between current and previous epochs - // multiple future epochs could be created in the current batch - error::Error::GroveDB(grovedb::Error::PathNotFound(_)) - | error::Error::GroveDB(grovedb::Error::PathKeyNotFound(_)) - | error::Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => { - Ok(0u64) - } - _ => Err(e), - })?; - - // TODO: It's not convenient and confusing when in one case you should push operation to batch - // and sometimes you pass batch inside to add operations. Also, in future a single operation function - // could become a multiple operations function so you need to change many code. Also, you can't use helpers which batch provides - batch.push(epoch_tree.update_storage_fee_pool_operation( - current_epoch_pool_storage_credits + epoch_fee_share, - )); - - storage_distribution_leftover_credits = storage_distribution_leftover_credits - .checked_sub(epoch_fee_share) - .ok_or(Error::Execution(ExecutionError::Overflow( - "leftover storage not fitting in a u64", - )))?; - } + let mut leftovers = distribute_storage_fee_to_epochs( + storage_distribution_fees.to_signed()?, + current_epoch_index, + &mut credits_per_epochs, + )?; + + // TODO Better to use iterator do not load everything into memory + for (epoch_index, credits) in self.drive.fetch_pending_updates(transaction)? { + credits + .checked_add(leftovers) + .ok_or_else(|| get_overflow_error("dasd")) } Ok(storage_distribution_leftover_credits) @@ -131,7 +88,7 @@ impl Platform { #[cfg(test)] mod tests { - mod distribute_storage_fee_distribution_pool { + mod add_distribute_storage_fees_to_epochs_operations { use crate::common::helpers::setup::setup_platform_with_initial_state_structure; use drive::common::helpers::epoch::get_storage_credits_for_distribution_for_epochs_in_range; use drive::drive::batch::GroveDbOpBatch; diff --git a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs index 8f87672c5f6..f934a323d7e 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs @@ -112,8 +112,6 @@ impl Platform { return Ok(None); } - // TODO: Distribute pending updates - // Distribute storage fees accumulated during previous epoch let storage_distribution_leftover_credits = self .add_distribute_storage_fee_to_epochs_operations( diff --git a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs index 8963802426b..ca1826ce651 100644 --- a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs +++ b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs @@ -33,29 +33,66 @@ use crate::drive::batch::GroveDbOpBatch; use crate::drive::fee_pools::pools_pending_updates_path; use crate::drive::Drive; use crate::error::drive::DriveError; -use crate::error::fee::FeeError; use crate::error::Error; -use crate::fee::epoch::CreditsPerEpoch; +use crate::fee::credits::{Creditable, SignedCredits}; +use crate::fee::epoch::SignedCreditsPerEpoch; use crate::fee::get_overflow_error; use grovedb::query_result_type::QueryResultType; use grovedb::{Element, PathQuery, Query, TransactionArg}; -use itertools::Itertools; impl Drive { + /// Fetches all pending epoch pool updates + pub fn fetch_pending_updates( + &self, + transaction: TransactionArg, + ) -> Result { + let mut query = Query::new(); + + query.insert_all(); + + let (query_result, _) = self + .grove + .query_raw( + &PathQuery::new_unsized(pools_pending_updates_path(), query), + QueryResultType::QueryKeyElementPairResultType, + transaction, + ) + .unwrap() + .map_err(Error::GroveDB)?; + + query_result.to_key_elements().into_iter().map(|(epoch_index_key, element)| { + let epoch_index = + u16::from_be_bytes(epoch_index_key.as_slice().try_into().map_err(|_| { + Error::Drive(DriveError::CorruptedSerialization( + "epoch index for pending pool updates must be i64", + )) + })?); + + let Element::SumItem(..) = element else { + return Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates credits must be sum items"))); + }; + + let credits: SignedCredits = element.sum_value().ok_or( + Error::Drive(DriveError::CorruptedCodeExecution("pending updates credits must have value") + ))?; + + Ok((epoch_index, credits)) + }).collect::>() + } + /// Fetches existing pending epoch pool updates using specified epochs /// and returns merged result pub fn fetch_and_merge_with_existing_pending_epoch_storage_pool_updates( &self, - mut credits_per_epoch: CreditsPerEpoch, + mut credits_per_epoch: SignedCreditsPerEpoch, transaction: TransactionArg, - ) -> Result { + ) -> Result { let mut query = Query::new(); - for (epoch_index_key, _) in credits_per_epoch.iter() { - let epoch_index = epoch_index_key.to_owned() as u16; - let encoded_epoch_index = epoch_index.to_be_bytes().to_vec(); + for epoch_index in credits_per_epoch.keys() { + let epoch_index_key = epoch_index.to_be_bytes().to_vec(); - query.insert_key(encoded_epoch_index); + query.insert_key(epoch_index_key); } // Query existing pending updates @@ -70,9 +107,9 @@ impl Drive { .map_err(Error::GroveDB)?; // Merge with existing pending updates - for (encoded_epoch_index, element) in query_result.to_key_elements() { + for (epoch_index_key, element) in query_result.to_key_elements() { let epoch_index = - u16::from_be_bytes(encoded_epoch_index.as_slice().try_into().map_err(|_| { + u16::from_be_bytes(epoch_index_key.as_slice().try_into().map_err(|_| { Error::Drive(DriveError::CorruptedSerialization( "epoch index for pending pool updates must be u16", )) @@ -82,30 +119,21 @@ impl Drive { return Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates should contain fetched epochs"))); }; - let Element::Item(encoded_existing_signed_credits, _) = element else { - return Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates should contain only items"))); + let Element::SumItem(..) = element else { + return Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates credits must be sum items"))); }; - let existing_signed_credits = i64::from_be_bytes( - encoded_existing_signed_credits - .as_slice() - .try_into() - .map_err(|_| { - Error::Drive(DriveError::CorruptedSerialization( - "credits for pending pool updates must be i64", - )) - })?, - ); - - let existing_credits = existing_signed_credits.unsigned_abs(); - - let result_credits = - credits_to_update - .checked_add(existing_credits) - .ok_or(Error::Fee(FeeError::Overflow( - "pending updates overflow error", + let existing_credits: SignedCredits = + element + .sum_value() + .ok_or(Error::Drive(DriveError::CorruptedCodeExecution( + "pending updates credits must have value", )))?; + let result_credits = credits_to_update + .checked_add(existing_credits) + .ok_or_else(|| get_overflow_error("pending updates credits overflow"))?; + credits_per_epoch.insert(epoch_index, result_credits); } @@ -116,7 +144,7 @@ impl Drive { pub fn add_delete_pending_epoch_storage_pool_updates_except_specified_operations( &self, batch: &mut GroveDbOpBatch, - credits_per_epoch: &CreditsPerEpoch, + credits_per_epoch: &SignedCreditsPerEpoch, transaction: TransactionArg, ) -> Result<(), Error> { // TODO: Replace with key iterator @@ -134,11 +162,11 @@ impl Drive { .unwrap() .map_err(Error::GroveDB)?; - for (encoded_epoch_index, _) in query_result.to_key_elements() { + for (epoch_index_key, _) in query_result.to_key_elements() { let epoch_index = - u16::from_be_bytes(encoded_epoch_index.as_slice().try_into().map_err(|_| { + u16::from_be_bytes(epoch_index_key.as_slice().try_into().map_err(|_| { Error::Drive(DriveError::CorruptedSerialization( - "epoch index for pending pool updates must be u16", + "pending updates epoch index for must be u16", )) })?); @@ -146,7 +174,7 @@ impl Drive { continue; } - batch.add_delete(pools_pending_updates_path(), encoded_epoch_index); + batch.add_delete(pools_pending_updates_path(), epoch_index_key); } Ok(()) @@ -156,19 +184,14 @@ impl Drive { /// Adds GroveDB batch operations to update pending epoch storage pool updates pub fn add_update_pending_epoch_storage_pool_update_operations( batch: &mut GroveDbOpBatch, - credits_per_epoch: CreditsPerEpoch, + credits_per_epoch: SignedCreditsPerEpoch, ) -> Result<(), Error> { - for (epoch_index_key, credits) in credits_per_epoch.into_iter().sorted_by_key(|x| x.0) { - let epoch_index = epoch_index_key as u16; - let encoded_epoch_index = epoch_index.to_be_bytes().to_vec(); - - let signed_credits = -i64::try_from(credits).map_err(|_| { - get_overflow_error("can't convert credits to negative amount for pending updates") - })?; + for (epoch_index, credits) in credits_per_epoch { + let epoch_index_key = epoch_index.to_be_bytes().to_vec(); - let element = Element::new_item(signed_credits.to_be_bytes().to_vec()); + let element = Element::new_sum_item(credits.to_signed()?); - batch.add_insert(pools_pending_updates_path(), encoded_epoch_index, element); + batch.add_insert(pools_pending_updates_path(), epoch_index_key, element); } Ok(()) @@ -191,7 +214,7 @@ mod tests { // Store initial set of pending updates let initial_pending_updates = - CreditsPerEpoch::from_iter(vec![(1, 15), (3, 25), (7, 95), (9, 100), (12, 120)]); + SignedCreditsPerEpoch::from_iter([(1, 15), (3, 25), (7, 95), (9, 100), (12, 120)]); let mut batch = GroveDbOpBatch::new(); @@ -208,7 +231,7 @@ mod tests { // Fetch and merge let new_pending_updates = - CreditsPerEpoch::from_iter(vec![(1, 15), (3, 25), (30, 195), (41, 150)]); + SignedCreditsPerEpoch::from_iter([(1, 15), (3, 25), (30, 195), (41, 150)]); let updated_pending_updates = drive .fetch_and_merge_with_existing_pending_epoch_storage_pool_updates( @@ -218,7 +241,7 @@ mod tests { .expect("should fetch and merge pending updates"); let expected_pending_updates = - CreditsPerEpoch::from_iter(vec![(1, 30), (3, 50), (30, 195), (41, 150)]); + SignedCreditsPerEpoch::from_iter([(1, 30), (3, 50), (30, 195), (41, 150)]); assert_eq!(updated_pending_updates, expected_pending_updates); } @@ -237,7 +260,7 @@ mod tests { // Store initial set of pending updates let initial_pending_updates = - CreditsPerEpoch::from_iter(vec![(1, 15), (3, 25), (7, 95), (9, 100), (12, 120)]); + SignedCreditsPerEpoch::from_iter([(1, 15), (3, 25), (7, 95), (9, 100), (12, 120)]); let mut batch = GroveDbOpBatch::new(); @@ -253,7 +276,7 @@ mod tests { // Delete existing pending updates expect specified pending updates - let new_pending_updates = CreditsPerEpoch::from_iter(vec![(1, 15), (3, 25)]); + let new_pending_updates = SignedCreditsPerEpoch::from_iter([(1, 15), (3, 25)]); let mut batch = GroveDbOpBatch::new(); @@ -266,7 +289,7 @@ mod tests { .expect("should fetch and merge pending updates"); let expected_pending_updates = - CreditsPerEpoch::from_iter(vec![(7, 95), (9, 100), (12, 120)]); + SignedCreditsPerEpoch::from_iter([(7, 95), (9, 100), (12, 120)]); assert_eq!(batch.len(), expected_pending_updates.len()); @@ -275,9 +298,9 @@ mod tests { assert_eq!(operation.path.to_path(), pools_pending_updates_path()); - let encoded_epoch_index = operation.key.get_key(); + let epoch_index_key = operation.key.get_key(); let epoch_index = u16::from_be_bytes( - encoded_epoch_index + epoch_index_key .try_into() .expect("should convert to u16 bytes"), ); diff --git a/packages/rs-drive/src/drive/fee_pools/storage_fee_distribution_pool.rs b/packages/rs-drive/src/drive/fee_pools/storage_fee_distribution_pool.rs index 37692ed6ec9..8a880687fdf 100644 --- a/packages/rs-drive/src/drive/fee_pools/storage_fee_distribution_pool.rs +++ b/packages/rs-drive/src/drive/fee_pools/storage_fee_distribution_pool.rs @@ -36,6 +36,7 @@ use grovedb::{Element, TransactionArg}; use crate::error::fee::FeeError; use crate::error::Error; +use crate::fee::credits::Credits; use crate::fee_pools::epochs_root_tree_key_constants::KEY_STORAGE_FEE_POOL; impl Drive { @@ -43,7 +44,7 @@ impl Drive { pub fn get_aggregate_storage_fees_from_distribution_pool( &self, transaction: TransactionArg, - ) -> Result { + ) -> Result { match self .grove .get(pools_path(), KEY_STORAGE_FEE_POOL.as_slice(), transaction) diff --git a/packages/rs-drive/src/fee/credits.rs b/packages/rs-drive/src/fee/credits.rs new file mode 100644 index 00000000000..857d0e74546 --- /dev/null +++ b/packages/rs-drive/src/fee/credits.rs @@ -0,0 +1,65 @@ +// MIT LICENSE +// +// Copyright (c) 2021 Dash Core Group +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +//! Fee pool constants. +//! +//! This module defines constants related to fee distribution pools. +//! + +use crate::error::Error; +use crate::fee::get_overflow_error; +use rust_decimal::Decimal; + +pub type Credits = u64; +pub type SignedCredits = i64; + +pub trait Creditable: Into { + fn to_signed(&self) -> Result; + fn to_unsigned(&self) -> Credits; +} + +impl Creditable for Credits { + fn to_signed(&self) -> Result { + SignedCredits::try_from(self.clone()) + .map_err(|_| get_overflow_error("credits are too big to convert to signed value")) + } + + fn to_unsigned(&self) -> Credits { + self.clone() + } +} +impl Creditable for SignedCredits { + fn to_signed(&self) -> Result { + Ok(self.clone()) + } + + fn to_unsigned(&self) -> Credits { + self.clone() as Credits + } +} diff --git a/packages/rs-drive/src/fee/epoch/distribution.rs b/packages/rs-drive/src/fee/epoch/distribution.rs index b63f97494e6..1aaae840917 100644 --- a/packages/rs-drive/src/fee/epoch/distribution.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -34,8 +34,11 @@ use crate::error::fee::FeeError; use crate::error::Error; -use crate::fee::epoch::{CreditsPerEpoch, EpochIndex, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS}; -use crate::fee::{get_overflow_error, Credits}; +use crate::fee::credits::{Creditable, SignedCredits}; +use crate::fee::epoch::{ + EpochIndex, SignedCreditsPerEpoch, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS, +}; +use crate::fee::get_overflow_error; use rust_decimal::prelude::*; use rust_decimal::Decimal; use rust_decimal_macros::dec; @@ -57,20 +60,20 @@ pub const FEE_DISTRIBUTION_TABLE: [Decimal; PERPETUAL_STORAGE_YEARS as usize] = dec!(0.00325), dec!(0.00275), dec!(0.00225), dec!(0.00175), dec!(0.00125), ]; -pub type DistributionLeftoverCredits = Credits; +pub type DistributionLeftoverCredits = SignedCredits; pub fn distribute_storage_fee_to_epochs( - storage_fee: u64, + storage_fee: SignedCredits, start_epoch_index: EpochIndex, - credits_per_epochs: Option, + credits_per_epochs: &mut SignedCreditsPerEpoch, ) -> Result { - let storage_fee_dec = Decimal::from_u64(storage_fee).ok_or(Error::Fee( - FeeError::CorruptedCodeExecution("storage fees are not fitting in a Decimal"), - ))?; + if storage_fee == 0 { + return Ok(0); + } - let mut distribution_leftover_credits = storage_fee; + let storage_fee_dec: Decimal = storage_fee.into(); - let mut credits_per_epochs = credits_per_epochs.unwrap_or_default(); + let mut distribution_leftover_credits = storage_fee; let epochs_per_year = Decimal::from(EPOCHS_PER_YEAR); @@ -81,19 +84,19 @@ pub fn distribute_storage_fee_to_epochs( let epoch_fee_share_dec = year_fee_share / epochs_per_year; - let epoch_fee_share = epoch_fee_share_dec + let epoch_fee_share: SignedCredits = epoch_fee_share_dec .floor() - .to_u64() + .to_i64() .ok_or_else(|| get_overflow_error("storage fees are not fitting in a u64"))?; let year_start_epoch_index = start_epoch_index + EPOCHS_PER_YEAR * year; for epoch_index in year_start_epoch_index..year_start_epoch_index + EPOCHS_PER_YEAR { - let current_epoch_credits = credits_per_epochs + let current_epoch_credits: SignedCredits = credits_per_epochs .get(&epoch_index) .map_or(0, |i| i.to_owned()); - let result_storage_fee = current_epoch_credits + let result_storage_fee: SignedCredits = current_epoch_credits .checked_add(epoch_fee_share) .ok_or_else(|| get_overflow_error("storage fees are not fitting in a u64"))?; diff --git a/packages/rs-drive/src/fee/epoch/mod.rs b/packages/rs-drive/src/fee/epoch/mod.rs index 1fa5df5b4e8..b2f35d33a8b 100644 --- a/packages/rs-drive/src/fee/epoch/mod.rs +++ b/packages/rs-drive/src/fee/epoch/mod.rs @@ -1,7 +1,7 @@ -use crate::fee::Credits; +use crate::fee::credits::{Credits, SignedCredits}; use nohash_hasher::IntMap; -mod distribution; +pub mod distribution; /// Genesis epoch index pub const GENESIS_EPOCH_INDEX: u16 = 0; @@ -18,3 +18,4 @@ pub const PERPETUAL_STORAGE_EPOCHS: u16 = PERPETUAL_STORAGE_YEARS * EPOCHS_PER_Y pub type EpochIndex = u16; pub type CreditsPerEpoch = IntMap; +pub type SignedCreditsPerEpoch = IntMap; diff --git a/packages/rs-drive/src/fee/mod.rs b/packages/rs-drive/src/fee/mod.rs index b75c0972c1d..d1122404006 100644 --- a/packages/rs-drive/src/fee/mod.rs +++ b/packages/rs-drive/src/fee/mod.rs @@ -28,6 +28,7 @@ // use enum_map::EnumMap; +use rust_decimal::Decimal; use crate::error::fee::FeeError; use crate::error::Error; @@ -35,6 +36,7 @@ use crate::fee::op::{BaseOp, DriveOperation}; use crate::fee::result::FeeResult; use crate::fee_pools::epochs::Epoch; +pub mod credits; pub mod default_costs; pub mod epoch; pub mod op; @@ -43,8 +45,6 @@ pub mod result; /// Default original fee multiplier pub const DEFAULT_ORIGINAL_FEE_MULTIPLIER: f64 = 2.0; -pub type Credits = u64; - /// Calculates fees for the given operations. Returns the storage and processing costs. pub fn calculate_fee( base_operations: Option>, diff --git a/packages/rs-drive/src/fee/result/refunds.rs b/packages/rs-drive/src/fee/result/refunds.rs index aeee97359e4..7850463f58d 100644 --- a/packages/rs-drive/src/fee/result/refunds.rs +++ b/packages/rs-drive/src/fee/result/refunds.rs @@ -35,7 +35,7 @@ use crate::error::fee::FeeError; use crate::error::Error; use crate::fee::default_costs::STORAGE_DISK_USAGE_CREDIT_PER_BYTE; -use crate::fee::epoch::CreditsPerEpoch; +use crate::fee::epoch::SignedCreditsPerEpoch; use crate::fee::get_overflow_error; use bincode::Options; use costs::storage_cost::removal::{Identifier, StorageRemovalPerEpochByIdentifier}; @@ -44,11 +44,11 @@ use std::collections::btree_map::{IntoIter, Iter}; use std::collections::BTreeMap; /// Credits per Epoch by Identifier -pub type CreditsPerEpochByIdentifier = BTreeMap; +pub type SignedCreditsPerEpochByIdentifier = BTreeMap; /// Fee refunds to identities based on removed data from specific epochs #[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)] -pub struct FeeRefunds(pub CreditsPerEpochByIdentifier); +pub struct FeeRefunds(pub SignedCreditsPerEpochByIdentifier); impl FeeRefunds { /// Create fee refunds from GroveDB's StorageRemovalPerEpochByIdentifier @@ -65,17 +65,17 @@ impl FeeRefunds { // TODO We should use multipliers - (bytes as u64) - .checked_mul(STORAGE_DISK_USAGE_CREDIT_PER_BYTE) + (bytes as i64) + .checked_mul(STORAGE_DISK_USAGE_CREDIT_PER_BYTE as i64) .ok_or_else(|| { get_overflow_error("storage written bytes cost overflow") }) .map(|credits| (epoch_index, credits)) }) - .collect::>() + .collect::>() .map(|credits_per_epochs| (identifier, credits_per_epochs)) }) - .collect::>()?; + .collect::>()?; Ok(Self(refunds_per_epoch_by_identifier)) } @@ -96,7 +96,7 @@ impl FeeRefunds { }; combined.map(|c| (k, c)) }) - .collect::>()?; + .collect::>()?; intersection.into_iter().chain(int_map_b).collect() } else { int_map_b @@ -108,17 +108,17 @@ impl FeeRefunds { } /// Passthrough method for get - pub fn get(&self, key: &Identifier) -> Option<&CreditsPerEpoch> { + pub fn get(&self, key: &Identifier) -> Option<&SignedCreditsPerEpoch> { self.0.get(key) } /// Passthrough method for iteration - pub fn iter(&self) -> Iter { + pub fn iter(&self) -> Iter { self.0.iter() } /// Passthrough method for into iteration - pub fn into_iter(self) -> IntoIter { + pub fn into_iter(self) -> IntoIter { self.0.into_iter() } From feb1be07698505dd598b40f9a154005eb4418333 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Fri, 16 Dec 2022 21:35:35 +0800 Subject: [PATCH 10/54] chore: continue distribution --- .../fee_pools/distribute_storage_pool.rs | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs index 0be0a3488bf..9deb7cd16cd 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs @@ -38,9 +38,13 @@ use crate::error::Error; use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; use drive::drive::fee_pools::epochs::constants::{EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS}; +use drive::error::drive::DriveError; +use drive::error::fee::FeeError; use drive::fee::constants; use drive::fee::credits::{Creditable, SignedCredits}; -use drive::fee::epoch::distribution::distribute_storage_fee_to_epochs; +use drive::fee::epoch::distribution::{ + distribute_storage_fee_to_epochs, DistributionLeftoverCredits, +}; use drive::fee::epoch::{ EpochIndex, SignedCreditsPerEpoch, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS, }; @@ -51,7 +55,6 @@ use rust_decimal::prelude::{FromPrimitive, ToPrimitive}; use rust_decimal::Decimal; /// Leftover credits after adding up and distributing storage fees. -pub type StorageDistributionLeftoverCredits = u64; impl Platform { /// Adds operations to the GroveDB op batch which calculate and distribute storage fees @@ -61,7 +64,7 @@ impl Platform { current_epoch_index: EpochIndex, transaction: TransactionArg, batch: &mut GroveDbOpBatch, - ) -> Result { + ) -> Result { let storage_distribution_fees = self .drive .get_aggregate_storage_fees_from_distribution_pool(transaction)?; @@ -76,12 +79,25 @@ impl Platform { // TODO Better to use iterator do not load everything into memory for (epoch_index, credits) in self.drive.fetch_pending_updates(transaction)? { - credits - .checked_add(leftovers) - .ok_or_else(|| get_overflow_error("dasd")) + let credits_to_distribute = credits.checked_add(leftovers).ok_or_else(|| { + Error::Execution(ExecutionError::Overflow( + "can't add leftovers to pending storage fees", + )) + })?; + + // TODO: We should calculate for epochs which are before current one. + // leftovers also should be less + + leftovers = distribute_storage_fee_to_epochs( + credits_to_distribute, + epoch_index, + &mut credits_per_epochs, + )?; } - Ok(storage_distribution_leftover_credits) + // TODO: Nil negative pools + + Ok(leftovers) } } From e4545528a1441425a842b3e84e17798954dffac9 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sat, 17 Dec 2022 14:55:13 +0800 Subject: [PATCH 11/54] chore: accomplish distribution --- .../fee_pools/distribute_storage_pool.rs | 12 ++++--- packages/rs-drive/src/drive/fee_pools/mod.rs | 30 ++++++++++++----- packages/rs-drive/src/fee/credits.rs | 33 ++++++++++++++++++- .../rs-drive/src/fee/epoch/distribution.rs | 11 +++++-- 4 files changed, 70 insertions(+), 16 deletions(-) diff --git a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs index 9deb7cd16cd..136532a23c3 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs @@ -49,6 +49,7 @@ use drive::fee::epoch::{ EpochIndex, SignedCreditsPerEpoch, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS, }; use drive::fee_pools::epochs::Epoch; +use drive::fee_pools::update_storage_fee_distribution_pool_operation; use drive::grovedb::TransactionArg; use drive::{error, grovedb}; use rust_decimal::prelude::{FromPrimitive, ToPrimitive}; @@ -74,6 +75,7 @@ impl Platform { let mut leftovers = distribute_storage_fee_to_epochs( storage_distribution_fees.to_signed()?, current_epoch_index, + current_epoch_index, &mut credits_per_epochs, )?; @@ -85,17 +87,19 @@ impl Platform { )) })?; - // TODO: We should calculate for epochs which are before current one. - // leftovers also should be less - leftovers = distribute_storage_fee_to_epochs( credits_to_distribute, epoch_index, + current_epoch_index, &mut credits_per_epochs, )?; } - // TODO: Nil negative pools + self.drive.add_update_epoch_storage_fee_pools_operations( + batch, + credits_per_epochs, + transaction, + )?; Ok(leftovers) } diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index bfa0bf17f5a..a2523d5b96b 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -31,14 +31,15 @@ use crate::drive::batch::GroveDbOpBatch; use crate::drive::{Drive, RootTree}; use crate::error::drive::DriveError; use crate::error::Error; -use crate::fee::epoch::CreditsPerEpoch; +use crate::fee::credits::{Creditable, SignedCredits}; +use crate::fee::epoch::SignedCreditsPerEpoch; +use crate::fee::get_overflow_error; use crate::fee_pools::epochs::epoch_key_constants::KEY_POOL_STORAGE_FEES; use crate::fee_pools::epochs::{paths, Epoch}; use crate::fee_pools::epochs_root_tree_key_constants::{ KEY_PENDING_POOL_UPDATES, KEY_STORAGE_FEE_POOL, }; -use crate::query::QueryItem; -use grovedb::{PathQuery, Query, TransactionArg}; +use grovedb::{Element, PathQuery, Query, TransactionArg}; use itertools::Itertools; /// Epochs module @@ -82,7 +83,7 @@ impl Drive { pub fn add_update_epoch_storage_fee_pools_operations( &self, batch: &mut GroveDbOpBatch, - credits_per_epochs: CreditsPerEpoch, + credits_per_epochs: SignedCreditsPerEpoch, transaction: TransactionArg, ) -> Result<(), Error> { if credits_per_epochs.len() == 0 { @@ -110,7 +111,7 @@ impl Drive { epochs_query.set_subquery(storage_fee_pool_query); - let (storage_fee_pools, _) = self + let (mut storage_fee_pools, _) = self .grove .query( &PathQuery::new_unsized(pools_vec_path(), epochs_query), @@ -125,13 +126,26 @@ impl Drive { ))); } - for (i, (epoch_index_key, credits)) in credits_per_epochs + for (i, (epoch_index, credits)) in credits_per_epochs .into_iter() .sorted_by_key(|x| x.0) .enumerate() { - let encoded_epoch_index_key = - paths::encode_epoch_index_key(epoch_index_key as u16)?.to_vec(); + let encoded_epoch_index_key = paths::encode_epoch_index_key(epoch_index)?.to_vec(); + + let existing_storage_fee = SignedCredits::from_vec_bytes(storage_fee_pools.remove(i))?; + + let credits_to_update = existing_storage_fee.checked_add(credits).ok_or_else(|| { + get_overflow_error("can't add credits to existing epoch pool storage fee") + })?; + + let element = Element::new_item(credits_to_update.to_unsigned().to_vec_bytes()); + + batch.add_insert( + Epoch::new(epoch_index).get_vec_path(), + KEY_POOL_STORAGE_FEES.to_vec(), + element, + ); } Ok(()) diff --git a/packages/rs-drive/src/fee/credits.rs b/packages/rs-drive/src/fee/credits.rs index 857d0e74546..67c470e267b 100644 --- a/packages/rs-drive/src/fee/credits.rs +++ b/packages/rs-drive/src/fee/credits.rs @@ -32,6 +32,7 @@ //! This module defines constants related to fee distribution pools. //! +use crate::error::drive::DriveError; use crate::error::Error; use crate::fee::get_overflow_error; use rust_decimal::Decimal; @@ -42,6 +43,8 @@ pub type SignedCredits = i64; pub trait Creditable: Into { fn to_signed(&self) -> Result; fn to_unsigned(&self) -> Credits; + fn from_vec_bytes(vec: Vec) -> Result; + fn to_vec_bytes(&self) -> Vec; } impl Creditable for Credits { @@ -53,6 +56,20 @@ impl Creditable for Credits { fn to_unsigned(&self) -> Credits { self.clone() } + + fn from_vec_bytes(vec: Vec) -> Result { + Ok(Self::from_be_bytes(vec.as_slice().try_into().map_err( + |_| { + Error::Drive(DriveError::CorruptedSerialization( + "pending updates epoch index for must be u16", + )) + }, + )?)) + } + + fn to_vec_bytes(&self) -> Vec { + self.to_be_bytes().to_vec() + } } impl Creditable for SignedCredits { fn to_signed(&self) -> Result { @@ -60,6 +77,20 @@ impl Creditable for SignedCredits { } fn to_unsigned(&self) -> Credits { - self.clone() as Credits + self.unsigned_abs() + } + + fn from_vec_bytes(vec: Vec) -> Result { + Ok(Self::from_be_bytes(vec.as_slice().try_into().map_err( + |_| { + Error::Drive(DriveError::CorruptedSerialization( + "pending updates epoch index for must be u16", + )) + }, + )?)) + } + + fn to_vec_bytes(&self) -> Vec { + self.to_be_bytes().to_vec() } } diff --git a/packages/rs-drive/src/fee/epoch/distribution.rs b/packages/rs-drive/src/fee/epoch/distribution.rs index 1aaae840917..94b34375081 100644 --- a/packages/rs-drive/src/fee/epoch/distribution.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -34,7 +34,7 @@ use crate::error::fee::FeeError; use crate::error::Error; -use crate::fee::credits::{Creditable, SignedCredits}; +use crate::fee::credits::SignedCredits; use crate::fee::epoch::{ EpochIndex, SignedCreditsPerEpoch, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS, }; @@ -65,6 +65,7 @@ pub type DistributionLeftoverCredits = SignedCredits; pub fn distribute_storage_fee_to_epochs( storage_fee: SignedCredits, start_epoch_index: EpochIndex, + from_epoch_index: EpochIndex, credits_per_epochs: &mut SignedCreditsPerEpoch, ) -> Result { if storage_fee == 0 { @@ -92,11 +93,15 @@ pub fn distribute_storage_fee_to_epochs( let year_start_epoch_index = start_epoch_index + EPOCHS_PER_YEAR * year; for epoch_index in year_start_epoch_index..year_start_epoch_index + EPOCHS_PER_YEAR { - let current_epoch_credits: SignedCredits = credits_per_epochs + if epoch_index < from_epoch_index { + continue; + } + + let epoch_credits: SignedCredits = credits_per_epochs .get(&epoch_index) .map_or(0, |i| i.to_owned()); - let result_storage_fee: SignedCredits = current_epoch_credits + let result_storage_fee: SignedCredits = epoch_credits .checked_add(epoch_fee_share) .ok_or_else(|| get_overflow_error("storage fees are not fitting in a u64"))?; From 28afc1dc178a14935f0adfc75cd7844519971edb Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sat, 17 Dec 2022 20:02:52 +0800 Subject: [PATCH 12/54] chore: accomplish distribution --- Cargo.lock | 6 +++ packages/rs-drive-abci/Cargo.toml | 3 +- packages/rs-drive-abci/src/abci/messages.rs | 7 ++-- .../fee_pools/distribute_storage_pool.rs | 2 +- .../execution/fee_pools/fee_distribution.rs | 13 +++++-- .../execution/fee_pools/process_block_fees.rs | 12 +++--- packages/rs-drive/Cargo.toml | 1 + packages/rs-drive/src/drive/fee_pools/mod.rs | 5 +-- packages/rs-drive/src/fee/credits.rs | 9 +++++ .../rs-drive/src/fee/epoch/distribution.rs | 2 + packages/rs-drive/src/fee/epoch/mod.rs | 37 +++++++++++++++++++ packages/rs-drive/src/fee/op.rs | 8 ++-- packages/rs-drive/src/fee/result/mod.rs | 2 +- packages/rs-drive/src/fee_pools/mod.rs | 2 +- 14 files changed, 84 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d5dc63d65e..d826fa3c320 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -915,6 +915,7 @@ dependencies = [ "hex", "indexmap", "integer-encoding", + "intmap", "itertools", "moka", "nohash-hasher", @@ -942,6 +943,8 @@ dependencies = [ "drive", "hex", "rand 0.8.5", + "rust_decimal", + "rust_decimal_macros", "serde", "serde_json", "tempfile", @@ -1433,6 +1436,9 @@ name = "intmap" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee87fd093563344074bacf24faa0bb0227fb6969fb223e922db798516de924d6" +dependencies = [ + "serde", +] [[package]] name = "iso8601" diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml index 282f3fefd59..18b26a94175 100644 --- a/packages/rs-drive-abci/Cargo.toml +++ b/packages/rs-drive-abci/Cargo.toml @@ -19,4 +19,5 @@ bs58 = "0.4.0" base64 = "0.13.0" hex = "0.4.3" dashcore = { git="https://github.com/dashpay/rust-dashcore", features=["no-std", "secp-recovery", "rand", "signer"], default-features = false, branch="master" } - +rust_decimal = "1.2.5" +rust_decimal_macros = "1.25.0" diff --git a/packages/rs-drive-abci/src/abci/messages.rs b/packages/rs-drive-abci/src/abci/messages.rs index 50478d6c10f..3ae8df54f78 100644 --- a/packages/rs-drive-abci/src/abci/messages.rs +++ b/packages/rs-drive-abci/src/abci/messages.rs @@ -37,9 +37,8 @@ use crate::error::serialization::SerializationError; use crate::error::Error; use crate::execution::fee_pools::epoch::EpochInfo; use crate::execution::fee_pools::process_block_fees::ProcessedBlockFeesResult; -use drive::fee::refunds::CreditsPerEpoch; -use serde::{Deserialize, Deserializer, Serialize}; -use std::ops::Deref; +use drive::fee::epoch::SignedCreditsPerEpoch; +use serde::{Deserialize, Serialize}; /// A struct for handling chain initialization requests #[derive(Serialize, Deserialize)] @@ -95,7 +94,7 @@ pub struct BlockFees { /// Storage fee pub storage_fee: u64, /// Fee refunds - pub fee_refunds: CreditsPerEpoch, + pub fee_refunds: SignedCreditsPerEpoch, } impl BlockFees { diff --git a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs index 136532a23c3..1783ffa3337 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs @@ -70,7 +70,7 @@ impl Platform { .drive .get_aggregate_storage_fees_from_distribution_pool(transaction)?; - let credits_per_epochs = SignedCreditsPerEpoch::new(); + let mut credits_per_epochs = SignedCreditsPerEpoch::default(); let mut leftovers = distribute_storage_fee_to_epochs( storage_distribution_fees.to_signed()?, diff --git a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs index 607a2f3c972..d47a682949e 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs @@ -38,7 +38,6 @@ use crate::abci::messages::BlockFees; use crate::error::Error; use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; -use drive::drive::fee_pools::epochs::constants::GENESIS_EPOCH_INDEX; use drive::error::fee::FeeError; use drive::fee::epoch::GENESIS_EPOCH_INDEX; use drive::fee_pools::epochs::Epoch; @@ -293,7 +292,7 @@ impl Platform { )) })?; - let share_percentage = Decimal::from(share_percentage_integer) / dec!(10000); + let share_percentage = Decimal::from(share_percentage_integer) / dec!(10000.0); let reward = masternode_reward * share_percentage; @@ -434,6 +433,8 @@ mod tests { mod add_distribute_fees_from_oldest_unpaid_epoch_pool_to_proposers_operations { use super::*; + use drive::error::Error as DriveError; + #[test] fn test_nothing_to_distribute_if_there_is_no_epochs_needing_payment() { let platform = setup_platform_with_initial_state_structure(); @@ -780,7 +781,9 @@ mod tests { ) { Ok(_) => assert!(false, "expect tree not exists"), Err(e) => match e { - Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => assert!(true), + DriveError::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => { + assert!(true) + } _ => assert!(false, "invalid error type"), }, } @@ -889,7 +892,9 @@ mod tests { ) { Ok(_) => assert!(false, "expect tree not exists"), Err(e) => match e { - Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => assert!(true), + DriveError::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => { + assert!(true) + } _ => assert!(false, "invalid error type"), }, } diff --git a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs index f934a323d7e..1a8536f8ebe 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs @@ -36,7 +36,6 @@ use std::option::Option::None; use drive::drive::batch::GroveDbOpBatch; -use drive::drive::fee_pools::epochs::constants::{GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; use drive::drive::fee_pools::pending_epoch_updates::add_update_pending_epoch_storage_pool_update_operations; use drive::fee_pools::epochs::Epoch; use drive::grovedb::TransactionArg; @@ -44,11 +43,10 @@ use drive::grovedb::TransactionArg; use crate::abci::messages::BlockFees; use crate::block::BlockInfo; use crate::error::Error; -use crate::execution::fee_pools::distribute_storage_pool::StorageDistributionLeftoverCredits; use crate::execution::fee_pools::epoch::EpochInfo; use crate::execution::fee_pools::fee_distribution::{FeesInPools, ProposersPayouts}; use crate::platform::Platform; -use drive::fee::constants::DEFAULT_ORIGINAL_FEE_MULTIPLIER; +use drive::fee::epoch::distribution::DistributionLeftoverCredits; use drive::fee::epoch::{GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; use drive::fee::DEFAULT_ORIGINAL_FEE_MULTIPLIER; @@ -78,7 +76,7 @@ impl Platform { /// as well as the current+1000 epoch, then distributes storage fees accumulated /// during the previous epoch. /// - /// `StorageDistributionLeftoverCredits` will be returned, except if we are at Genesis Epoch. + /// `DistributionLeftoverCredits` will be returned, except if we are at Genesis Epoch. fn add_process_epoch_change_operations( &self, block_info: &BlockInfo, @@ -86,7 +84,7 @@ impl Platform { block_fees: &BlockFees, transaction: TransactionArg, batch: &mut GroveDbOpBatch, - ) -> Result, Error> { + ) -> Result, Error> { // init next thousandth empty epochs since last initiated let last_initiated_epoch_index = epoch_info .previous_epoch_index @@ -145,7 +143,7 @@ impl Platform { let mut batch = GroveDbOpBatch::new(); - let storage_distribution_leftover_credits = if epoch_info.is_epoch_change { + let storage_fee_distribution_leftover_credits = if epoch_info.is_epoch_change { self.add_process_epoch_change_operations( block_info, epoch_info, @@ -194,7 +192,7 @@ impl Platform { ¤t_epoch, &block_fees, // Add leftovers after storage fee pool distribution to the current block storage fees - storage_distribution_leftover_credits, + storage_fee_distribution_leftover_credits, transaction, &mut batch, )?; diff --git a/packages/rs-drive/Cargo.toml b/packages/rs-drive/Cargo.toml index e688515eb0b..a69b34ac68b 100644 --- a/packages/rs-drive/Cargo.toml +++ b/packages/rs-drive/Cargo.toml @@ -24,6 +24,7 @@ sqlparser = "0.13.0" enum-map = "2.0.3" thiserror = "1.0.30" moka = "0.8.1" +intmap = { version="2.0.0", features=["serde"] } nohash-hasher = "0.2.0" chrono = "0.4.20" bincode = "1.3.3" diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index a2523d5b96b..9201019f5a6 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -80,13 +80,14 @@ pub fn aggregate_storage_fees_distribution_pool_vec_path() -> Vec> { } impl Drive { + /// Adds GroveDB operations to update epoch storage fee pools with specified map of credits to epochs pub fn add_update_epoch_storage_fee_pools_operations( &self, batch: &mut GroveDbOpBatch, credits_per_epochs: SignedCreditsPerEpoch, transaction: TransactionArg, ) -> Result<(), Error> { - if credits_per_epochs.len() == 0 { + if credits_per_epochs.is_empty() { return Ok(()); } @@ -131,8 +132,6 @@ impl Drive { .sorted_by_key(|x| x.0) .enumerate() { - let encoded_epoch_index_key = paths::encode_epoch_index_key(epoch_index)?.to_vec(); - let existing_storage_fee = SignedCredits::from_vec_bytes(storage_fee_pools.remove(i))?; let credits_to_update = existing_storage_fee.checked_add(credits).ok_or_else(|| { diff --git a/packages/rs-drive/src/fee/credits.rs b/packages/rs-drive/src/fee/credits.rs index 67c470e267b..33381f2b1bd 100644 --- a/packages/rs-drive/src/fee/credits.rs +++ b/packages/rs-drive/src/fee/credits.rs @@ -37,13 +37,22 @@ use crate::error::Error; use crate::fee::get_overflow_error; use rust_decimal::Decimal; +/// Credits type pub type Credits = u64; + +/// Signed Credits type is used for internal computations and total credits +/// balance verification pub type SignedCredits = i64; +/// Trait for signed and unsigned credits pub trait Creditable: Into { + /// Convert unsigned credit to singed fn to_signed(&self) -> Result; + /// Convert singed credit to unsigned fn to_unsigned(&self) -> Credits; + /// Decode bytes to credits fn from_vec_bytes(vec: Vec) -> Result; + /// Encode credits to bytes fn to_vec_bytes(&self) -> Vec; } diff --git a/packages/rs-drive/src/fee/epoch/distribution.rs b/packages/rs-drive/src/fee/epoch/distribution.rs index 94b34375081..ab290e78cbf 100644 --- a/packages/rs-drive/src/fee/epoch/distribution.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -60,8 +60,10 @@ pub const FEE_DISTRIBUTION_TABLE: [Decimal; PERPETUAL_STORAGE_YEARS as usize] = dec!(0.00325), dec!(0.00275), dec!(0.00225), dec!(0.00175), dec!(0.00125), ]; +/// Leftovers in result of divisions and rounding after storage fee distribution to epochs pub type DistributionLeftoverCredits = SignedCredits; +/// Distributes storage fees to epochs and returns `DistributionLeftoverCredits` pub fn distribute_storage_fee_to_epochs( storage_fee: SignedCredits, start_epoch_index: EpochIndex, diff --git a/packages/rs-drive/src/fee/epoch/mod.rs b/packages/rs-drive/src/fee/epoch/mod.rs index b2f35d33a8b..2e2a23895c1 100644 --- a/packages/rs-drive/src/fee/epoch/mod.rs +++ b/packages/rs-drive/src/fee/epoch/mod.rs @@ -1,3 +1,37 @@ +// MIT LICENSE +// +// Copyright (c) 2021 Dash Core Group +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +//! Fee pool constants. +//! +//! This module defines constants related to fee distribution pools. +//! + use crate::fee::credits::{Credits, SignedCredits}; use nohash_hasher::IntMap; @@ -15,7 +49,10 @@ pub const PERPETUAL_STORAGE_YEARS: u16 = 50; /// Perpetual storage epochs pub const PERPETUAL_STORAGE_EPOCHS: u16 = PERPETUAL_STORAGE_YEARS * EPOCHS_PER_YEAR; +/// Epoch index type pub type EpochIndex = u16; +/// Credits per epoch map pub type CreditsPerEpoch = IntMap; +/// Signed credits per epoch map pub type SignedCreditsPerEpoch = IntMap; diff --git a/packages/rs-drive/src/fee/op.rs b/packages/rs-drive/src/fee/op.rs index 648751bee7a..b4ad4b65535 100644 --- a/packages/rs-drive/src/fee/op.rs +++ b/packages/rs-drive/src/fee/op.rs @@ -191,13 +191,15 @@ impl DriveOperation { // this is not always considered an error (FeeRefunds::default(), amount) } - SectionedStorageRemoval(mut removalPerEpochByIdentifier) => { - let system_amount = removalPerEpochByIdentifier + SectionedStorageRemoval(mut removal_per_epoch_by_identifier) => { + let system_amount = removal_per_epoch_by_identifier .remove(&Identifier::default()) .map_or(0, |a| a.values().sum()); ( - FeeRefunds::from_storage_removal(removalPerEpochByIdentifier)?, + FeeRefunds::from_storage_removal( + removal_per_epoch_by_identifier, + )?, system_amount, ) } diff --git a/packages/rs-drive/src/fee/result/mod.rs b/packages/rs-drive/src/fee/result/mod.rs index fc2b4fedee0..fb58c9fe489 100644 --- a/packages/rs-drive/src/fee/result/mod.rs +++ b/packages/rs-drive/src/fee/result/mod.rs @@ -34,8 +34,8 @@ use crate::error::fee::FeeError; use crate::error::Error; +use crate::fee::credits::Credits; use crate::fee::result::refunds::FeeRefunds; -use crate::fee::Credits; pub mod refunds; diff --git a/packages/rs-drive/src/fee_pools/mod.rs b/packages/rs-drive/src/fee_pools/mod.rs index 537b742164f..50c37726203 100644 --- a/packages/rs-drive/src/fee_pools/mod.rs +++ b/packages/rs-drive/src/fee_pools/mod.rs @@ -29,8 +29,8 @@ use crate::drive::batch::GroveDbOpBatch; use crate::drive::fee_pools::pools_vec_path; +use crate::fee::credits::Credits; use crate::fee::epoch::{EpochIndex, GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; -use crate::fee::Credits; use crate::fee_pools::epochs::Epoch; use crate::fee_pools::epochs_root_tree_key_constants::{ KEY_PENDING_POOL_UPDATES, KEY_STORAGE_FEE_POOL, KEY_UNPAID_EPOCH_INDEX, From 081c26c7b27fa739c7af0d0722636a078fa6782d Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sun, 18 Dec 2022 13:49:22 +0800 Subject: [PATCH 13/54] test: fix some tests --- .../fee/calculateStateTransitionFee.js | 3 ++- packages/rs-drive-abci/src/abci/handlers.rs | 1 - .../fee_pools/distribute_storage_pool.rs | 26 +++++-------------- .../execution/fee_pools/fee_distribution.rs | 3 ++- .../execution/fee_pools/process_block_fees.rs | 3 ++- packages/rs-drive-nodejs/src/fee_result.rs | 2 +- packages/rs-drive-nodejs/src/lib.rs | 9 +++---- .../rs-drive/src/drive/document/delete.rs | 5 ++-- .../rs-drive/src/drive/document/update.rs | 13 +++++----- .../epochs/credit_distribution_pools.rs | 10 ++++--- .../rs-drive/src/fee/epoch/distribution.rs | 9 +++---- 11 files changed, 37 insertions(+), 47 deletions(-) diff --git a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js index af0f2dc321e..caa13de6bfe 100644 --- a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js +++ b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js @@ -48,7 +48,8 @@ function calculateStateTransitionFee(stateTransition, options = {}) { .reduce((sum, [, credits]) => sum + credits, 0); } - const total = (storageFee + processingFee - feeRefundsSum) + DEFAULT_USER_TIP; + // Fee refunds are negative + const total = (storageFee + processingFee + feeRefundsSum) + DEFAULT_USER_TIP; executionContext.setLastCalculatedFeeDetails({ ...calculatedFees, feeRefundsSum, total }); diff --git a/packages/rs-drive-abci/src/abci/handlers.rs b/packages/rs-drive-abci/src/abci/handlers.rs index 03adc29aef0..7a537cf2b89 100644 --- a/packages/rs-drive-abci/src/abci/handlers.rs +++ b/packages/rs-drive-abci/src/abci/handlers.rs @@ -187,7 +187,6 @@ mod tests { use chrono::{Duration, Utc}; use drive::common::helpers::identities::create_test_masternode_identities; use drive::drive::batch::GroveDbOpBatch; - use drive::fee::FeeResult; use rust_decimal::prelude::ToPrimitive; use std::ops::Div; diff --git a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs index 1783ffa3337..b1ed9554679 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs @@ -37,25 +37,13 @@ use crate::error::execution::ExecutionError; use crate::error::Error; use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; -use drive::drive::fee_pools::epochs::constants::{EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS}; -use drive::error::drive::DriveError; -use drive::error::fee::FeeError; -use drive::fee::constants; -use drive::fee::credits::{Creditable, SignedCredits}; -use drive::fee::epoch::distribution::{ - distribute_storage_fee_to_epochs, DistributionLeftoverCredits, -}; -use drive::fee::epoch::{ - EpochIndex, SignedCreditsPerEpoch, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS, -}; -use drive::fee_pools::epochs::Epoch; -use drive::fee_pools::update_storage_fee_distribution_pool_operation; +use drive::fee::credits::{Creditable, Credits}; +use drive::fee::epoch::distribution::distribute_storage_fee_to_epochs; +use drive::fee::epoch::{EpochIndex, SignedCreditsPerEpoch}; use drive::grovedb::TransactionArg; -use drive::{error, grovedb}; -use rust_decimal::prelude::{FromPrimitive, ToPrimitive}; -use rust_decimal::Decimal; -/// Leftover credits after adding up and distributing storage fees. +/// Leftovers in result of divisions and rounding after storage fee distribution to epochs +pub type DistributionLeftoverCredits = Credits; impl Platform { /// Adds operations to the GroveDB op batch which calculate and distribute storage fees @@ -101,7 +89,7 @@ impl Platform { transaction, )?; - Ok(leftovers) + Ok(leftovers.to_unsigned()) } } @@ -318,7 +306,7 @@ mod tests { let total_distributed: u64 = storage_fees.iter().sum(); - assert_eq!(total_distributed + leftovers, storage_pool); + assert_eq!(total_distributed + leftovers as u64, storage_pool); /* diff --git a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs index d47a682949e..590a50fdead 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs @@ -39,6 +39,7 @@ use crate::error::Error; use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; use drive::error::fee::FeeError; +use drive::fee::credits::Credits; use drive::fee::epoch::GENESIS_EPOCH_INDEX; use drive::fee_pools::epochs::Epoch; use drive::fee_pools::{ @@ -384,7 +385,7 @@ impl Platform { &self, current_epoch: &Epoch, block_fees: &BlockFees, - cached_aggregated_storage_fees: Option, + cached_aggregated_storage_fees: Option, transaction: TransactionArg, batch: &mut GroveDbOpBatch, ) -> Result { diff --git a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs index 1a8536f8ebe..bb8c5ab0f04 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs @@ -37,16 +37,17 @@ use std::option::Option::None; use drive::drive::batch::GroveDbOpBatch; use drive::drive::fee_pools::pending_epoch_updates::add_update_pending_epoch_storage_pool_update_operations; +use drive::fee::credits::Credits; use drive::fee_pools::epochs::Epoch; use drive::grovedb::TransactionArg; use crate::abci::messages::BlockFees; use crate::block::BlockInfo; use crate::error::Error; +use crate::execution::fee_pools::distribute_storage_pool::DistributionLeftoverCredits; use crate::execution::fee_pools::epoch::EpochInfo; use crate::execution::fee_pools::fee_distribution::{FeesInPools, ProposersPayouts}; use crate::platform::Platform; -use drive::fee::epoch::distribution::DistributionLeftoverCredits; use drive::fee::epoch::{GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; use drive::fee::DEFAULT_ORIGINAL_FEE_MULTIPLIER; diff --git a/packages/rs-drive-nodejs/src/fee_result.rs b/packages/rs-drive-nodejs/src/fee_result.rs index 28d8e69ea4d..1b9821c1c6a 100644 --- a/packages/rs-drive-nodejs/src/fee_result.rs +++ b/packages/rs-drive-nodejs/src/fee_result.rs @@ -1,4 +1,4 @@ -use drive::fee::FeeResult; +use drive::fee::result::FeeResult; use neon::prelude::*; use std::ops::Deref; diff --git a/packages/rs-drive-nodejs/src/lib.rs b/packages/rs-drive-nodejs/src/lib.rs index 793cf8a6c0f..a1cc5613d4f 100644 --- a/packages/rs-drive-nodejs/src/lib.rs +++ b/packages/rs-drive-nodejs/src/lib.rs @@ -1,9 +1,7 @@ mod converter; mod fee_result; -use neon::object::PropertyKey; use std::num::ParseIntError; -use std::ops::Deref; use std::{option::Option::None, path::Path, sync::mpsc, thread}; use crate::fee_result::FeeResultWrapper; @@ -12,7 +10,8 @@ use drive::drive::batch::GroveDbOpBatch; use drive::drive::config::DriveConfig; use drive::drive::flags::StorageFlags; use drive::error::Error; -use drive::fee::refunds::CreditsPerEpoch; +use drive::fee::credits::SignedCredits; +use drive::fee::epoch::SignedCreditsPerEpoch; use drive::fee_pools::epochs::Epoch; use drive::grovedb::{PathQuery, Transaction}; use drive::query::TransactionArg; @@ -2040,7 +2039,7 @@ impl PlatformWrapper { let js_fee_refunds: Handle = js_fees.get(&mut cx, "feeRefunds")?; - let mut fee_refunds: CreditsPerEpoch = Default::default(); + let mut fee_refunds: SignedCreditsPerEpoch = Default::default(); for js_epoch_index_value in js_fee_refunds .get_own_property_names(&mut cx)? @@ -2054,7 +2053,7 @@ impl PlatformWrapper { .or_else(|e: ParseIntError| cx.throw_error(e.to_string()))?; let js_credits: Handle = js_fee_refunds.get(&mut cx, js_epoch_index)?; - let credits = js_credits.value(&mut cx) as u64; + let credits = js_credits.value(&mut cx) as SignedCredits; fee_refunds.insert(epoch_index, credits); } diff --git a/packages/rs-drive/src/drive/document/delete.rs b/packages/rs-drive/src/drive/document/delete.rs index 5d27514deab..e4180bc46f1 100644 --- a/packages/rs-drive/src/drive/document/delete.rs +++ b/packages/rs-drive/src/drive/document/delete.rs @@ -733,6 +733,7 @@ mod tests { use crate::drive::object_size_info::DocumentAndContractInfo; use crate::drive::object_size_info::DocumentInfo::DocumentRefAndSerialization; use crate::drive::Drive; + use crate::fee::credits::Creditable; use crate::fee::default_costs::STORAGE_DISK_USAGE_CREDIT_PER_BYTE; use crate::fee_pools::epochs::Epoch; use crate::query::DriveQuery; @@ -1453,10 +1454,10 @@ mod tests { .fee_refunds .get(&random_owner_id) .unwrap() - .get(0) + .get(&0) .unwrap(); - let removed_bytes = removed_credits / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; + let removed_bytes = removed_credits.to_unsigned() / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; assert_eq!(added_bytes, removed_bytes); } diff --git a/packages/rs-drive/src/drive/document/update.rs b/packages/rs-drive/src/drive/document/update.rs index 50f54289b52..cf965526bfc 100644 --- a/packages/rs-drive/src/drive/document/update.rs +++ b/packages/rs-drive/src/drive/document/update.rs @@ -644,6 +644,7 @@ mod tests { use crate::drive::object_size_info::DocumentAndContractInfo; use crate::drive::object_size_info::DocumentInfo::DocumentRefAndSerialization; use crate::drive::{defaults, Drive}; + use crate::fee::credits::Creditable; use crate::fee::default_costs::STORAGE_DISK_USAGE_CREDIT_PER_BYTE; use crate::query::DriveQuery; use crate::{ @@ -1520,10 +1521,10 @@ mod tests { .fee_refunds .get(&owner_id) .unwrap() - .get(0) + .get(&0) .unwrap(); - let removed_bytes = removed_credits / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; + let removed_bytes = removed_credits.to_unsigned() / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; assert_eq!(original_bytes, removed_bytes); @@ -1639,10 +1640,10 @@ mod tests { .fee_refunds .get(&owner_id) .unwrap() - .get(0) + .get(&0) .unwrap(); - let removed_bytes = removed_credits / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; + let removed_bytes = removed_credits.to_unsigned() / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; assert_eq!(original_bytes, removed_bytes); @@ -1678,10 +1679,10 @@ mod tests { .fee_refunds .get(&owner_id) .unwrap() - .get(0) + .get(&0) .unwrap(); - let removed_bytes = removed_credits / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; + let removed_bytes = removed_credits.to_unsigned() / STORAGE_DISK_USAGE_CREDIT_PER_BYTE; // We added one byte, and since it is an index, and keys are doubled it's 2 extra bytes let expected_added_bytes = if using_history { 607 } else { 605 }; diff --git a/packages/rs-drive/src/drive/fee_pools/epochs/credit_distribution_pools.rs b/packages/rs-drive/src/drive/fee_pools/epochs/credit_distribution_pools.rs index 00f8954512c..59d45ae1cde 100644 --- a/packages/rs-drive/src/drive/fee_pools/epochs/credit_distribution_pools.rs +++ b/packages/rs-drive/src/drive/fee_pools/epochs/credit_distribution_pools.rs @@ -37,6 +37,7 @@ use grovedb::{Element, TransactionArg}; use crate::drive::Drive; use crate::error::fee::FeeError; use crate::error::Error; +use crate::fee::credits::Credits; use crate::fee::get_overflow_error; use crate::fee_pools::epochs::Epoch; @@ -79,7 +80,7 @@ impl Drive { &self, epoch_tree: &Epoch, transaction: TransactionArg, - ) -> Result { + ) -> Result { let element = self .grove .get( @@ -141,7 +142,7 @@ impl Drive { &self, epoch_tree: &Epoch, transaction: TransactionArg, - ) -> Result { + ) -> Result { let storage_pool_credits = self.get_epoch_storage_credits_for_distribution(epoch_tree, transaction)?; @@ -160,6 +161,7 @@ mod tests { use crate::drive::batch::GroveDbOpBatch; use crate::error; use crate::error::fee::FeeError; + use crate::fee::credits::Credits; use crate::fee_pools::epochs::epoch_key_constants; use crate::fee_pools::epochs::Epoch; use grovedb::Element; @@ -260,8 +262,8 @@ mod tests { let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let processing_fee: u64 = 42; - let storage_fee: u64 = 1000; + let processing_fee: Credits = 42; + let storage_fee: Credits = 1000; let epoch = Epoch::new(0); diff --git a/packages/rs-drive/src/fee/epoch/distribution.rs b/packages/rs-drive/src/fee/epoch/distribution.rs index ab290e78cbf..69574b11353 100644 --- a/packages/rs-drive/src/fee/epoch/distribution.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -34,7 +34,7 @@ use crate::error::fee::FeeError; use crate::error::Error; -use crate::fee::credits::SignedCredits; +use crate::fee::credits::{Creditable, Credits, SignedCredits}; use crate::fee::epoch::{ EpochIndex, SignedCreditsPerEpoch, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS, }; @@ -60,16 +60,13 @@ pub const FEE_DISTRIBUTION_TABLE: [Decimal; PERPETUAL_STORAGE_YEARS as usize] = dec!(0.00325), dec!(0.00275), dec!(0.00225), dec!(0.00175), dec!(0.00125), ]; -/// Leftovers in result of divisions and rounding after storage fee distribution to epochs -pub type DistributionLeftoverCredits = SignedCredits; - -/// Distributes storage fees to epochs and returns `DistributionLeftoverCredits` +/// Distributes storage fees to epochs and returns leftovers pub fn distribute_storage_fee_to_epochs( storage_fee: SignedCredits, start_epoch_index: EpochIndex, from_epoch_index: EpochIndex, credits_per_epochs: &mut SignedCreditsPerEpoch, -) -> Result { +) -> Result { if storage_fee == 0 { return Ok(0); } From e372dad01db05a16626d9ff24f51ec4f4253bdf4 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sun, 18 Dec 2022 13:52:12 +0800 Subject: [PATCH 14/54] test: fix some tests --- .../lib/stateTransition/fee/calculateStateTransitionFee.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js index caa13de6bfe..515119f82a7 100644 --- a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js +++ b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js @@ -44,6 +44,7 @@ function calculateStateTransitionFee(stateTransition, options = {}) { throw new Error('State Transition removed bytes from different identity'); } + // TODO: We should deal with leftovers feeRefundsSum = feeRefunds[0].creditsPerEpoch.entries() .reduce((sum, [, credits]) => sum + credits, 0); } From 9a1a071353f88df6d1d7dd166509cbf36abe9456 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sun, 18 Dec 2022 16:20:51 +0800 Subject: [PATCH 15/54] test: implement tests for distribution --- .../fee_pools/distribute_storage_pool.rs | 100 ++++----- packages/rs-drive/src/drive/fee_pools/mod.rs | 16 +- packages/rs-drive/src/fee/credits.rs | 3 + .../rs-drive/src/fee/epoch/distribution.rs | 208 ++++++++++++++++-- packages/rs-drive/src/fee/epoch/mod.rs | 8 +- 5 files changed, 258 insertions(+), 77 deletions(-) diff --git a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs index b1ed9554679..843b74ee0e5 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs @@ -101,20 +101,22 @@ mod tests { use drive::common::helpers::epoch::get_storage_credits_for_distribution_for_epochs_in_range; use drive::drive::batch::GroveDbOpBatch; use drive::error::drive::DriveError; + use drive::fee::credits::Credits; + use drive::fee::epoch::GENESIS_EPOCH_INDEX; use drive::fee_pools::epochs::Epoch; use drive::fee_pools::update_storage_fee_distribution_pool_operation; - + #[test] fn test_nothing_to_distribute() { let platform = setup_platform_with_initial_state_structure(); let transaction = platform.drive.grove.start_transaction(); - + let epoch_index = 0; - + // Storage fee distribution pool is 0 after fee pools initialization - + let mut batch = GroveDbOpBatch::new(); - + platform .add_distribute_storage_fee_to_epochs_operations( epoch_index, @@ -122,7 +124,7 @@ mod tests { &mut batch, ) .expect("should distribute storage fee pool"); - + match platform .drive .grove_apply_batch(batch, false, Some(&transaction)) @@ -133,38 +135,38 @@ mod tests { _ => assert!(false, "invalid error type"), }, } - + let storage_fees = get_storage_credits_for_distribution_for_epochs_in_range( &platform.drive, epoch_index..1000, Some(&transaction), ); - + let reference_fees: Vec = (0..1000).map(|_| 0u64).collect(); - + assert_eq!(storage_fees, reference_fees); } - + #[test] fn test_distribution_overflow() { let platform = setup_platform_with_initial_state_structure(); let transaction = platform.drive.grove.start_transaction(); - - let storage_pool = u64::MAX; - let epoch_index = 0; - + + let storage_pool = i64::MAX as u64; + let epoch_index = GENESIS_EPOCH_INDEX; + let mut batch = GroveDbOpBatch::new(); - + batch.push(update_storage_fee_distribution_pool_operation(storage_pool)); - + // Apply storage fee distribution pool update platform .drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - + let mut batch = GroveDbOpBatch::new(); - + let leftovers = platform .add_distribute_storage_fee_to_epochs_operations( epoch_index, @@ -172,42 +174,42 @@ mod tests { &mut batch, ) .expect("should distribute storage fee pool"); - + platform .drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - + // check leftover - assert_eq!(leftovers, 515); + assert_eq!(leftovers, 507); } - + #[test] fn test_deterministic_distribution() { let platform = setup_platform_with_initial_state_structure(); let transaction = platform.drive.grove.start_transaction(); - + let storage_pool = 1000000; let epoch_index = 42; - + let mut batch = GroveDbOpBatch::new(); - + // init additional epochs pools as it will be done in epoch_change for i in 1000..=1000 + epoch_index { let epoch = Epoch::new(i); epoch.add_init_empty_operations(&mut batch); } - + batch.push(update_storage_fee_distribution_pool_operation(storage_pool)); - + // Apply storage fee distribution pool update platform .drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - + let mut batch = GroveDbOpBatch::new(); - + let leftovers = platform .add_distribute_storage_fee_to_epochs_operations( epoch_index, @@ -215,22 +217,22 @@ mod tests { &mut batch, ) .expect("should distribute storage fee pool"); - + platform .drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - + // check leftover assert_eq!(leftovers, 180); - + // collect all the storage fee values of the 1000 epochs pools let storage_fees = get_storage_credits_for_distribution_for_epochs_in_range( &platform.drive, epoch_index..epoch_index + 1000, Some(&transaction), ); - + // compare them with reference table #[rustfmt::skip] let reference_fees: [u64; 1000] = [ @@ -301,32 +303,32 @@ mod tests { 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62 ]; - + assert_eq!(storage_fees, reference_fees); - - let total_distributed: u64 = storage_fees.iter().sum(); - - assert_eq!(total_distributed + leftovers as u64, storage_pool); - + + let total_distributed: Credits = storage_fees.iter().sum(); + + assert_eq!(total_distributed + leftovers, storage_pool); + /* - + Repeat distribution to ensure deterministic results - + */ - + let mut batch = GroveDbOpBatch::new(); - + // refill storage fee pool once more batch.push(update_storage_fee_distribution_pool_operation(storage_pool)); - + // Apply storage fee distribution pool update platform .drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - + let mut batch = GroveDbOpBatch::new(); - + // distribute fees once more platform .add_distribute_storage_fee_to_epochs_operations( @@ -335,19 +337,19 @@ mod tests { &mut batch, ) .expect("should distribute storage fee pool"); - + platform .drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - + // collect all the storage fee values of the 1000 epochs pools again let storage_fees = get_storage_credits_for_distribution_for_epochs_in_range( &platform.drive, epoch_index..epoch_index + 1000, Some(&transaction), ); - + // assert that all the values doubled meaning that distribution is reproducible assert_eq!( storage_fees, diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index 9201019f5a6..6220a4d641c 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -127,12 +127,16 @@ impl Drive { ))); } - for (i, (epoch_index, credits)) in credits_per_epochs - .into_iter() - .sorted_by_key(|x| x.0) - .enumerate() - { - let existing_storage_fee = SignedCredits::from_vec_bytes(storage_fee_pools.remove(i))?; + // Sort epochs to reverse order to pop existing values + for (epoch_index, credits) in credits_per_epochs.into_iter().sorted_by_key(|x| !x.0) { + let encoded_existing_storage_fee = + storage_fee_pools + .pop() + .ok_or(Error::Drive(DriveError::CorruptedCodeExecution( + "value must be present", + )))?; + + let existing_storage_fee = SignedCredits::from_vec_bytes(encoded_existing_storage_fee)?; let credits_to_update = existing_storage_fee.checked_add(credits).ok_or_else(|| { get_overflow_error("can't add credits to existing epoch pool storage fee") diff --git a/packages/rs-drive/src/fee/credits.rs b/packages/rs-drive/src/fee/credits.rs index 33381f2b1bd..90d4a2bf5fd 100644 --- a/packages/rs-drive/src/fee/credits.rs +++ b/packages/rs-drive/src/fee/credits.rs @@ -44,6 +44,9 @@ pub type Credits = u64; /// balance verification pub type SignedCredits = i64; +/// Maximum value of credits +pub const MAX: Credits = SignedCredits::MAX as Credits; + /// Trait for signed and unsigned credits pub trait Creditable: Into { /// Convert unsigned credit to singed diff --git a/packages/rs-drive/src/fee/epoch/distribution.rs b/packages/rs-drive/src/fee/epoch/distribution.rs index 69574b11353..7262827e569 100644 --- a/packages/rs-drive/src/fee/epoch/distribution.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -34,7 +34,7 @@ use crate::error::fee::FeeError; use crate::error::Error; -use crate::fee::credits::{Creditable, Credits, SignedCredits}; +use crate::fee::credits::SignedCredits; use crate::fee::epoch::{ EpochIndex, SignedCreditsPerEpoch, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS, }; @@ -119,27 +119,199 @@ pub fn distribute_storage_fee_to_epochs( #[cfg(test)] mod tests { - use rust_decimal::Decimal; - use rust_decimal_macros::dec; - - #[test] - fn test_distribution_table_sum() { - assert_eq!( - super::FEE_DISTRIBUTION_TABLE.iter().sum::(), - dec!(1.0), - ); + use super::*; + + use crate::fee::credits::{Creditable, MAX}; + use crate::fee::epoch::GENESIS_EPOCH_INDEX; + + mod fee_distribution_table { + use super::*; + + #[test] + fn should_have_sum_of_1() { + assert_eq!(FEE_DISTRIBUTION_TABLE.iter().sum::(), dec!(1.0),); + } + + #[test] + fn should_distribute_value() { + let mut buffer = dec!(0.0); + let value = Decimal::from(i64::MAX); + + for i in FEE_DISTRIBUTION_TABLE { + let share = value + * FEE_DISTRIBUTION_TABLE[i.to_usize().expect("should be convertable to usize")]; + + buffer += share; + } + + assert_eq!(buffer, value); + } } - #[test] - fn test_distribution_of_value() { - let mut buffer = dec!(0.0); - let value = Decimal::new(i64::MAX, 0); + mod distribute_storage_fee_to_epochs { + use super::*; + + #[test] + fn should_distribute_nothing_if_storage_fee_are_zero() { + let mut credits_per_epochs = SignedCreditsPerEpoch::default(); + + let leftovers = distribute_storage_fee_to_epochs( + 0, + GENESIS_EPOCH_INDEX, + GENESIS_EPOCH_INDEX, + &mut credits_per_epochs, + ) + .expect("should distribute storage fee"); - for i in 0..50 { - let share = value * super::FEE_DISTRIBUTION_TABLE[i]; - buffer += share; + assert_eq!(credits_per_epochs.len(), 0); + assert_eq!(leftovers, 0); } - assert_eq!(buffer, value); + #[test] + fn should_distribute_max_credits_value_without_overflow() { + let storage_fee = MAX.to_signed().expect("should convert signed credits"); + + let mut credits_per_epochs = SignedCreditsPerEpoch::default(); + + let leftovers = distribute_storage_fee_to_epochs( + storage_fee, + GENESIS_EPOCH_INDEX, + GENESIS_EPOCH_INDEX, + &mut credits_per_epochs, + ) + .expect("should distribute storage fee"); + + // check leftover + assert_eq!(leftovers, 507); + } + + #[test] + fn should_deterministically_distribute_fees() { + let storage_fee = 1000000; + let current_epoch_index = 42; + + let mut credits_per_epochs = SignedCreditsPerEpoch::default(); + + let leftovers = distribute_storage_fee_to_epochs( + storage_fee, + current_epoch_index, + current_epoch_index, + &mut credits_per_epochs, + ) + .expect("should distribute storage fee"); + + // check leftover + assert_eq!(leftovers, 180); + + // compare them with reference table + #[rustfmt::skip] + let reference_fees: [SignedCredits; 1000] = [ + 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, + 2500, 2500, 2500, 2500, 2500, 2500, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, + 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2300, 2300, + 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, + 2300, 2300, 2300, 2300, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, + 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2100, 2100, 2100, 2100, + 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, + 2100, 2100, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, + 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 1925, 1925, 1925, 1925, 1925, 1925, + 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, + 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, + 1850, 1850, 1850, 1850, 1850, 1850, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, + 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1700, 1700, + 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, + 1700, 1700, 1700, 1700, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, + 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1550, 1550, 1550, 1550, + 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, + 1550, 1550, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, + 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1425, 1425, 1425, 1425, 1425, 1425, + 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, + 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, + 1375, 1375, 1375, 1375, 1375, 1375, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, + 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1275, 1275, + 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, + 1275, 1275, 1275, 1275, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, + 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1175, 1175, 1175, 1175, + 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, + 1175, 1175, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, + 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1075, 1075, 1075, 1075, 1075, 1075, + 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, + 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, + 1025, 1025, 1025, 1025, 1025, 1025, 975, 975, 975, 975, 975, 975, 975, 975, 975, + 975, 975, 975, 975, 975, 975, 975, 975, 975, 975, 975, 937, 937, 937, 937, 937, + 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 900, + 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, + 900, 900, 900, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, + 862, 862, 862, 862, 862, 862, 862, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, 787, 787, 787, 787, 787, + 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 750, + 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, + 750, 750, 750, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, + 712, 712, 712, 712, 712, 712, 712, 675, 675, 675, 675, 675, 675, 675, 675, 675, + 675, 675, 675, 675, 675, 675, 675, 675, 675, 675, 675, 637, 637, 637, 637, 637, + 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 600, + 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, + 600, 600, 600, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, + 562, 562, 562, 562, 562, 562, 562, 525, 525, 525, 525, 525, 525, 525, 525, 525, + 525, 525, 525, 525, 525, 525, 525, 525, 525, 525, 525, 487, 487, 487, 487, 487, + 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 450, + 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, + 450, 450, 450, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, + 412, 412, 412, 412, 412, 412, 412, 375, 375, 375, 375, 375, 375, 375, 375, 375, + 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 337, 337, 337, 337, 337, + 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 300, + 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, + 300, 300, 300, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, + 262, 262, 262, 262, 262, 262, 262, 237, 237, 237, 237, 237, 237, 237, 237, 237, + 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, 212, 212, 212, 212, 212, + 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 187, + 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + 187, 187, 187, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, + 162, 162, 162, 162, 162, 162, 162, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 112, 112, 112, 112, 112, + 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 87, + 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62 + ]; + + assert_eq!( + credits_per_epochs + .clone() + .into_values() + .collect::>(), + reference_fees + ); + + let total_distributed: SignedCredits = credits_per_epochs.values().sum(); + + assert_eq!(total_distributed + leftovers, storage_fee); + + /* + + Repeat distribution to ensure deterministic results + + */ + + let leftovers = distribute_storage_fee_to_epochs( + storage_fee, + current_epoch_index, + current_epoch_index, + &mut credits_per_epochs, + ) + .expect("should distribute storage fee"); + + // assert that all the values doubled meaning that distribution is reproducible + assert_eq!( + credits_per_epochs + .into_values() + .collect::>(), + reference_fees + .into_iter() + .map(|val| val * 2) + .collect::>() + ); + + assert_eq!(leftovers, 180); + } } } diff --git a/packages/rs-drive/src/fee/epoch/mod.rs b/packages/rs-drive/src/fee/epoch/mod.rs index 2e2a23895c1..01c203d10b7 100644 --- a/packages/rs-drive/src/fee/epoch/mod.rs +++ b/packages/rs-drive/src/fee/epoch/mod.rs @@ -37,8 +37,11 @@ use nohash_hasher::IntMap; pub mod distribution; +/// Epoch index type +pub type EpochIndex = u16; + /// Genesis epoch index -pub const GENESIS_EPOCH_INDEX: u16 = 0; +pub const GENESIS_EPOCH_INDEX: EpochIndex = 0; /// Epochs per year pub const EPOCHS_PER_YEAR: u16 = 20; @@ -49,9 +52,6 @@ pub const PERPETUAL_STORAGE_YEARS: u16 = 50; /// Perpetual storage epochs pub const PERPETUAL_STORAGE_EPOCHS: u16 = PERPETUAL_STORAGE_YEARS * EPOCHS_PER_YEAR; -/// Epoch index type -pub type EpochIndex = u16; - /// Credits per epoch map pub type CreditsPerEpoch = IntMap; /// Signed credits per epoch map From 36282443d6699ba3b4021e038d869e467e15b7c2 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Sun, 18 Dec 2022 23:19:43 +0800 Subject: [PATCH 16/54] tests: update_epoch_storage_fee_pools in progress --- .../src/drive/batch/grovedb_op_batch.rs | 11 +++ packages/rs-drive/src/drive/fee_pools/mod.rs | 99 ++++++++++++++----- .../drive/fee_pools/pending_epoch_updates.rs | 2 +- .../rs-drive/src/fee/epoch/distribution.rs | 5 + packages/rs-drive/src/fee_pools/mod.rs | 47 +++++++++ 5 files changed, 137 insertions(+), 27 deletions(-) diff --git a/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs b/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs index 52898488b7b..e010f418ac0 100644 --- a/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs +++ b/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs @@ -35,6 +35,8 @@ use crate::drive::flags::StorageFlags; use grovedb::batch::{GroveDbOp, GroveDbOpConsistencyResults}; use grovedb::Element; +// use std::slice::Iter; +// use std::vec::IntoIter; /// A batch of GroveDB operations as a vector. // TODO move to GroveDB @@ -114,3 +116,12 @@ impl GroveDbOpBatch { GroveDbOp::verify_consistency_of_operations(&self.operations) } } + +impl IntoIterator for GroveDbOpBatch { + type Item = GroveDbOp; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.operations.into_iter() + } +} diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index 6220a4d641c..03b57bb5f77 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -123,7 +123,7 @@ impl Drive { if storage_fee_pools.len() != credits_per_epochs.len() { return Err(Error::Drive(DriveError::CorruptedCodeExecution( - "number of fetched epoch storage fee pools must be equal to requested for update", + "number of fetched epoch storage fee pools must be equal to requested for update", ))); } @@ -157,46 +157,93 @@ impl Drive { #[cfg(test)] mod tests { + use super::*; + use crate::common::helpers::setup::setup_drive_with_initial_state_structure; - use crate::error; - use crate::fee_pools::epochs::Epoch; - mod create_fee_pool_trees { + mod add_update_epoch_storage_fee_pools_operations { + use super::*; + use crate::fee::credits::Credits; + use crate::fee::epoch::{EpochIndex, GENESIS_EPOCH_INDEX}; + use grovedb::batch::GroveDbOp; + #[test] - fn test_values_are_set() { - let drive = super::setup_drive_with_initial_state_structure(); + fn should_do_nothing_if_credits_per_epoch_are_empty() { + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let storage_fee_pool = drive - .get_aggregate_storage_fees_from_distribution_pool(Some(&transaction)) - .expect("should get storage fee pool"); + let credits_per_epoch = SignedCreditsPerEpoch::default(); + + let mut batch = GroveDbOpBatch::new(); + + drive + .add_update_epoch_storage_fee_pools_operations( + &mut batch, + credits_per_epoch, + Some(&transaction), + ) + .expect("should update epoch storage pools"); - assert_eq!(storage_fee_pool, 0u64); + assert_eq!(batch.len(), 0); } #[test] - fn test_epoch_trees_are_created() { - let drive = super::setup_drive_with_initial_state_structure(); + fn should_update_epoch_storage_fee_pools() { + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - for epoch_index in 0..1000 { - let epoch = super::Epoch::new(epoch_index); + const TO_EPOCH_INDEX: EpochIndex = 1; - let storage_fee = drive - .get_epoch_storage_credits_for_distribution(&epoch, Some(&transaction)) - .expect("should get storage fee"); + // Store initial epoch storage pool values + let operations = (GENESIS_EPOCH_INDEX..TO_EPOCH_INDEX) + .map(|epoch_index| { + let credits = 10 - epoch_index as Credits; - assert_eq!(storage_fee, 0); - } + let element = Element::new_item(credits.to_unsigned().to_vec_bytes()); + + GroveDbOp::insert_op( + Epoch::new(epoch_index).get_vec_path(), + KEY_POOL_STORAGE_FEES.to_vec(), + element, + ) + }) + .collect(); + + let batch = GroveDbOpBatch::from_operations(operations); + + drive + .grove_apply_batch(batch, false, Some(&transaction)) + .expect("should apply batch"); + + let credits_to_epochs: SignedCreditsPerEpoch = (GENESIS_EPOCH_INDEX..TO_EPOCH_INDEX) + .map(|epoch_index| (epoch_index, epoch_index as SignedCredits)) + .collect(); + + let mut batch = GroveDbOpBatch::new(); + + drive + .add_update_epoch_storage_fee_pools_operations( + &mut batch, + credits_to_epochs, + Some(&transaction), + ) + .expect("should update epoch storage pools"); + + assert_eq!(batch.len(), TO_EPOCH_INDEX as usize); + + for operation in batch.into_iter() { + assert!(matches!(operation.op, Op::Delete)); + + assert_eq!(operation.path.to_path(), pools_pending_updates_path()); - let epoch = super::Epoch::new(1000); // 1001th epochs pool + let epoch_index_key = operation.key.get_key(); + let epoch_index = u16::from_be_bytes( + epoch_index_key + .try_into() + .expect("should convert to u16 bytes"), + ); - match drive.get_epoch_storage_credits_for_distribution(&epoch, Some(&transaction)) { - Ok(_) => assert!(false, "must be an error"), - Err(e) => match e { - super::error::Error::GroveDB(_) => assert!(true), - _ => assert!(false, "invalid error type"), - }, + assert!(expected_pending_updates.contains_key(&epoch_index)); } } } diff --git a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs index ca1826ce651..847bf89f628 100644 --- a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs +++ b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs @@ -293,7 +293,7 @@ mod tests { assert_eq!(batch.len(), expected_pending_updates.len()); - for operation in batch.operations { + for operation in batch.into_iter() { assert!(matches!(operation.op, Op::Delete)); assert_eq!(operation.path.to_path(), pools_pending_updates_path()); diff --git a/packages/rs-drive/src/fee/epoch/distribution.rs b/packages/rs-drive/src/fee/epoch/distribution.rs index 7262827e569..1361ba51ee3 100644 --- a/packages/rs-drive/src/fee/epoch/distribution.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -313,5 +313,10 @@ mod tests { assert_eq!(leftovers, 180); } + + #[test] + fn should_distribute_from_specific_epoch() { + todo!(); + } } } diff --git a/packages/rs-drive/src/fee_pools/mod.rs b/packages/rs-drive/src/fee_pools/mod.rs index 50c37726203..ff72b8ffb3d 100644 --- a/packages/rs-drive/src/fee_pools/mod.rs +++ b/packages/rs-drive/src/fee_pools/mod.rs @@ -85,3 +85,50 @@ pub fn update_unpaid_epoch_index_operation(epoch_index: EpochIndex) -> GroveDbOp } // TODD: Find tests + +#[cfg(test)] +mod tests { + use super::*; + use crate::common::helpers::setup::setup_drive_with_initial_state_structure; + + mod add_create_fee_pool_trees_operations { + use crate::error::Error; + + #[test] + fn test_values_are_set() { + let drive = super::setup_drive_with_initial_state_structure(); + let transaction = drive.grove.start_transaction(); + + let storage_fee_pool = drive + .get_aggregate_storage_fees_from_distribution_pool(Some(&transaction)) + .expect("should get storage fee pool"); + + assert_eq!(storage_fee_pool, 0u64); + } + + #[test] + fn test_epoch_trees_are_created() { + let drive = super::setup_drive_with_initial_state_structure(); + let transaction = drive.grove.start_transaction(); + + for epoch_index in 0..1000 { + let epoch = super::Epoch::new(epoch_index); + + let storage_fee = drive + .get_epoch_storage_credits_for_distribution(&epoch, Some(&transaction)) + .expect("should get storage fee"); + + assert_eq!(storage_fee, 0); + } + + let epoch = super::Epoch::new(1000); // 1001th epochs pool + + match drive.get_epoch_storage_credits_for_distribution(&epoch, Some(&transaction)) { + Ok(_) => unreachable!("must be an error"), + Err(e) => { + assert!(matches!(e, Error::GroveDB(..))); + } + } + } + } +} From 0e1773d07abf4c3a122f62ab4daaf9417c2258b4 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Mon, 19 Dec 2022 13:55:30 +0800 Subject: [PATCH 17/54] tests: add_update_epoch_storage_fee_pools_operations --- packages/rs-drive/src/drive/fee_pools/mod.rs | 28 +++++++++++--------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index 03b57bb5f77..d54641adb0c 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -165,7 +165,7 @@ mod tests { use super::*; use crate::fee::credits::Credits; use crate::fee::epoch::{EpochIndex, GENESIS_EPOCH_INDEX}; - use grovedb::batch::GroveDbOp; + use grovedb::batch::{GroveDbOp, Op}; #[test] fn should_do_nothing_if_credits_per_epoch_are_empty() { @@ -192,10 +192,11 @@ mod tests { let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - const TO_EPOCH_INDEX: EpochIndex = 1; + const TO_EPOCH_INDEX: EpochIndex = 10; // Store initial epoch storage pool values let operations = (GENESIS_EPOCH_INDEX..TO_EPOCH_INDEX) + .into_iter() .map(|epoch_index| { let credits = 10 - epoch_index as Credits; @@ -231,19 +232,22 @@ mod tests { assert_eq!(batch.len(), TO_EPOCH_INDEX as usize); - for operation in batch.into_iter() { - assert!(matches!(operation.op, Op::Delete)); + for (i, operation) in batch.into_iter().rev().enumerate() { + assert_eq!(operation.key.get_key(), KEY_POOL_STORAGE_FEES); - assert_eq!(operation.path.to_path(), pools_pending_updates_path()); - - let epoch_index_key = operation.key.get_key(); - let epoch_index = u16::from_be_bytes( - epoch_index_key - .try_into() - .expect("should convert to u16 bytes"), + assert_eq!( + operation.path.to_path(), + Epoch::new(i as u16).get_vec_path() ); - assert!(expected_pending_updates.contains_key(&epoch_index)); + let Op::Insert{ element: Element::Item (encoded_credits, _)} = operation.op else { + panic!("invalid operation"); + }; + + let credits = + Credits::from_vec_bytes(encoded_credits).expect("should decide credits"); + + assert_eq!(credits, 10); } } } From dffb8bea8acd9c2bc95ee64eef26b2f1150d094d Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Mon, 19 Dec 2022 23:32:45 +0800 Subject: [PATCH 18/54] tests: fix all tests --- Cargo.lock | 1 + .../fee/calculateStateTransitionFee.js | 3 +- packages/rs-drive-abci/Cargo.toml | 2 + .../fee_pools/distribute_storage_pool.rs | 410 ++++++------------ packages/rs-drive/src/drive/fee_pools/mod.rs | 36 +- .../drive/fee_pools/pending_epoch_updates.rs | 4 + .../src/drive/fee_pools/unpaid_epoch.rs | 3 +- .../rs-drive/src/fee/epoch/distribution.rs | 239 ++++++++-- packages/rs-drive/src/fee/result/refunds.rs | 9 +- packages/rs-drive/src/fee_pools/epochs/mod.rs | 5 +- packages/rs-drive/src/fee_pools/mod.rs | 29 +- 11 files changed, 391 insertions(+), 350 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d826fa3c320..ce090108135 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -942,6 +942,7 @@ dependencies = [ "dashcore 0.29.1 (git+https://github.com/dashpay/rust-dashcore?branch=master)", "drive", "hex", + "itertools", "rand 0.8.5", "rust_decimal", "rust_decimal_macros", diff --git a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js index 515119f82a7..a63581f3b2a 100644 --- a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js +++ b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js @@ -44,7 +44,8 @@ function calculateStateTransitionFee(stateTransition, options = {}) { throw new Error('State Transition removed bytes from different identity'); } - // TODO: We should deal with leftovers + // TODO: We should deduct leftovers + feeRefundsSum = feeRefunds[0].creditsPerEpoch.entries() .reduce((sum, [, credits]) => sum + credits, 0); } diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml index 18b26a94175..0e2ac9a22cb 100644 --- a/packages/rs-drive-abci/Cargo.toml +++ b/packages/rs-drive-abci/Cargo.toml @@ -21,3 +21,5 @@ hex = "0.4.3" dashcore = { git="https://github.com/dashpay/rust-dashcore", features=["no-std", "secp-recovery", "rand", "signer"], default-features = false, branch="master" } rust_decimal = "1.2.5" rust_decimal_macros = "1.25.0" +itertools = { version = "0.10.5" } + diff --git a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs index 843b74ee0e5..7d1617e2c5e 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs @@ -38,7 +38,7 @@ use crate::error::Error; use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; use drive::fee::credits::{Creditable, Credits}; -use drive::fee::epoch::distribution::distribute_storage_fee_to_epochs; +use drive::fee::epoch::distribution::distribute_storage_fee_to_epochs_collection; use drive::fee::epoch::{EpochIndex, SignedCreditsPerEpoch}; use drive::grovedb::TransactionArg; @@ -60,34 +60,41 @@ impl Platform { let mut credits_per_epochs = SignedCreditsPerEpoch::default(); - let mut leftovers = distribute_storage_fee_to_epochs( + // Distribute from storage distribution pool + let leftovers = distribute_storage_fee_to_epochs_collection( + &mut credits_per_epochs, storage_distribution_fees.to_signed()?, current_epoch_index, - current_epoch_index, - &mut credits_per_epochs, + None, )?; + // Deduct refunds since epoch where data was removed skipping already paid or pay-in-progress epochs. + // Leftovers are ignored since they already deducted from Identity's refund amount + + let mut unpaid_epoch_index = self.drive.get_unpaid_epoch_index(transaction)?; + + // In case if we paying for older than previous epoch + // we need to switch to next one which we are not paying yet + if unpaid_epoch_index < current_epoch_index { + unpaid_epoch_index += 1; + }; + // TODO Better to use iterator do not load everything into memory for (epoch_index, credits) in self.drive.fetch_pending_updates(transaction)? { - let credits_to_distribute = credits.checked_add(leftovers).ok_or_else(|| { - Error::Execution(ExecutionError::Overflow( - "can't add leftovers to pending storage fees", - )) - })?; - - leftovers = distribute_storage_fee_to_epochs( - credits_to_distribute, - epoch_index, - current_epoch_index, + distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, + credits, + epoch_index, + Some(unpaid_epoch_index), )?; } - self.drive.add_update_epoch_storage_fee_pools_operations( - batch, - credits_per_epochs, - transaction, - )?; + self.drive + .add_update_epoch_storage_fee_pools_sequence_operations( + batch, + credits_per_epochs, + transaction, + )?; Ok(leftovers.to_unsigned()) } @@ -95,331 +102,166 @@ impl Platform { #[cfg(test)] mod tests { + use super::*; + + use crate::common::helpers::setup::setup_platform_with_initial_state_structure; + use drive::common::helpers::epoch::get_storage_credits_for_distribution_for_epochs_in_range; - mod add_distribute_storage_fees_to_epochs_operations { - use crate::common::helpers::setup::setup_platform_with_initial_state_structure; - use drive::common::helpers::epoch::get_storage_credits_for_distribution_for_epochs_in_range; - use drive::drive::batch::GroveDbOpBatch; - use drive::error::drive::DriveError; - use drive::fee::credits::Credits; - use drive::fee::epoch::GENESIS_EPOCH_INDEX; + mod add_distribute_storage_fee_to_epochs_operations { + use drive::drive::fee_pools::pending_epoch_updates::add_update_pending_epoch_storage_pool_update_operations; + use drive::fee::credits::SignedCredits; + use drive::fee::epoch::{GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; use drive::fee_pools::epochs::Epoch; - use drive::fee_pools::update_storage_fee_distribution_pool_operation; - - #[test] - fn test_nothing_to_distribute() { - let platform = setup_platform_with_initial_state_structure(); - let transaction = platform.drive.grove.start_transaction(); - - let epoch_index = 0; - - // Storage fee distribution pool is 0 after fee pools initialization - - let mut batch = GroveDbOpBatch::new(); - - platform - .add_distribute_storage_fee_to_epochs_operations( - epoch_index, - Some(&transaction), - &mut batch, - ) - .expect("should distribute storage fee pool"); - - match platform - .drive - .grove_apply_batch(batch, false, Some(&transaction)) - { - Ok(()) => assert!(false, "should return BatchIsEmpty error"), - Err(e) => match e { - drive::error::Error::Drive(DriveError::BatchIsEmpty()) => assert!(true), - _ => assert!(false, "invalid error type"), - }, - } - - let storage_fees = get_storage_credits_for_distribution_for_epochs_in_range( - &platform.drive, - epoch_index..1000, - Some(&transaction), - ); - - let reference_fees: Vec = (0..1000).map(|_| 0u64).collect(); - - assert_eq!(storage_fees, reference_fees); - } - + use drive::fee_pools::{ + update_storage_fee_distribution_pool_operation, update_unpaid_epoch_index_operation, + }; + + use super::*; + #[test] - fn test_distribution_overflow() { + fn should_add_operations_to_distribute_distribution_storage_pool_and_refunds() { let platform = setup_platform_with_initial_state_structure(); let transaction = platform.drive.grove.start_transaction(); - - let storage_pool = i64::MAX as u64; - let epoch_index = GENESIS_EPOCH_INDEX; - + + /* + Initial distribution + */ + + let storage_pool = 1000000; + let current_epoch_index = 0; + let mut batch = GroveDbOpBatch::new(); - + + // Store distribution storage fees batch.push(update_storage_fee_distribution_pool_operation(storage_pool)); - - // Apply storage fee distribution pool update + platform .drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - + let mut batch = GroveDbOpBatch::new(); - + let leftovers = platform .add_distribute_storage_fee_to_epochs_operations( - epoch_index, + current_epoch_index, Some(&transaction), &mut batch, ) .expect("should distribute storage fee pool"); - + platform .drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - - // check leftover - assert_eq!(leftovers, 507); - } - - #[test] - fn test_deterministic_distribution() { - let platform = setup_platform_with_initial_state_structure(); - let transaction = platform.drive.grove.start_transaction(); - + + /* + Distribute since epoch 2 with refunds + */ + let storage_pool = 1000000; - let epoch_index = 42; - + let current_epoch_index = 3; + let unpaid_epoch = 1; + let mut batch = GroveDbOpBatch::new(); - + // init additional epochs pools as it will be done in epoch_change - for i in 1000..=1000 + epoch_index { + for i in PERPETUAL_STORAGE_EPOCHS..=PERPETUAL_STORAGE_EPOCHS + current_epoch_index { let epoch = Epoch::new(i); epoch.add_init_empty_operations(&mut batch); } - + + // Store unpaid epoch index + batch.push(update_unpaid_epoch_index_operation(unpaid_epoch)); + + // Store distribution storage fees batch.push(update_storage_fee_distribution_pool_operation(storage_pool)); - - // Apply storage fee distribution pool update + + // Add pending refunds + + let refunds = SignedCreditsPerEpoch::from_iter([ + (0, -10000), + (1, -15000), + (2, -20000), + (3, -25000), + ]); + + add_update_pending_epoch_storage_pool_update_operations(&mut batch, refunds.clone()) + .expect("should update pending updates"); + platform .drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - + let mut batch = GroveDbOpBatch::new(); - + let leftovers = platform .add_distribute_storage_fee_to_epochs_operations( - epoch_index, + current_epoch_index, Some(&transaction), &mut batch, ) .expect("should distribute storage fee pool"); - + platform .drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - + // check leftover assert_eq!(leftovers, 180); - + // collect all the storage fee values of the 1000 epochs pools let storage_fees = get_storage_credits_for_distribution_for_epochs_in_range( &platform.drive, - epoch_index..epoch_index + 1000, - Some(&transaction), - ); - - // compare them with reference table - #[rustfmt::skip] - let reference_fees: [u64; 1000] = [ - 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, - 2500, 2500, 2500, 2500, 2500, 2500, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, - 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2300, 2300, - 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, - 2300, 2300, 2300, 2300, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, - 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2100, 2100, 2100, 2100, - 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, - 2100, 2100, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, - 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 1925, 1925, 1925, 1925, 1925, 1925, - 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, - 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, - 1850, 1850, 1850, 1850, 1850, 1850, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, - 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1700, 1700, - 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, - 1700, 1700, 1700, 1700, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, - 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1550, 1550, 1550, 1550, - 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, - 1550, 1550, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, - 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1425, 1425, 1425, 1425, 1425, 1425, - 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, - 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, - 1375, 1375, 1375, 1375, 1375, 1375, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, - 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1275, 1275, - 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, - 1275, 1275, 1275, 1275, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, - 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1175, 1175, 1175, 1175, - 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, - 1175, 1175, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, - 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1075, 1075, 1075, 1075, 1075, 1075, - 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, - 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, - 1025, 1025, 1025, 1025, 1025, 1025, 975, 975, 975, 975, 975, 975, 975, 975, 975, - 975, 975, 975, 975, 975, 975, 975, 975, 975, 975, 975, 937, 937, 937, 937, 937, - 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 900, - 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, - 900, 900, 900, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, - 862, 862, 862, 862, 862, 862, 862, 825, 825, 825, 825, 825, 825, 825, 825, 825, - 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, 787, 787, 787, 787, 787, - 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 750, - 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, - 750, 750, 750, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, - 712, 712, 712, 712, 712, 712, 712, 675, 675, 675, 675, 675, 675, 675, 675, 675, - 675, 675, 675, 675, 675, 675, 675, 675, 675, 675, 675, 637, 637, 637, 637, 637, - 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 600, - 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, - 600, 600, 600, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, - 562, 562, 562, 562, 562, 562, 562, 525, 525, 525, 525, 525, 525, 525, 525, 525, - 525, 525, 525, 525, 525, 525, 525, 525, 525, 525, 525, 487, 487, 487, 487, 487, - 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 450, - 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, - 450, 450, 450, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, - 412, 412, 412, 412, 412, 412, 412, 375, 375, 375, 375, 375, 375, 375, 375, 375, - 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 337, 337, 337, 337, 337, - 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 300, - 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, - 300, 300, 300, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, - 262, 262, 262, 262, 262, 262, 262, 237, 237, 237, 237, 237, 237, 237, 237, 237, - 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, 212, 212, 212, 212, 212, - 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 187, - 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, - 187, 187, 187, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, - 162, 162, 162, 162, 162, 162, 162, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 112, 112, 112, 112, 112, - 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 87, - 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 62, - 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62 - ]; - - assert_eq!(storage_fees, reference_fees); - - let total_distributed: Credits = storage_fees.iter().sum(); - - assert_eq!(total_distributed + leftovers, storage_pool); - - /* - - Repeat distribution to ensure deterministic results - - */ - - let mut batch = GroveDbOpBatch::new(); - - // refill storage fee pool once more - batch.push(update_storage_fee_distribution_pool_operation(storage_pool)); - - // Apply storage fee distribution pool update - platform - .drive - .grove_apply_batch(batch, false, Some(&transaction)) - .expect("should apply batch"); - - let mut batch = GroveDbOpBatch::new(); - - // distribute fees once more - platform - .add_distribute_storage_fee_to_epochs_operations( - epoch_index, - Some(&transaction), - &mut batch, - ) - .expect("should distribute storage fee pool"); - - platform - .drive - .grove_apply_batch(batch, false, Some(&transaction)) - .expect("should apply batch"); - - // collect all the storage fee values of the 1000 epochs pools again - let storage_fees = get_storage_credits_for_distribution_for_epochs_in_range( - &platform.drive, - epoch_index..epoch_index + 1000, + GENESIS_EPOCH_INDEX..current_epoch_index + PERPETUAL_STORAGE_EPOCHS, Some(&transaction), ); - - // assert that all the values doubled meaning that distribution is reproducible - assert_eq!( - storage_fees, - reference_fees - .iter() - .map(|val| val * 2) - .collect::>() - ); - } - } - - mod update_storage_fee_distribution_pool { - use crate::common::helpers::setup::{ - setup_platform, setup_platform_with_initial_state_structure, - }; - use drive::drive::batch::GroveDbOpBatch; - use drive::error::Error as DriveError; - use drive::fee_pools::update_storage_fee_distribution_pool_operation; - use drive::grovedb; - #[test] - fn test_error_if_pool_is_not_initiated() { - let platform = setup_platform(); - let transaction = platform.drive.grove.start_transaction(); + // Assert total distributed fees - let storage_fee = 42; - - let mut batch = GroveDbOpBatch::new(); - - batch.push(update_storage_fee_distribution_pool_operation(storage_fee)); - - match platform - .drive - .grove_apply_batch(batch, false, Some(&transaction)) - { - Ok(_) => assert!( - false, - "should not be able to update genesis time on uninit fee pools" - ), - Err(e) => match e { - DriveError::GroveDB(grovedb::Error::InvalidPath(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } - } + let total_storage_pool_distribution = ((storage_pool - leftovers) * 2) as SignedCredits; - #[test] - fn test_update_and_get_value() { - let platform = setup_platform_with_initial_state_structure(); - let transaction = platform.drive.grove.start_transaction(); + let total_refunds: SignedCredits = refunds + .into_iter() + .map(|(epoch_index, credits)| { + let mut credits_per_epochs = SignedCreditsPerEpoch::default(); - let storage_fee = 42; + let leftovers = distribute_storage_fee_to_epochs_collection( + &mut credits_per_epochs, + credits, + epoch_index, + None, + ) + .expect("should distribute refunds"); - let mut batch = GroveDbOpBatch::new(); + let already_paid_epochs = unpaid_epoch as i64 + 1 - epoch_index as i64; - batch.push(update_storage_fee_distribution_pool_operation(storage_fee)); + let already_paid_credits = if already_paid_epochs > 0 { + credits_per_epochs + .into_iter() + .take(already_paid_epochs as usize) + .map(|(_, credits)| credits) + .sum() + } else { + 0 + }; - platform - .drive - .grove_apply_batch(batch, false, Some(&transaction)) - .expect("should apply batch"); + credits - leftovers - already_paid_credits + }) + .sum(); - let stored_storage_fee = platform - .drive - .get_aggregate_storage_fees_from_distribution_pool(Some(&transaction)) - .expect("should get storage fee pool"); + let total_distributed = storage_fees + .into_iter() + .sum::() + .to_signed() + .expect("shouldn't overflow"); - assert_eq!(storage_fee, stored_storage_fee); + assert_eq!( + total_distributed, + total_storage_pool_distribution + total_refunds + ); } } } diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index d54641adb0c..d7097193161 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -81,7 +81,8 @@ pub fn aggregate_storage_fees_distribution_pool_vec_path() -> Vec> { impl Drive { /// Adds GroveDB operations to update epoch storage fee pools with specified map of credits to epochs - pub fn add_update_epoch_storage_fee_pools_operations( + /// This method optimized to update sequence of epoch pools without gaps + pub fn add_update_epoch_storage_fee_pools_sequence_operations( &self, batch: &mut GroveDbOpBatch, credits_per_epochs: SignedCreditsPerEpoch, @@ -103,6 +104,12 @@ impl Drive { let max_epoch_index = max_epoch_index_key.to_owned() as u16; let max_encoded_epoch_index = paths::encode_epoch_index_key(max_epoch_index)?.to_vec(); + if max_epoch_index - min_epoch_index + 1 != credits_per_epochs.len() as u16 { + return Err(Error::Drive(DriveError::CorruptedCodeExecution( + "gaps in credits per epoch are not supported", + ))); + } + let mut storage_fee_pool_query = Query::new(); storage_fee_pool_query.insert_key(KEY_POOL_STORAGE_FEES.to_vec()); @@ -121,20 +128,15 @@ impl Drive { .unwrap() .map_err(Error::GroveDB)?; - if storage_fee_pools.len() != credits_per_epochs.len() { - return Err(Error::Drive(DriveError::CorruptedCodeExecution( - "number of fetched epoch storage fee pools must be equal to requested for update", - ))); - } - // Sort epochs to reverse order to pop existing values - for (epoch_index, credits) in credits_per_epochs.into_iter().sorted_by_key(|x| !x.0) { - let encoded_existing_storage_fee = - storage_fee_pools - .pop() - .ok_or(Error::Drive(DriveError::CorruptedCodeExecution( - "value must be present", - )))?; + for (i, (epoch_index, credits)) in credits_per_epochs + .into_iter() + .sorted_by_key(|x| x.0) + .enumerate() + { + let encoded_existing_storage_fee = storage_fee_pools + .get(i) + .map_or(0u64.to_vec_bytes(), |c| c.to_owned()); let existing_storage_fee = SignedCredits::from_vec_bytes(encoded_existing_storage_fee)?; @@ -177,7 +179,7 @@ mod tests { let mut batch = GroveDbOpBatch::new(); drive - .add_update_epoch_storage_fee_pools_operations( + .add_update_epoch_storage_fee_pools_sequence_operations( &mut batch, credits_per_epoch, Some(&transaction), @@ -223,7 +225,7 @@ mod tests { let mut batch = GroveDbOpBatch::new(); drive - .add_update_epoch_storage_fee_pools_operations( + .add_update_epoch_storage_fee_pools_sequence_operations( &mut batch, credits_to_epochs, Some(&transaction), @@ -232,7 +234,7 @@ mod tests { assert_eq!(batch.len(), TO_EPOCH_INDEX as usize); - for (i, operation) in batch.into_iter().rev().enumerate() { + for (i, operation) in batch.into_iter().enumerate() { assert_eq!(operation.key.get_key(), KEY_POOL_STORAGE_FEES); assert_eq!( diff --git a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs index 847bf89f628..5145d42db31 100644 --- a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs +++ b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs @@ -87,6 +87,10 @@ impl Drive { mut credits_per_epoch: SignedCreditsPerEpoch, transaction: TransactionArg, ) -> Result { + if credits_per_epoch.is_empty() { + return Ok(credits_per_epoch); + } + let mut query = Query::new(); for epoch_index in credits_per_epoch.keys() { diff --git a/packages/rs-drive/src/drive/fee_pools/unpaid_epoch.rs b/packages/rs-drive/src/drive/fee_pools/unpaid_epoch.rs index 9202d7f4499..74f292c4ec9 100644 --- a/packages/rs-drive/src/drive/fee_pools/unpaid_epoch.rs +++ b/packages/rs-drive/src/drive/fee_pools/unpaid_epoch.rs @@ -34,12 +34,13 @@ use crate::drive::fee_pools::pools_path; use crate::drive::Drive; use crate::error::fee::FeeError; use crate::error::Error; +use crate::fee::epoch::EpochIndex; use crate::fee_pools::epochs_root_tree_key_constants::KEY_UNPAID_EPOCH_INDEX; use grovedb::{Element, TransactionArg}; impl Drive { /// Returns the index of the unpaid Epoch. - pub fn get_unpaid_epoch_index(&self, transaction: TransactionArg) -> Result { + pub fn get_unpaid_epoch_index(&self, transaction: TransactionArg) -> Result { let element = self .grove .get(pools_path(), KEY_UNPAID_EPOCH_INDEX, transaction) diff --git a/packages/rs-drive/src/fee/epoch/distribution.rs b/packages/rs-drive/src/fee/epoch/distribution.rs index 1361ba51ee3..941c0f15dc7 100644 --- a/packages/rs-drive/src/fee/epoch/distribution.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -60,13 +60,59 @@ pub const FEE_DISTRIBUTION_TABLE: [Decimal; PERPETUAL_STORAGE_YEARS as usize] = dec!(0.00325), dec!(0.00275), dec!(0.00225), dec!(0.00175), dec!(0.00125), ]; -/// Distributes storage fees to epochs and returns leftovers -pub fn distribute_storage_fee_to_epochs( +/// Distributes storage fees to epochs into `SignedCreditsPerEpoch` and returns leftovers +/// It skips epochs up to specified `skip_up_to_epoch_index` +pub fn distribute_storage_fee_to_epochs_collection( + credits_per_epochs: &mut SignedCreditsPerEpoch, + storage_fee: SignedCredits, + start_epoch_index: EpochIndex, + skip_up_to_epoch_index: Option, +) -> Result { + distribution_storage_fee_to_epochs_map( + storage_fee, + start_epoch_index, + |epoch_index, epoch_fee_share| { + if let Some(skip_epoch_index) = skip_up_to_epoch_index { + if epoch_index < skip_epoch_index { + return Ok(()); + } + } + + let epoch_credits: SignedCredits = credits_per_epochs + .get(&epoch_index) + .map_or(0, |i| i.to_owned()); + + let result_storage_fee: SignedCredits = epoch_credits + .checked_add(epoch_fee_share) + .ok_or_else(|| get_overflow_error("storage fees are not fitting in a u64"))?; + + credits_per_epochs.insert(epoch_index, result_storage_fee); + + Ok(()) + }, + ) +} + +/// Calculates leftovers from distribution storage fees to epochs +pub fn calculate_distribution_storage_fee_to_epochs_leftovers( storage_fee: SignedCredits, start_epoch_index: EpochIndex, - from_epoch_index: EpochIndex, - credits_per_epochs: &mut SignedCreditsPerEpoch, ) -> Result { + // To do not mess `distribution_storage_fee_to_epochs_map` arguments better to pass dummy + // function. Rust optimizer reduce it and won't call anything. + distribution_storage_fee_to_epochs_map(storage_fee, start_epoch_index, |_, _| Ok(())) +} + +/// Distributes storage fees to epochs and call function for each epoch. +/// Returns leftovers +fn distribution_storage_fee_to_epochs_map( + storage_fee: SignedCredits, + start_epoch_index: EpochIndex, + mut f: F, +) -> Result +where + F: FnMut(EpochIndex, SignedCredits) -> Result<(), Error>, +{ if storage_fee == 0 { return Ok(0); } @@ -92,19 +138,7 @@ pub fn distribute_storage_fee_to_epochs( let year_start_epoch_index = start_epoch_index + EPOCHS_PER_YEAR * year; for epoch_index in year_start_epoch_index..year_start_epoch_index + EPOCHS_PER_YEAR { - if epoch_index < from_epoch_index { - continue; - } - - let epoch_credits: SignedCredits = credits_per_epochs - .get(&epoch_index) - .map_or(0, |i| i.to_owned()); - - let result_storage_fee: SignedCredits = epoch_credits - .checked_add(epoch_fee_share) - .ok_or_else(|| get_overflow_error("storage fees are not fitting in a u64"))?; - - credits_per_epochs.insert(epoch_index, result_storage_fee); + f(epoch_index, epoch_fee_share)?; distribution_leftover_credits = distribution_leftover_credits .checked_sub(epoch_fee_share) @@ -134,32 +168,30 @@ mod tests { #[test] fn should_distribute_value() { - let mut buffer = dec!(0.0); let value = Decimal::from(i64::MAX); - for i in FEE_DISTRIBUTION_TABLE { - let share = value - * FEE_DISTRIBUTION_TABLE[i.to_usize().expect("should be convertable to usize")]; - - buffer += share; - } + let calculated_value: Decimal = FEE_DISTRIBUTION_TABLE + .into_iter() + .map(|ratio| value * ratio) + .sum(); - assert_eq!(buffer, value); + assert_eq!(calculated_value, value); } } - mod distribute_storage_fee_to_epochs { + mod distribute_storage_fee_to_epochs_collection { use super::*; + use crate::fee::epoch::PERPETUAL_STORAGE_EPOCHS; #[test] fn should_distribute_nothing_if_storage_fee_are_zero() { let mut credits_per_epochs = SignedCreditsPerEpoch::default(); - let leftovers = distribute_storage_fee_to_epochs( + let leftovers = distribute_storage_fee_to_epochs_collection( + &mut credits_per_epochs, 0, GENESIS_EPOCH_INDEX, - GENESIS_EPOCH_INDEX, - &mut credits_per_epochs, + None, ) .expect("should distribute storage fee"); @@ -173,11 +205,11 @@ mod tests { let mut credits_per_epochs = SignedCreditsPerEpoch::default(); - let leftovers = distribute_storage_fee_to_epochs( + let leftovers = distribute_storage_fee_to_epochs_collection( + &mut credits_per_epochs, storage_fee, GENESIS_EPOCH_INDEX, - GENESIS_EPOCH_INDEX, - &mut credits_per_epochs, + None, ) .expect("should distribute storage fee"); @@ -192,11 +224,11 @@ mod tests { let mut credits_per_epochs = SignedCreditsPerEpoch::default(); - let leftovers = distribute_storage_fee_to_epochs( + let leftovers = distribute_storage_fee_to_epochs_collection( + &mut credits_per_epochs, storage_fee, current_epoch_index, - current_epoch_index, - &mut credits_per_epochs, + None, ) .expect("should distribute storage fee"); @@ -205,7 +237,7 @@ mod tests { // compare them with reference table #[rustfmt::skip] - let reference_fees: [SignedCredits; 1000] = [ + let reference_fees: [SignedCredits; PERPETUAL_STORAGE_EPOCHS as usize] = [ 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2300, 2300, @@ -292,11 +324,11 @@ mod tests { */ - let leftovers = distribute_storage_fee_to_epochs( + let leftovers = distribute_storage_fee_to_epochs_collection( + &mut credits_per_epochs, storage_fee, current_epoch_index, - current_epoch_index, - &mut credits_per_epochs, + None, ) .expect("should distribute storage fee"); @@ -315,8 +347,133 @@ mod tests { } #[test] - fn should_distribute_from_specific_epoch() { - todo!(); + fn should_add_to_collection_from_specific_epoch() { + let storage_fee: SignedCredits = 1000000; + let start_epoch_index: EpochIndex = 0; + + const SKIP_UP_TO_EPOCH_INDEX: EpochIndex = 42; + + let mut credits_per_epochs = SignedCreditsPerEpoch::default(); + + let leftovers = distribute_storage_fee_to_epochs_collection( + &mut credits_per_epochs, + storage_fee, + start_epoch_index, + Some(SKIP_UP_TO_EPOCH_INDEX), + ) + .expect("should distribute storage fee"); + + // check leftover + assert_eq!(leftovers, 180); + + // compare them with reference table + #[rustfmt::skip] + let reference_fees: [SignedCredits; (PERPETUAL_STORAGE_EPOCHS - SKIP_UP_TO_EPOCH_INDEX) as usize] = [ + 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, + 2300, 2300, 2300, 2300, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, + 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2100, 2100, 2100, 2100, + 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, + 2100, 2100, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, + 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 1925, 1925, 1925, 1925, 1925, 1925, + 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, + 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, + 1850, 1850, 1850, 1850, 1850, 1850, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, + 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1700, 1700, + 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, + 1700, 1700, 1700, 1700, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, + 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1550, 1550, 1550, 1550, + 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, + 1550, 1550, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, + 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1425, 1425, 1425, 1425, 1425, 1425, + 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, + 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, + 1375, 1375, 1375, 1375, 1375, 1375, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, + 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1275, 1275, + 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, + 1275, 1275, 1275, 1275, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, + 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1175, 1175, 1175, 1175, + 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, + 1175, 1175, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, + 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1075, 1075, 1075, 1075, 1075, 1075, + 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, + 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, + 1025, 1025, 1025, 1025, 1025, 1025, 975, 975, 975, 975, 975, 975, 975, 975, 975, + 975, 975, 975, 975, 975, 975, 975, 975, 975, 975, 975, 937, 937, 937, 937, 937, + 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 900, + 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, + 900, 900, 900, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, + 862, 862, 862, 862, 862, 862, 862, 825, 825, 825, 825, 825, 825, 825, 825, 825, + 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, 787, 787, 787, 787, 787, + 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 750, + 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, + 750, 750, 750, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, + 712, 712, 712, 712, 712, 712, 712, 675, 675, 675, 675, 675, 675, 675, 675, 675, + 675, 675, 675, 675, 675, 675, 675, 675, 675, 675, 675, 637, 637, 637, 637, 637, + 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 600, + 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, + 600, 600, 600, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, + 562, 562, 562, 562, 562, 562, 562, 525, 525, 525, 525, 525, 525, 525, 525, 525, + 525, 525, 525, 525, 525, 525, 525, 525, 525, 525, 525, 487, 487, 487, 487, 487, + 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 450, + 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, + 450, 450, 450, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, + 412, 412, 412, 412, 412, 412, 412, 375, 375, 375, 375, 375, 375, 375, 375, 375, + 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 337, 337, 337, 337, 337, + 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 300, + 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, + 300, 300, 300, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, + 262, 262, 262, 262, 262, 262, 262, 237, 237, 237, 237, 237, 237, 237, 237, 237, + 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, 212, 212, 212, 212, 212, + 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 187, + 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + 187, 187, 187, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, + 162, 162, 162, 162, 162, 162, 162, 137, 137, 137, 137, 137, 137, 137, 137, 137, + 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 112, 112, 112, 112, 112, + 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 87, + 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 62, + 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62 + ]; + + let skipped_reference_fees: [SignedCredits; SKIP_UP_TO_EPOCH_INDEX as usize] = [ + 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, + 2500, 2500, 2500, 2500, 2500, 2500, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, + 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2300, 2300, + ]; + + assert_eq!( + credits_per_epochs + .clone() + .into_values() + .collect::>(), + reference_fees + ); + + let total_distributed: SignedCredits = credits_per_epochs.values().sum(); + + assert_eq!( + total_distributed + + leftovers + + skipped_reference_fees.into_iter().sum::(), + storage_fee + ); + } + } + + mod calculate_distribution_storage_fee_to_epochs_leftovers { + use super::*; + + #[test] + fn should_calculate_leftovers() { + let storage_fee = 10000; + + let leftovers = calculate_distribution_storage_fee_to_epochs_leftovers( + storage_fee, + GENESIS_EPOCH_INDEX, + ) + .expect("should distribute storage fee"); + + // check leftover + assert_eq!(leftovers, 400); } } } diff --git a/packages/rs-drive/src/fee/result/refunds.rs b/packages/rs-drive/src/fee/result/refunds.rs index 7850463f58d..b57b8901b10 100644 --- a/packages/rs-drive/src/fee/result/refunds.rs +++ b/packages/rs-drive/src/fee/result/refunds.rs @@ -34,7 +34,9 @@ use crate::error::fee::FeeError; use crate::error::Error; +use crate::fee::credits::SignedCredits; use crate::fee::default_costs::STORAGE_DISK_USAGE_CREDIT_PER_BYTE; +use crate::fee::epoch::distribution::calculate_distribution_storage_fee_to_epochs_leftovers; use crate::fee::epoch::SignedCreditsPerEpoch; use crate::fee::get_overflow_error; use bincode::Options; @@ -65,12 +67,13 @@ impl FeeRefunds { // TODO We should use multipliers - (bytes as i64) + let credits: SignedCredits = (bytes as i64) .checked_mul(STORAGE_DISK_USAGE_CREDIT_PER_BYTE as i64) .ok_or_else(|| { get_overflow_error("storage written bytes cost overflow") - }) - .map(|credits| (epoch_index, credits)) + })?; + + Ok((epoch_index, -credits)) }) .collect::>() .map(|credits_per_epochs| (identifier, credits_per_epochs)) diff --git a/packages/rs-drive/src/fee_pools/epochs/mod.rs b/packages/rs-drive/src/fee_pools/epochs/mod.rs index 8f4db031149..7ea6834b199 100644 --- a/packages/rs-drive/src/fee_pools/epochs/mod.rs +++ b/packages/rs-drive/src/fee_pools/epochs/mod.rs @@ -3,6 +3,7 @@ pub mod epoch_key_constants; pub mod operations_factory; pub mod paths; +use crate::fee::epoch::EpochIndex; use serde::{Deserialize, Serialize}; // TODO: I would call it EpochTree because it represent pool, @@ -13,14 +14,14 @@ use serde::{Deserialize, Serialize}; #[serde(rename_all = "camelCase")] pub struct Epoch { /// Epoch index - pub index: u16, + pub index: EpochIndex, /// Epoch key pub(crate) key: [u8; 2], } impl Epoch { /// Create new epoch - pub fn new(index: u16) -> Self { + pub fn new(index: EpochIndex) -> Self { let key = paths::encode_epoch_index_key(index).expect("epoch index is too high"); Self { index, key } diff --git a/packages/rs-drive/src/fee_pools/mod.rs b/packages/rs-drive/src/fee_pools/mod.rs index ff72b8ffb3d..184391a3376 100644 --- a/packages/rs-drive/src/fee_pools/mod.rs +++ b/packages/rs-drive/src/fee_pools/mod.rs @@ -90,9 +90,10 @@ pub fn update_unpaid_epoch_index_operation(epoch_index: EpochIndex) -> GroveDbOp mod tests { use super::*; use crate::common::helpers::setup::setup_drive_with_initial_state_structure; + use crate::error::Error; mod add_create_fee_pool_trees_operations { - use crate::error::Error; + use super::*; #[test] fn test_values_are_set() { @@ -131,4 +132,30 @@ mod tests { } } } + + mod update_storage_fee_distribution_pool_operation { + use super::*; + + #[test] + fn test_update_and_get_value() { + let drive = setup_drive_with_initial_state_structure(); + let transaction = drive.grove.start_transaction(); + + let storage_fee = 42; + + let mut batch = GroveDbOpBatch::new(); + + batch.push(update_storage_fee_distribution_pool_operation(storage_fee)); + + drive + .grove_apply_batch(batch, false, Some(&transaction)) + .expect("should apply batch"); + + let stored_storage_fee = drive + .get_aggregate_storage_fees_from_distribution_pool(Some(&transaction)) + .expect("should get storage fee pool"); + + assert_eq!(storage_fee, stored_storage_fee); + } + } } From 0208c151fba878d6f5317753da4461fcda753223 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Mon, 19 Dec 2022 23:58:26 +0800 Subject: [PATCH 19/54] tests: fix tests --- ...ynchronizeMasternodeIdentitiesFactory.spec.js | 16 ++++++++-------- packages/rs-drive-abci/src/abci/handlers.rs | 13 +++++++++++-- .../execution/fee_pools/process_block_fees.rs | 14 ++++++++++++-- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/packages/js-drive/test/integration/identity/masternode/synchronizeMasternodeIdentitiesFactory.spec.js b/packages/js-drive/test/integration/identity/masternode/synchronizeMasternodeIdentitiesFactory.spec.js index fadf5e51574..0236f177c64 100644 --- a/packages/js-drive/test/integration/identity/masternode/synchronizeMasternodeIdentitiesFactory.spec.js +++ b/packages/js-drive/test/integration/identity/masternode/synchronizeMasternodeIdentitiesFactory.spec.js @@ -367,7 +367,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { beforeEach(async function beforeEach() { coreHeight = 3; - firstSyncAppHash = '20a1b452ad2b98fb8c6250d501c636e1b6a3bf95af1dc952fc7cf21904f226a7'; + firstSyncAppHash = '2b791519644290d49e47024885d37f66da9a278830ef1872e170191acae6d4b9'; blockInfo = new BlockInfo(10, 0, 1668702100799); container = await createTestDIContainer(); @@ -695,7 +695,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { // Nothing happened - await expectDeterministicAppHash('34cfbd49f7d8b3b18791c51966821fd5343efb67d3bd41805585ce21dad6fe91'); + await expectDeterministicAppHash('b3470ce12f4f0a59cab7a24de9607b6cbfb0419ff9597f388c61e631372c9a25'); // Core RPC should be called @@ -750,7 +750,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { expect(result.updatedEntities).to.have.lengthOf(0); expect(result.removedEntities).to.have.lengthOf(0); - await expectDeterministicAppHash('c5bfc7b21f420b4dac0201c6a941aa0801ff0aa55d0f0ff70ca5d369ec31bd65'); + await expectDeterministicAppHash('ddd75c33b08e43bfe6f437befdef22b30f1e806fa5d97b39138c1a110fdd8f43'); // New masternode identity should be created @@ -834,7 +834,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { expect(result.updatedEntities).to.have.lengthOf(0); expect(result.removedEntities).to.have.lengthOf(1); - await expectDeterministicAppHash('25bd044189691f61a872ab485c70554fa86136d8aa34b3d0b30cf5e75f15670a'); + await expectDeterministicAppHash('57b1c71c3bd783030a7e4d2f17431ff1c632a6cf1c25bf2724b1505ee30dcc3a'); // Masternode identity should stay @@ -902,7 +902,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { expect(result.updatedEntities).to.have.lengthOf(0); expect(result.removedEntities).to.have.lengthOf(1); - await expectDeterministicAppHash('25bd044189691f61a872ab485c70554fa86136d8aa34b3d0b30cf5e75f15670a'); + await expectDeterministicAppHash('57b1c71c3bd783030a7e4d2f17431ff1c632a6cf1c25bf2724b1505ee30dcc3a'); const invalidMasternodeIdentifier = Identifier.from( Buffer.from(invalidSmlEntry.proRegTxHash, 'hex'), @@ -952,7 +952,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { expect(result.updatedEntities).to.have.lengthOf(3); expect(result.removedEntities).to.have.lengthOf(0); - await expectDeterministicAppHash('955896708aa0372060544d21311e2fc8f21205fe3c52e53e6777787f443f08bc'); + await expectDeterministicAppHash('d32960fd6e1751d2c3b74574e6b5e23e2f6cac514e3d12a6011d920e0924cbed'); // Masternode identity should stay @@ -1027,7 +1027,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { await synchronizeMasternodeIdentities(coreHeight + 1, blockInfo); - await expectDeterministicAppHash('1910be205225620460d2205149138ef3c0ef8fcef883e6894280a37edf0dee65'); + await expectDeterministicAppHash('e6fec7d1664736157634e73f3dde7818394969323296dbfa880792c2ce472dd9'); // Masternode identity should contain new public key @@ -1110,7 +1110,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { await synchronizeMasternodeIdentities(coreHeight, blockInfo); - await expectDeterministicAppHash('5b0ff77efd9d3a4894263c284ebb9be416bfb15ba64b59e584405cfa0fd5ab24'); + await expectDeterministicAppHash('934bb8c9dd4f33564ba8392942b2c3a7e5408dabc9d1c5405e8c28ac936ebcc3'); const votingIdentifier = createVotingIdentifier(smlFixture[0]); diff --git a/packages/rs-drive-abci/src/abci/handlers.rs b/packages/rs-drive-abci/src/abci/handlers.rs index 7a537cf2b89..46036148586 100644 --- a/packages/rs-drive-abci/src/abci/handlers.rs +++ b/packages/rs-drive-abci/src/abci/handlers.rs @@ -187,6 +187,7 @@ mod tests { use chrono::{Duration, Utc}; use drive::common::helpers::identities::create_test_masternode_identities; use drive::drive::batch::GroveDbOpBatch; + use drive::fee::epoch::SignedCreditsPerEpoch; use rust_decimal::prelude::ToPrimitive; use std::ops::Div; @@ -359,7 +360,11 @@ mod tests { } let block_end_request = BlockEndRequest { - fees: BlockFees::from_fees(storage_fees_per_block, 1600), + fees: BlockFees { + storage_fee: storage_fees_per_block, + processing_fee: 1600, + fee_refunds: SignedCreditsPerEpoch::from_iter([(0, -100)]), + }, }; let block_end_response = platform @@ -511,7 +516,11 @@ mod tests { ); let block_end_request = BlockEndRequest { - fees: BlockFees::from_fees(storage_fees_per_block, 1600), + fees: BlockFees { + storage_fee: storage_fees_per_block, + processing_fee: 1600, + fee_refunds: SignedCreditsPerEpoch::from_iter([(0, -100)]), + }, }; let block_end_response = platform diff --git a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs index bb8c5ab0f04..00c31f74b3d 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs @@ -236,6 +236,7 @@ mod tests { mod helpers { use super::*; + use drive::fee::epoch::SignedCreditsPerEpoch; /// Process and validate an epoch change pub fn process_and_validate_epoch_change( @@ -289,7 +290,11 @@ mod tests { EpochInfo::from_genesis_time_and_block_info(genesis_time_ms, &block_info) .expect("should calculate epoch info"); - let block_fees = BlockFees::default(); + let block_fees = BlockFees { + storage_fee: 1000000000, + processing_fee: 10000, + fee_refunds: SignedCreditsPerEpoch::from_iter([(0, -10000)]), + }; let mut batch = GroveDbOpBatch::new(); @@ -422,6 +427,7 @@ mod tests { mod helpers { use super::*; + use drive::fee::epoch::SignedCreditsPerEpoch; /// Process and validate block fees pub fn process_and_validate_block_fees( @@ -449,7 +455,11 @@ mod tests { EpochInfo::from_genesis_time_and_block_info(genesis_time_ms, &block_info) .expect("should calculate epoch info"); - let block_fees = BlockFees::from_fees(1000, 10000); + let block_fees = BlockFees { + storage_fee: 1000, + processing_fee: 10000, + fee_refunds: SignedCreditsPerEpoch::from_iter([(epoch_index, -100)]), + }; let distribute_storage_pool_result = platform .process_block_fees(&block_info, &epoch_info, block_fees.clone(), transaction) From 1e1c1b5f5079ca828a7de4a14d05a2c3ede95a17 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Tue, 20 Dec 2022 00:44:52 +0800 Subject: [PATCH 20/54] chore: misc changes --- Cargo.lock | 1 - .../stateTransition/AbstractStateTransition.js | 6 ++---- .../fee/calculateStateTransitionFee.js | 14 +------------- .../test/integration/fee/feesPrediction.spec.js | 1 + .../js-drive/test/integration/fee/pools.spec.js | 2 +- .../prepareProposalHandlerFactory.spec.js | 6 +++--- .../handlers/proposal/deliverTxFactory.spec.js | 8 ++++---- .../handlers/proposal/endBlockFactory.spec.js | 4 ++-- .../proposal/processProposalFactory.spec.js | 8 ++++---- packages/rs-drive-abci/Cargo.toml | 1 - packages/rs-drive-abci/src/abci/handlers.rs | 2 -- packages/rs-drive-nodejs/src/lib.rs | 12 ++++++------ packages/rs-drive-nodejs/test/Drive.spec.js | 4 ++-- .../rs-drive/src/drive/batch/grovedb_op_batch.rs | 2 -- packages/rs-drive/src/drive/fee_pools/mod.rs | 15 +++++++-------- packages/rs-drive/src/fee/credits.rs | 6 +++--- packages/rs-drive/src/fee/mod.rs | 1 - packages/rs-drive/src/fee/result/refunds.rs | 1 - 18 files changed, 36 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce090108135..d826fa3c320 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -942,7 +942,6 @@ dependencies = [ "dashcore 0.29.1 (git+https://github.com/dashpay/rust-dashcore?branch=master)", "drive", "hex", - "itertools", "rand 0.8.5", "rust_decimal", "rust_decimal_macros", diff --git a/packages/js-dpp/lib/stateTransition/AbstractStateTransition.js b/packages/js-dpp/lib/stateTransition/AbstractStateTransition.js index f7ad8208eb9..fe552a8dcfe 100644 --- a/packages/js-dpp/lib/stateTransition/AbstractStateTransition.js +++ b/packages/js-dpp/lib/stateTransition/AbstractStateTransition.js @@ -295,12 +295,10 @@ class AbstractStateTransition { /** * Calculate ST fee in credits * - * @param {Object} options - * @param {boolean} [options.useCache=false] * @return {number} */ - calculateFee(options = {}) { - return calculateStateTransitionFee(this, options); + calculateFee() { + return calculateStateTransitionFee(this); } /** diff --git a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js index a63581f3b2a..2ae07eee671 100644 --- a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js +++ b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js @@ -7,23 +7,11 @@ const calculateOperationFees = require('./calculateOperationFees'); /** * @typedef {calculateStateTransitionFee} * @param {AbstractStateTransition} stateTransition - * @param {Object} options - * @param {boolean} [options.useCache=false] * @return {number} */ -function calculateStateTransitionFee(stateTransition, options = {}) { +function calculateStateTransitionFee(stateTransition) { const executionContext = stateTransition.getExecutionContext(); - if (options.useCache) { - const calculatedFeeDetails = executionContext.getLastCalculatedFeeDetails(); - - if (!calculatedFeeDetails) { - throw new Error('State Transition Execution context doesn\'t contain cached fee calculation'); - } - - return calculatedFeeDetails.total; - } - const calculatedFees = calculateOperationFees(executionContext.getOperations()); const { diff --git a/packages/js-drive/test/integration/fee/feesPrediction.spec.js b/packages/js-drive/test/integration/fee/feesPrediction.spec.js index 2c5505b27ec..6e455f70f19 100644 --- a/packages/js-drive/test/integration/fee/feesPrediction.spec.js +++ b/packages/js-drive/test/integration/fee/feesPrediction.spec.js @@ -17,6 +17,7 @@ const BlsSignatures = require('@dashevo/dpp/lib/bls/bls'); const createTestDIContainer = require('../../../lib/test/createTestDIContainer'); const createDataContractDocuments = require('../../../lib/test/fixtures/createDataContractDocuments'); const BlockInfo = require('../../../lib/blockExecution/BlockInfo'); +const calculateStateTransitionFee = require("@dashevo/dpp/lib/stateTransition/fee/calculateStateTransitionFee"); /** * @param {DashPlatformProtocol} dpp diff --git a/packages/js-drive/test/integration/fee/pools.spec.js b/packages/js-drive/test/integration/fee/pools.spec.js index 7d7ce9ab004..8d82a5a7e3e 100644 --- a/packages/js-drive/test/integration/fee/pools.spec.js +++ b/packages/js-drive/test/integration/fee/pools.spec.js @@ -107,7 +107,7 @@ describe('Fee Pools', () => { processingFee: 1000, storageFee: 10000 + 15, feeRefunds: { - 1: 15, + 1: -15, }, feeRefundsSum: 15, }, diff --git a/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js index c52e03cb2b0..671781ced40 100644 --- a/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js @@ -78,7 +78,7 @@ describe('prepareProposalHandlerFactory', () => { processingFee: 10, storageFee: 100, feeRefunds: { - 1: 15, + 1: -15, }, feeRefundsSum: 15, }, @@ -183,9 +183,9 @@ describe('prepareProposalHandlerFactory', () => { processingFee: 10 * 3, storageFee: 100 * 3, feeRefunds: { - 1: 15 * 3, + 1: -15 * 3, }, - feeRefundsSum: 15 * 3, + feeRefundsSum: -15 * 3, }, coreChainLockedHeight: request.coreChainLockedHeight, }, diff --git a/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js index a925a9025d7..6fc6baeaa69 100644 --- a/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js @@ -124,9 +124,9 @@ describe('deliverTxFactory', () => { processingFee: 10, storageFee: 100, feeRefunds: { - 1: 15, + 1: -15, }, - feeRefundsSum: 15, + feeRefundsSum: -15, }, }); @@ -165,9 +165,9 @@ describe('deliverTxFactory', () => { processingFee: 10, storageFee: 100, feeRefunds: { - 1: 15, + 1: -15, }, - feeRefundsSum: 15, + feeRefundsSum: -15, }, }); diff --git a/packages/js-drive/test/unit/abci/handlers/proposal/endBlockFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/proposal/endBlockFactory.spec.js index cbb0ca7a2cc..30360947c93 100644 --- a/packages/js-drive/test/unit/abci/handlers/proposal/endBlockFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/proposal/endBlockFactory.spec.js @@ -36,9 +36,9 @@ describe('endBlockFactory', () => { processingFee: 10, storageFee: 100, feeRefunds: { - 1: 15, + 1: -15, }, - feeRefundsSum: 15, + feeRefundsSum: -15, }; executionTimerMock = { diff --git a/packages/js-drive/test/unit/abci/handlers/proposal/processProposalFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/proposal/processProposalFactory.spec.js index cda64a71068..b45d1d7ff0c 100644 --- a/packages/js-drive/test/unit/abci/handlers/proposal/processProposalFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/proposal/processProposalFactory.spec.js @@ -70,9 +70,9 @@ describe('processProposalFactory', () => { processingFee: 10, storageFee: 100, feeRefunds: { - 1: 15, + 1: -15, }, - feeRefundsSum: 15, + feeRefundsSum: -15, }, }); @@ -155,9 +155,9 @@ describe('processProposalFactory', () => { processingFee: 10 * 3, storageFee: 100 * 3, feeRefunds: { - 1: 15 * 3, + 1: -15 * 3, }, - feeRefundsSum: 15 * 3, + feeRefundsSum: -15 * 3, }, coreChainLockedHeight: request.coreChainLockedHeight, }, diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml index 0e2ac9a22cb..b27cb8e974a 100644 --- a/packages/rs-drive-abci/Cargo.toml +++ b/packages/rs-drive-abci/Cargo.toml @@ -21,5 +21,4 @@ hex = "0.4.3" dashcore = { git="https://github.com/dashpay/rust-dashcore", features=["no-std", "secp-recovery", "rand", "signer"], default-features = false, branch="master" } rust_decimal = "1.2.5" rust_decimal_macros = "1.25.0" -itertools = { version = "0.10.5" } diff --git a/packages/rs-drive-abci/src/abci/handlers.rs b/packages/rs-drive-abci/src/abci/handlers.rs index 46036148586..940bac27abd 100644 --- a/packages/rs-drive-abci/src/abci/handlers.rs +++ b/packages/rs-drive-abci/src/abci/handlers.rs @@ -173,8 +173,6 @@ impl TenderdashAbci for Platform { drive_cache.cached_contracts.clear_block_cache(); - // Super slow operation of preparing chunks - Ok(AfterFinalizeBlockResponse {}) } } diff --git a/packages/rs-drive-nodejs/src/lib.rs b/packages/rs-drive-nodejs/src/lib.rs index a1cc5613d4f..430a7b61daf 100644 --- a/packages/rs-drive-nodejs/src/lib.rs +++ b/packages/rs-drive-nodejs/src/lib.rs @@ -503,7 +503,7 @@ impl PlatformWrapper { let callback_arguments: Vec> = match result { Ok(fee_result) => { let js_fee_result = - task_context.boxed(FeeResultWrapper::new(fee_result.clone())); + task_context.boxed(FeeResultWrapper::new(fee_result)); // First parameter of JS callbacks is error, which is null in this case vec![task_context.null().upcast(), js_fee_result.upcast()] @@ -573,7 +573,7 @@ impl PlatformWrapper { let callback_arguments: Vec> = match result { Ok(fee_result) => { let js_fee_result = - task_context.boxed(FeeResultWrapper::new(fee_result.clone())); + task_context.boxed(FeeResultWrapper::new(fee_result)); // First parameter of JS callbacks is error, which is null in this case vec![task_context.null().upcast(), js_fee_result.upcast()] @@ -656,7 +656,7 @@ impl PlatformWrapper { let callback_arguments: Vec> = match result { Ok(fee_result) => { let js_fee_result = - task_context.boxed(FeeResultWrapper::new(fee_result.clone())); + task_context.boxed(FeeResultWrapper::new(fee_result)); // First parameter of JS callbacks is error, which is null in this case vec![task_context.null().upcast(), js_fee_result.upcast()] @@ -736,7 +736,7 @@ impl PlatformWrapper { let callback_arguments: Vec> = match result { Ok(fee_result) => { let js_fee_result = - task_context.boxed(FeeResultWrapper::new(fee_result.clone())); + task_context.boxed(FeeResultWrapper::new(fee_result)); // First parameter of JS callbacks is error, which is null in this case vec![task_context.null().upcast(), js_fee_result.upcast()] @@ -810,7 +810,7 @@ impl PlatformWrapper { let callback_arguments: Vec> = match result { Ok(fee_result) => { let js_fee_result = - task_context.boxed(FeeResultWrapper::new(fee_result.clone())); + task_context.boxed(FeeResultWrapper::new(fee_result)); // First parameter of JS callbacks is error, which is null in this case vec![task_context.null().upcast(), js_fee_result.upcast()] @@ -886,7 +886,7 @@ impl PlatformWrapper { let callback_arguments: Vec> = match result { Ok(fee_result) => { let js_fee_result = - task_context.boxed(FeeResultWrapper::new(fee_result.clone())); + task_context.boxed(FeeResultWrapper::new(fee_result)); // First parameter of JS callbacks is error, which is null in this case vec![task_context.null().upcast(), js_fee_result.upcast()] diff --git a/packages/rs-drive-nodejs/test/Drive.spec.js b/packages/rs-drive-nodejs/test/Drive.spec.js index 5610c0e623e..450433282dd 100644 --- a/packages/rs-drive-nodejs/test/Drive.spec.js +++ b/packages/rs-drive-nodejs/test/Drive.spec.js @@ -570,8 +570,8 @@ describe('Drive', () => { storageFee: 100, processingFee: 100, feeRefunds: { - 1: 15, - 2: 16, + 1: -15, + 2: -16, }, }, }; diff --git a/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs b/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs index e010f418ac0..a45f971c2d2 100644 --- a/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs +++ b/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs @@ -35,8 +35,6 @@ use crate::drive::flags::StorageFlags; use grovedb::batch::{GroveDbOp, GroveDbOpConsistencyResults}; use grovedb::Element; -// use std::slice::Iter; -// use std::vec::IntoIter; /// A batch of GroveDB operations as a vector. // TODO move to GroveDB diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index d7097193161..2d1df94f278 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -92,17 +92,17 @@ impl Drive { return Ok(()); } - let min_epoch_index_key = credits_per_epochs.keys().min().ok_or(Error::Drive( + let min_epoch_index = credits_per_epochs.keys().min().ok_or(Error::Drive( DriveError::CorruptedCodeExecution("can't find min epoch index"), ))?; - let min_epoch_index = min_epoch_index_key.to_owned() as u16; - let min_encoded_epoch_index = paths::encode_epoch_index_key(min_epoch_index)?.to_vec(); + let min_encoded_epoch_index = + paths::encode_epoch_index_key(min_epoch_index.to_owned())?.to_vec(); - let max_epoch_index_key = credits_per_epochs.keys().max().ok_or(Error::Drive( + let max_epoch_index = credits_per_epochs.keys().max().ok_or(Error::Drive( DriveError::CorruptedCodeExecution("can't find max epoch index"), ))?; - let max_epoch_index = max_epoch_index_key.to_owned() as u16; - let max_encoded_epoch_index = paths::encode_epoch_index_key(max_epoch_index)?.to_vec(); + let max_encoded_epoch_index = + paths::encode_epoch_index_key(max_epoch_index.to_owned())?.to_vec(); if max_epoch_index - min_epoch_index + 1 != credits_per_epochs.len() as u16 { return Err(Error::Drive(DriveError::CorruptedCodeExecution( @@ -119,7 +119,7 @@ impl Drive { epochs_query.set_subquery(storage_fee_pool_query); - let (mut storage_fee_pools, _) = self + let (storage_fee_pools, _) = self .grove .query( &PathQuery::new_unsized(pools_vec_path(), epochs_query), @@ -128,7 +128,6 @@ impl Drive { .unwrap() .map_err(Error::GroveDB)?; - // Sort epochs to reverse order to pop existing values for (i, (epoch_index, credits)) in credits_per_epochs .into_iter() .sorted_by_key(|x| x.0) diff --git a/packages/rs-drive/src/fee/credits.rs b/packages/rs-drive/src/fee/credits.rs index 90d4a2bf5fd..45b0e2a9324 100644 --- a/packages/rs-drive/src/fee/credits.rs +++ b/packages/rs-drive/src/fee/credits.rs @@ -61,12 +61,12 @@ pub trait Creditable: Into { impl Creditable for Credits { fn to_signed(&self) -> Result { - SignedCredits::try_from(self.clone()) + SignedCredits::try_from(*self) .map_err(|_| get_overflow_error("credits are too big to convert to signed value")) } fn to_unsigned(&self) -> Credits { - self.clone() + *self } fn from_vec_bytes(vec: Vec) -> Result { @@ -85,7 +85,7 @@ impl Creditable for Credits { } impl Creditable for SignedCredits { fn to_signed(&self) -> Result { - Ok(self.clone()) + Ok(*self) } fn to_unsigned(&self) -> Credits { diff --git a/packages/rs-drive/src/fee/mod.rs b/packages/rs-drive/src/fee/mod.rs index d1122404006..b36e17ab494 100644 --- a/packages/rs-drive/src/fee/mod.rs +++ b/packages/rs-drive/src/fee/mod.rs @@ -28,7 +28,6 @@ // use enum_map::EnumMap; -use rust_decimal::Decimal; use crate::error::fee::FeeError; use crate::error::Error; diff --git a/packages/rs-drive/src/fee/result/refunds.rs b/packages/rs-drive/src/fee/result/refunds.rs index b57b8901b10..e3c68589a46 100644 --- a/packages/rs-drive/src/fee/result/refunds.rs +++ b/packages/rs-drive/src/fee/result/refunds.rs @@ -36,7 +36,6 @@ use crate::error::fee::FeeError; use crate::error::Error; use crate::fee::credits::SignedCredits; use crate::fee::default_costs::STORAGE_DISK_USAGE_CREDIT_PER_BYTE; -use crate::fee::epoch::distribution::calculate_distribution_storage_fee_to_epochs_leftovers; use crate::fee::epoch::SignedCreditsPerEpoch; use crate::fee::get_overflow_error; use bincode::Options; From 0aeab31f55b11652687eca7c5b9d09e350d9746a Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Tue, 20 Dec 2022 00:46:21 +0800 Subject: [PATCH 21/54] chore: remove unused import --- packages/js-drive/test/integration/fee/feesPrediction.spec.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/js-drive/test/integration/fee/feesPrediction.spec.js b/packages/js-drive/test/integration/fee/feesPrediction.spec.js index 6e455f70f19..2c5505b27ec 100644 --- a/packages/js-drive/test/integration/fee/feesPrediction.spec.js +++ b/packages/js-drive/test/integration/fee/feesPrediction.spec.js @@ -17,7 +17,6 @@ const BlsSignatures = require('@dashevo/dpp/lib/bls/bls'); const createTestDIContainer = require('../../../lib/test/createTestDIContainer'); const createDataContractDocuments = require('../../../lib/test/fixtures/createDataContractDocuments'); const BlockInfo = require('../../../lib/blockExecution/BlockInfo'); -const calculateStateTransitionFee = require("@dashevo/dpp/lib/stateTransition/fee/calculateStateTransitionFee"); /** * @param {DashPlatformProtocol} dpp From f0aa40ee303c87aebbc1d65a6f1532eb27a57cfc Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Tue, 20 Dec 2022 00:50:16 +0800 Subject: [PATCH 22/54] tests: update fixtures --- packages/js-drive/test/integration/fee/pools.spec.js | 2 +- .../unit/abci/handlers/prepareProposalHandlerFactory.spec.js | 2 +- .../test/unit/abci/handlers/proposal/deliverTxFactory.spec.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/js-drive/test/integration/fee/pools.spec.js b/packages/js-drive/test/integration/fee/pools.spec.js index 8d82a5a7e3e..2b2ff900308 100644 --- a/packages/js-drive/test/integration/fee/pools.spec.js +++ b/packages/js-drive/test/integration/fee/pools.spec.js @@ -109,7 +109,7 @@ describe('Fee Pools', () => { feeRefunds: { 1: -15, }, - feeRefundsSum: 15, + feeRefundsSum: -15, }, }; diff --git a/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js index 671781ced40..33f306de540 100644 --- a/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js @@ -80,7 +80,7 @@ describe('prepareProposalHandlerFactory', () => { feeRefunds: { 1: -15, }, - feeRefundsSum: 15, + feeRefundsSum: -15, }, }); diff --git a/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js index 6fc6baeaa69..bd7ab744c29 100644 --- a/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js @@ -55,8 +55,8 @@ describe('deliverTxFactory', () => { stateTransitionExecutionContextMock.setLastCalculatedFeeDetails({ storageFee: 100, processingFee: 10, - feeRefunds: [{ identifier: Buffer.alloc(32), creditsPerEpoch: { 1: 15 } }], - feeRefundsSum: 15, + feeRefunds: [{ identifier: Buffer.alloc(32), creditsPerEpoch: { 1: -15 } }], + feeRefundsSum: -15, total: 95, }); From 4b62a8fb27ca3aa282b0219e8c4ada8c5a5d9b52 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Tue, 20 Dec 2022 01:00:27 +0800 Subject: [PATCH 23/54] fix: Cannot read properties of undefined (reading 'creditsPerEpoch') --- .../lib/abci/handlers/proposal/deliverTxFactory.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js b/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js index b0951b48c1f..3d191028799 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js @@ -250,12 +250,17 @@ function deliverTxFactory( `${stateTransition.constructor.name} execution took ${deliverTxTiming} seconds and cost ${actualStateTransitionFees.total} credits`, ); + let feeRefunds = {}; + if (actualStateTransitionFees.feeRefunds.length > 0) { + feeRefunds = actualStateTransitionFees.feeRefunds[0].creditsPerEpoch; + } + return { code: 0, fees: { storageFee: actualStateTransitionFees.storageFee, processingFee: actualStateTransitionFees.processingFee, - feeRefunds: actualStateTransitionFees.feeRefunds[0].creditsPerEpoch, + feeRefunds, feeRefundsSum: actualStateTransitionFees.feeRefundsSum, }, }; From 8937ec297076ac13deb071c301ed3195a5e7e135 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Tue, 20 Dec 2022 11:52:57 +0800 Subject: [PATCH 24/54] revert: enable dry run fee validation --- .../unserializeStateTransitionFactory.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/js-drive/lib/abci/handlers/stateTransition/unserializeStateTransitionFactory.js b/packages/js-drive/lib/abci/handlers/stateTransition/unserializeStateTransitionFactory.js index 252fccf29d9..aaabd3db10d 100644 --- a/packages/js-drive/lib/abci/handlers/stateTransition/unserializeStateTransitionFactory.js +++ b/packages/js-drive/lib/abci/handlers/stateTransition/unserializeStateTransitionFactory.js @@ -83,16 +83,17 @@ function unserializeStateTransitionFactory(dpp, noopLogger) { executionTimer.startTimer(TIMERS.DELIVER_TX.VALIDATE_FEE); - const executionContext = stateTransition.getExecutionContext(); + // const executionContext = stateTransition.getExecutionContext(); + // TODO: Enable fee validation when RS Drive is ready // Pre-calculate fee for validateState and state transition apply // with worst case costs to validate the whole state transition execution cost - executionContext.enableDryRun(); - - await dpp.stateTransition.validateState(stateTransition); - await dpp.stateTransition.apply(stateTransition); - - executionContext.disableDryRun(); + // executionContext.enableDryRun(); + // + // await dpp.stateTransition.validateState(stateTransition); + // await dpp.stateTransition.apply(stateTransition); + // + // executionContext.disableDryRun(); result = await dpp.stateTransition.validateFee(stateTransition); From 61326dce831ff7baa96b7fdbc85073d4c34a28a7 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Tue, 20 Dec 2022 12:51:53 +0800 Subject: [PATCH 25/54] docs: add documentation for modules --- .../fee/calculateStateTransitionFee.js | 2 +- .../drive/fee_pools/pending_epoch_updates.rs | 8 +++++ packages/rs-drive/src/fee/credits.rs | 14 ++++++-- packages/rs-drive/src/fee/default_costs.rs | 4 +-- .../rs-drive/src/fee/epoch/distribution.rs | 8 +++-- packages/rs-drive/src/fee/epoch/mod.rs | 4 +-- packages/rs-drive/src/fee/mod.rs | 2 ++ packages/rs-drive/src/fee/result/mod.rs | 7 ++-- packages/rs-drive/src/fee/result/refunds.rs | 4 +-- packages/rs-drive/src/fee_pools/epochs/mod.rs | 32 +++++++++++++++++++ 10 files changed, 72 insertions(+), 13 deletions(-) diff --git a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js index 2ae07eee671..7c26d3d2229 100644 --- a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js +++ b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js @@ -32,7 +32,7 @@ function calculateStateTransitionFee(stateTransition) { throw new Error('State Transition removed bytes from different identity'); } - // TODO: We should deduct leftovers + // TODO: We should deduct leftovers and already paid epochs feeRefundsSum = feeRefunds[0].creditsPerEpoch.entries() .reduce((sum, [, credits]) => sum + credits, 0); diff --git a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs index 5145d42db31..0c3d61f36b7 100644 --- a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs +++ b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs @@ -28,6 +28,14 @@ //! Pending epoch pool updates //! +//! Credit refunds are calculated when data is removed for the state. +//! Identity is refunded immediately when we update Identity balance +//! after state transition execution, but epoch pools must be updated +//! as well to deduct refunded amount. To do not update every block all +//! storage epoch pools, we introduce additional structure which aggregate +//! all pending updates for epoch storage pools and apply them during +//! storage fee distribution on epoch change. +//! use crate::drive::batch::GroveDbOpBatch; use crate::drive::fee_pools::pools_pending_updates_path; diff --git a/packages/rs-drive/src/fee/credits.rs b/packages/rs-drive/src/fee/credits.rs index 45b0e2a9324..7444d29d2dc 100644 --- a/packages/rs-drive/src/fee/credits.rs +++ b/packages/rs-drive/src/fee/credits.rs @@ -27,10 +27,17 @@ // DEALINGS IN THE SOFTWARE. // -//! Fee pool constants. +//! Credits //! -//! This module defines constants related to fee distribution pools. +//! Credits are Platform native token and used for micro payments +//! between identities, state transitions fees and masternode rewards //! +//! Credits are minted on Platform by locking Dash on payment chain and +//! can be withdrawn back to the payment chain by burning them on Platform +//! and unlocking dash on the payment chain. +//! + +// TODO: Should be moved to DPP when integration is done use crate::error::drive::DriveError; use crate::error::Error; @@ -53,6 +60,9 @@ pub trait Creditable: Into { fn to_signed(&self) -> Result; /// Convert singed credit to unsigned fn to_unsigned(&self) -> Credits; + + // TODO: Should we implement serialize / unserialize traits instead? + /// Decode bytes to credits fn from_vec_bytes(vec: Vec) -> Result; /// Encode credits to bytes diff --git a/packages/rs-drive/src/fee/default_costs.rs b/packages/rs-drive/src/fee/default_costs.rs index 1cb6c6ca051..00fe6cec463 100644 --- a/packages/rs-drive/src/fee/default_costs.rs +++ b/packages/rs-drive/src/fee/default_costs.rs @@ -27,9 +27,9 @@ // DEALINGS IN THE SOFTWARE. // -//! Fee pool constants. +//! Fee costs //! -//! This module defines constants related to fee distribution pools. +//! Fee costs for Drive (GroveDB) operations //! /// Storage disk usage credit per byte diff --git a/packages/rs-drive/src/fee/epoch/distribution.rs b/packages/rs-drive/src/fee/epoch/distribution.rs index 941c0f15dc7..1553197aecd 100644 --- a/packages/rs-drive/src/fee/epoch/distribution.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -27,9 +27,13 @@ // DEALINGS IN THE SOFTWARE. // -//! Fee pool constants. +//! Storage fee distribution into epochs //! -//! This module defines constants related to fee distribution pools. +//! Data is stored in Platform "forever" currently, which is 50 years. +//! To incentivise masternodes to continue store and serve this data, +//! payments are distributed for entire period split into epochs. +//! Every epoch, new aggregated storage fees are distributed among epochs +//! and masternodes receive payouts for previous epoch. //! use crate::error::fee::FeeError; diff --git a/packages/rs-drive/src/fee/epoch/mod.rs b/packages/rs-drive/src/fee/epoch/mod.rs index 01c203d10b7..af2514c17fb 100644 --- a/packages/rs-drive/src/fee/epoch/mod.rs +++ b/packages/rs-drive/src/fee/epoch/mod.rs @@ -27,9 +27,9 @@ // DEALINGS IN THE SOFTWARE. // -//! Fee pool constants. +//! Epochs //! -//! This module defines constants related to fee distribution pools. +//! Fee distribution is based on epochs. One epoch is about 18 days //! use crate::fee::credits::{Credits, SignedCredits}; diff --git a/packages/rs-drive/src/fee/mod.rs b/packages/rs-drive/src/fee/mod.rs index b36e17ab494..0a4ceb2870a 100644 --- a/packages/rs-drive/src/fee/mod.rs +++ b/packages/rs-drive/src/fee/mod.rs @@ -27,6 +27,8 @@ // DEALINGS IN THE SOFTWARE. // +// TODO: Should be moved to DPP when integration is done + use enum_map::EnumMap; use crate::error::fee::FeeError; diff --git a/packages/rs-drive/src/fee/result/mod.rs b/packages/rs-drive/src/fee/result/mod.rs index fb58c9fe489..2b97e089782 100644 --- a/packages/rs-drive/src/fee/result/mod.rs +++ b/packages/rs-drive/src/fee/result/mod.rs @@ -27,9 +27,12 @@ // DEALINGS IN THE SOFTWARE. // -//! Fee pool constants. +//! Fee Result //! -//! This module defines constants related to fee distribution pools. +//! Each drive operation returns FeeResult after execution. +//! This result contains fees which are required to pay for +//! computation and storage. It also contains fees to refund +//! for removed data from the state. //! use crate::error::fee::FeeError; diff --git a/packages/rs-drive/src/fee/result/refunds.rs b/packages/rs-drive/src/fee/result/refunds.rs index e3c68589a46..0940aaa34f5 100644 --- a/packages/rs-drive/src/fee/result/refunds.rs +++ b/packages/rs-drive/src/fee/result/refunds.rs @@ -27,9 +27,9 @@ // DEALINGS IN THE SOFTWARE. // -//! Fee pool constants. +//! Fee Refunds //! -//! This module defines constants related to fee distribution pools. +//! Fee refunds are calculated based on removed bytes per epoch. //! use crate::error::fee::FeeError; diff --git a/packages/rs-drive/src/fee_pools/epochs/mod.rs b/packages/rs-drive/src/fee_pools/epochs/mod.rs index 7ea6834b199..f70cbcaec2b 100644 --- a/packages/rs-drive/src/fee_pools/epochs/mod.rs +++ b/packages/rs-drive/src/fee_pools/epochs/mod.rs @@ -1,3 +1,35 @@ +// MIT LICENSE +// +// Copyright (c) 2021 Dash Core Group +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +//! Epoch pools +//! + /// Epoch key constants module pub mod epoch_key_constants; pub mod operations_factory; From 6c9d2c4af3a5ac9e797e091c7ee65fa00def8577 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Tue, 20 Dec 2022 13:41:22 +0800 Subject: [PATCH 26/54] chore: switch from unpaid to current epoch index to simplify logic --- .../fee_pools/distribute_storage_pool.rs | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs index 7d1617e2c5e..5260c19c9a7 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs @@ -68,24 +68,16 @@ impl Platform { None, )?; - // Deduct refunds since epoch where data was removed skipping already paid or pay-in-progress epochs. + // Deduct refunds since epoch where data was removed skipping previous (already paid or pay-in-progress) epochs. // Leftovers are ignored since they already deducted from Identity's refund amount - let mut unpaid_epoch_index = self.drive.get_unpaid_epoch_index(transaction)?; - - // In case if we paying for older than previous epoch - // we need to switch to next one which we are not paying yet - if unpaid_epoch_index < current_epoch_index { - unpaid_epoch_index += 1; - }; - // TODO Better to use iterator do not load everything into memory for (epoch_index, credits) in self.drive.fetch_pending_updates(transaction)? { distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, credits, epoch_index, - Some(unpaid_epoch_index), + Some(current_epoch_index), )?; } @@ -161,7 +153,6 @@ mod tests { let storage_pool = 1000000; let current_epoch_index = 3; - let unpaid_epoch = 1; let mut batch = GroveDbOpBatch::new(); @@ -171,9 +162,6 @@ mod tests { epoch.add_init_empty_operations(&mut batch); } - // Store unpaid epoch index - batch.push(update_unpaid_epoch_index_operation(unpaid_epoch)); - // Store distribution storage fees batch.push(update_storage_fee_distribution_pool_operation(storage_pool)); @@ -236,7 +224,7 @@ mod tests { ) .expect("should distribute refunds"); - let already_paid_epochs = unpaid_epoch as i64 + 1 - epoch_index as i64; + let already_paid_epochs = current_epoch_index as i64 - epoch_index as i64; let already_paid_credits = if already_paid_epochs > 0 { credits_per_epochs From 592d9f27e6f3522fb95060618f1182eaa2a1544d Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Tue, 20 Dec 2022 23:56:59 +0800 Subject: [PATCH 27/54] feat: reduce paid epochs and leftovers from identity balance --- .../primitives/AbstractStateTransition.md | 8 -- .../js-dpp/lib/StateRepositoryInterface.js | 23 +-- .../applyDocumentsBatchTransitionFactory.js | 2 +- ...ateIdentityUpdateTransitionStateFactory.js | 2 +- .../AbstractStateTransition.js | 10 -- .../stateTransition/StateTransitionFacade.js | 4 +- .../fee/calculateOperationFees.js | 2 + .../fee/calculateStateTransitionFee.js | 49 ------- .../fee/calculateStateTransitionFeeFactory.js | 63 ++++++++ .../test/mocks/createStateRepositoryMock.js | 2 + ...entityUpdateTransitionStateFactory.spec.js | 2 +- ...tractStateTransitionIdentitySigned.spec.js | 11 -- .../StateTransitionFacade.spec.js | 2 +- .../calculateStateTransitionFee.spec.js | 32 ----- ...calculateStateTransitionFeeFactory.spec.js | 135 ++++++++++++++++++ ...plyDocumentsBatchTransitionFactory.spec.js | 2 +- ...cumentsBatchTransitionStateFactory.spec.js | 2 +- .../lib/dpp/CachedStateRepositoryDecorator.js | 18 ++- .../js-drive/lib/dpp/DriveStateRepository.js | 27 +++- .../lib/dpp/LoggedStateRepositoryDecorator.js | 26 +++- .../proposal/deliverTxFactory.spec.js | 14 +- .../LoggedStateRepositoryDecorator.spec.js | 2 +- packages/rs-drive-nodejs/Drive.js | 10 +- packages/rs-drive-nodejs/src/fee/mod.rs | 40 ++++++ .../src/{fee_result.rs => fee/result.rs} | 0 packages/rs-drive-nodejs/src/lib.rs | 10 +- .../rs-drive/src/fee/epoch/distribution.rs | 43 ++++-- ...entityUpdateTransitionStateFactory.spec.js | 2 +- ...tractStateTransitionIdentitySigned.spec.js | 1 - .../StateTransitionFacade.spec.js | 2 +- .../calculateStateTransitionFee.spec.js | 32 ----- ...calculateStateTransitionFeeFactory.spec.js | 135 ++++++++++++++++++ ...plyDocumentsBatchTransitionFactory.spec.js | 2 +- ...cumentsBatchTransitionStateFactory.spec.js | 2 +- 34 files changed, 521 insertions(+), 196 deletions(-) delete mode 100644 packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js create mode 100644 packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFeeFactory.js delete mode 100644 packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFee.spec.js create mode 100644 packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js create mode 100644 packages/rs-drive-nodejs/src/fee/mod.rs rename packages/rs-drive-nodejs/src/{fee_result.rs => fee/result.rs} (100%) delete mode 100644 packages/wasm-dpp/test/integration/stateTransition/calculateStateTransitionFee.spec.js create mode 100644 packages/wasm-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js diff --git a/packages/js-dpp/docs/primitives/AbstractStateTransition.md b/packages/js-dpp/docs/primitives/AbstractStateTransition.md index 607bb3df76f..a5be2d0dd27 100644 --- a/packages/js-dpp/docs/primitives/AbstractStateTransition.md +++ b/packages/js-dpp/docs/primitives/AbstractStateTransition.md @@ -73,14 +73,6 @@ **Returns**: {boolean} -## .calculateFee() - -**Description**: Calculate ST fee in credits - -**Parameters**: None. - -**Returns**: {number} - ## .toObject(options) **Description**: Return state transition as plain object diff --git a/packages/js-dpp/lib/StateRepositoryInterface.js b/packages/js-dpp/lib/StateRepositoryInterface.js index 9e35bb07479..49ae3a8f207 100644 --- a/packages/js-dpp/lib/StateRepositoryInterface.js +++ b/packages/js-dpp/lib/StateRepositoryInterface.js @@ -151,15 +151,6 @@ * @returns {Promise>} */ -/** - * Fetch the latest platform block time - * - * @async - * @method - * @name StateRepository#fetchLatestPlatformBlockTime - * @returns {Promise} - */ - /** * Fetch the latest platform block height * @@ -223,7 +214,19 @@ /** * Returns current block time in milliseconds * + * @async * @method * @name StateRepository#fetchLatestPlatformBlockTime - * @returns {number} + * @returns {Promise} + */ + +/** + * Calculates storage fee to epochs distribution amount and leftovers + * + * @async + * @method + * @name StateRepository#calculateStorageFeeDistributionAmountAndLeftovers + * @param {number} storageFee + * @param {number} startEpochIndex + * @returns {Promise<[number, number]>} */ diff --git a/packages/js-dpp/lib/document/stateTransition/DocumentsBatchTransition/applyDocumentsBatchTransitionFactory.js b/packages/js-dpp/lib/document/stateTransition/DocumentsBatchTransition/applyDocumentsBatchTransitionFactory.js index cc39498574f..a2b31575cbb 100644 --- a/packages/js-dpp/lib/document/stateTransition/DocumentsBatchTransition/applyDocumentsBatchTransitionFactory.js +++ b/packages/js-dpp/lib/document/stateTransition/DocumentsBatchTransition/applyDocumentsBatchTransitionFactory.js @@ -79,7 +79,7 @@ function applyDocumentsBatchTransitionFactory( case AbstractDocumentTransition.ACTIONS.REPLACE: { let document; if (executionContext.isDryRun()) { - const lastBlockHeaderTime = stateRepository.fetchLatestPlatformBlockTime(); + const lastBlockHeaderTime = await stateRepository.fetchLatestPlatformBlockTime(); document = new Document({ $protocolVersion: stateTransition.getProtocolVersion(), diff --git a/packages/js-dpp/lib/identity/stateTransition/IdentityUpdateTransition/validation/state/validateIdentityUpdateTransitionStateFactory.js b/packages/js-dpp/lib/identity/stateTransition/IdentityUpdateTransition/validation/state/validateIdentityUpdateTransitionStateFactory.js index acc3b51d0d3..969670b3ce2 100644 --- a/packages/js-dpp/lib/identity/stateTransition/IdentityUpdateTransition/validation/state/validateIdentityUpdateTransitionStateFactory.js +++ b/packages/js-dpp/lib/identity/stateTransition/IdentityUpdateTransition/validation/state/validateIdentityUpdateTransitionStateFactory.js @@ -89,7 +89,7 @@ function validateIdentityUpdateTransitionStateFactory( // Calculate time window for timestamps - const lastBlockHeaderTime = stateRepository.fetchLatestPlatformBlockTime(); + const lastBlockHeaderTime = await stateRepository.fetchLatestPlatformBlockTime(); const disabledAtTime = stateTransition.getPublicKeysDisabledAt(); diff --git a/packages/js-dpp/lib/stateTransition/AbstractStateTransition.js b/packages/js-dpp/lib/stateTransition/AbstractStateTransition.js index fe552a8dcfe..6818509ecf4 100644 --- a/packages/js-dpp/lib/stateTransition/AbstractStateTransition.js +++ b/packages/js-dpp/lib/stateTransition/AbstractStateTransition.js @@ -13,7 +13,6 @@ const stateTransitionTypes = require('./stateTransitionTypes'); const hashModule = require('../util/hash'); const serializer = require('../util/serializer'); -const calculateStateTransitionFee = require('./fee/calculateStateTransitionFee'); const IdentityPublicKey = require('../identity/IdentityPublicKey'); const InvalidIdentityPublicKeyTypeError = require('./errors/InvalidIdentityPublicKeyTypeError'); const blsPrivateKeyFactory = require('../bls/blsPrivateKeyFactory'); @@ -292,15 +291,6 @@ class AbstractStateTransition { return blsSignature.verify(); } - /** - * Calculate ST fee in credits - * - * @return {number} - */ - calculateFee() { - return calculateStateTransitionFee(this); - } - /** * Returns ids of entities affected by the state transition * @abstract diff --git a/packages/js-dpp/lib/stateTransition/StateTransitionFacade.js b/packages/js-dpp/lib/stateTransition/StateTransitionFacade.js index f3022675ef5..83f7552fd66 100644 --- a/packages/js-dpp/lib/stateTransition/StateTransitionFacade.js +++ b/packages/js-dpp/lib/stateTransition/StateTransitionFacade.js @@ -69,7 +69,6 @@ const applyIdentityUpdateTransitionFactory = require( '../identity/stateTransition/IdentityUpdateTransition/applyIdentityUpdateTransitionFactory', ); const validateInstantAssetLockProofStructureFactory = require('../identity/stateTransition/assetLockProof/instant/validateInstantAssetLockProofStructureFactory'); -const calculateStateTransitionFee = require('./fee/calculateStateTransitionFee'); const InstantAssetLockProof = require('../identity/stateTransition/assetLockProof/instant/InstantAssetLockProof'); const ChainAssetLockProof = require('../identity/stateTransition/assetLockProof/chain/ChainAssetLockProof'); const validateChainAssetLockProofStructureFactory = require('../identity/stateTransition/assetLockProof/chain/validateChainAssetLockProofStructureFactory'); @@ -93,6 +92,7 @@ const getPropertyDefinitionByPath = require('../dataContract/getPropertyDefiniti const identityJsonSchema = require('../../schema/identity/stateTransition/publicKey.json'); const validatePublicKeySignaturesFactory = require('../identity/stateTransition/validatePublicKeySignaturesFactory'); const StateTransitionExecutionContext = require('./StateTransitionExecutionContext'); +const calculateStateTransitionFeeFactory = require('./fee/calculateStateTransitionFeeFactory'); class StateTransitionFacade { /** @@ -298,6 +298,8 @@ class StateTransitionFacade { [stateTransitionTypes.IDENTITY_UPDATE]: validateIdentityUpdateTransitionState, }); + const calculateStateTransitionFee = calculateStateTransitionFeeFactory(this.stateRepository); + this.validateStateTransitionFee = validateStateTransitionFeeFactory( this.stateRepository, calculateStateTransitionFee, diff --git a/packages/js-dpp/lib/stateTransition/fee/calculateOperationFees.js b/packages/js-dpp/lib/stateTransition/fee/calculateOperationFees.js index 0af2c2e6b0b..d2c9e28edaa 100644 --- a/packages/js-dpp/lib/stateTransition/fee/calculateOperationFees.js +++ b/packages/js-dpp/lib/stateTransition/fee/calculateOperationFees.js @@ -5,6 +5,8 @@ const { /** * Calculate processing and storage fees based on operations * + * + * @typedef {calculateOperationFees} * @param {AbstractOperation[]} operations * * @returns {{ diff --git a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js deleted file mode 100644 index 7c26d3d2229..00000000000 --- a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFee.js +++ /dev/null @@ -1,49 +0,0 @@ -const { - DEFAULT_USER_TIP, -} = require('./constants'); - -const calculateOperationFees = require('./calculateOperationFees'); - -/** - * @typedef {calculateStateTransitionFee} - * @param {AbstractStateTransition} stateTransition - * @return {number} - */ -function calculateStateTransitionFee(stateTransition) { - const executionContext = stateTransition.getExecutionContext(); - - const calculatedFees = calculateOperationFees(executionContext.getOperations()); - - const { - storageFee, - processingFee, - feeRefunds, - } = calculatedFees; - - if (feeRefunds.length > 1) { - throw new Error('State Transition removed bytes from several identities that is not defined by protocol'); - } - - let feeRefundsSum = 0; - if (feeRefunds.length > 0) { - const stateTransitionIdentifier = stateTransition.getOwnerId(); - - if (!stateTransitionIdentifier.equals(feeRefunds[0].identifier)) { - throw new Error('State Transition removed bytes from different identity'); - } - - // TODO: We should deduct leftovers and already paid epochs - - feeRefundsSum = feeRefunds[0].creditsPerEpoch.entries() - .reduce((sum, [, credits]) => sum + credits, 0); - } - - // Fee refunds are negative - const total = (storageFee + processingFee + feeRefundsSum) + DEFAULT_USER_TIP; - - executionContext.setLastCalculatedFeeDetails({ ...calculatedFees, feeRefundsSum, total }); - - return total; -} - -module.exports = calculateStateTransitionFee; diff --git a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFeeFactory.js b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFeeFactory.js new file mode 100644 index 00000000000..0648fb987bd --- /dev/null +++ b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFeeFactory.js @@ -0,0 +1,63 @@ +const { + DEFAULT_USER_TIP, +} = require('./constants'); + +/** + * @param {StateRepository} stateRepository + * @param {calculateOperationFees} calculateOperationFees + * @returns {calculateStateTransitionFee} + */ +function calculateStateTransitionFeeFactory(stateRepository, calculateOperationFees) { + /** + * @typedef {calculateStateTransitionFee} + * @param {AbstractStateTransition} stateTransition + * @return {Promise} + */ + async function calculateStateTransitionFee(stateTransition) { + const executionContext = stateTransition.getExecutionContext(); + + const calculatedFees = calculateOperationFees(executionContext.getOperations()); + + const { + storageFee, + processingFee, + feeRefunds, + } = calculatedFees; + + if (feeRefunds.length > 1) { + throw new Error('State Transition removed bytes from several identities that is not defined by protocol'); + } + + let feeRefundsSum = 0; + if (feeRefunds.length > 0) { + const stateTransitionIdentifier = stateTransition.getOwnerId(); + + if (!stateTransitionIdentifier.equals(feeRefunds[0].identifier)) { + throw new Error('State Transition removed bytes from different identity'); + } + + feeRefundsSum = await Object.entries(feeRefunds[0].creditsPerEpoch) + .reduce(async (sum, [epochIndex, credits]) => { + const [amount, leftovers] = await stateRepository + .calculateStorageFeeDistributionAmountAndLeftovers(credits, Number(epochIndex)); + + return (await sum) + amount - leftovers; + }, 0); + } + + // Fee refunds are negative + const total = (storageFee + processingFee + feeRefundsSum) + DEFAULT_USER_TIP; + + executionContext.setLastCalculatedFeeDetails({ + ...calculatedFees, + feeRefundsSum, + total, + }); + + return total; + } + + return calculateStateTransitionFee; +} + +module.exports = calculateStateTransitionFeeFactory; diff --git a/packages/js-dpp/lib/test/mocks/createStateRepositoryMock.js b/packages/js-dpp/lib/test/mocks/createStateRepositoryMock.js index 71b99f19311..a4e75f55784 100644 --- a/packages/js-dpp/lib/test/mocks/createStateRepositoryMock.js +++ b/packages/js-dpp/lib/test/mocks/createStateRepositoryMock.js @@ -1,4 +1,5 @@ /** + * @typedef {createStateRepositoryMock} * @param sinonSandbox * @return {{ * fetchDataContract: *, @@ -51,5 +52,6 @@ module.exports = function createStateRepositoryMock(sinonSandbox) { fetchLatestWithdrawalTransactionIndex: sinonSandbox.stub(), enqueueWithdrawalTransaction: sinonSandbox.stub(), fetchLatestPlatformBlockTime: sinonSandbox.stub(), + calculateStorageFeeDistributionAmountAndLeftovers: sinonSandbox.stub(), }; }; diff --git a/packages/js-dpp/test/integration/identity/stateTransition/IdentityUpdateTransition/validation/state/validateIdentityUpdateTransitionStateFactory.spec.js b/packages/js-dpp/test/integration/identity/stateTransition/IdentityUpdateTransition/validation/state/validateIdentityUpdateTransitionStateFactory.spec.js index 41f9f488d5a..46070f5fcde 100644 --- a/packages/js-dpp/test/integration/identity/stateTransition/IdentityUpdateTransition/validation/state/validateIdentityUpdateTransitionStateFactory.spec.js +++ b/packages/js-dpp/test/integration/identity/stateTransition/IdentityUpdateTransition/validation/state/validateIdentityUpdateTransitionStateFactory.spec.js @@ -36,7 +36,7 @@ describe('validateIdentityUpdateTransitionStateFactory', () => { blockTime = Date.now(); - stateRepositoryMock.fetchLatestPlatformBlockTime.returns(blockTime); + stateRepositoryMock.fetchLatestPlatformBlockTime.resolves(blockTime); validateIdentityUpdateTransitionState = validateIdentityUpdateTransitionStateFactory( stateRepositoryMock, diff --git a/packages/js-dpp/test/integration/stateTransition/AbstractStateTransitionIdentitySigned.spec.js b/packages/js-dpp/test/integration/stateTransition/AbstractStateTransitionIdentitySigned.spec.js index ec7d36b181f..966d4e7313a 100644 --- a/packages/js-dpp/test/integration/stateTransition/AbstractStateTransitionIdentitySigned.spec.js +++ b/packages/js-dpp/test/integration/stateTransition/AbstractStateTransitionIdentitySigned.spec.js @@ -1,7 +1,6 @@ const { PrivateKey, crypto: { Hash } } = require('@dashevo/dashcore-lib'); const crypto = require('crypto'); -const calculateStateTransitionFee = require('../../../lib/stateTransition/fee/calculateStateTransitionFee'); const StateTransitionMock = require('../../../lib/test/mocks/StateTransitionMock'); const IdentityPublicKey = require('../../../lib/identity/IdentityPublicKey'); @@ -494,14 +493,4 @@ describe('AbstractStateTransitionIdentitySigned', () => { expect(stateTransition.signaturePublicKeyId).to.equal(signaturePublicKeyId); }); }); - - describe('#calculateFee', () => { - it('should calculate fee', () => { - const result = stateTransition.calculateFee(); - - const fee = calculateStateTransitionFee(stateTransition); - - expect(result).to.equal(fee); - }); - }); }); diff --git a/packages/js-dpp/test/integration/stateTransition/StateTransitionFacade.spec.js b/packages/js-dpp/test/integration/stateTransition/StateTransitionFacade.spec.js index 802756164bc..244428a9e7e 100644 --- a/packages/js-dpp/test/integration/stateTransition/StateTransitionFacade.spec.js +++ b/packages/js-dpp/test/integration/stateTransition/StateTransitionFacade.spec.js @@ -71,7 +71,7 @@ describe('StateTransitionFacade', () => { stateRepositoryMock = createStateRepositoryMock(this.sinonSandbox); stateRepositoryMock.fetchIdentity.resolves(identity); - stateRepositoryMock.fetchLatestPlatformBlockTime.returns(blockTime); + stateRepositoryMock.fetchLatestPlatformBlockTime.resolves(blockTime); dpp = new DashPlatformProtocol({ stateRepository: stateRepositoryMock, diff --git a/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFee.spec.js b/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFee.spec.js deleted file mode 100644 index b8ef6a2172b..00000000000 --- a/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFee.spec.js +++ /dev/null @@ -1,32 +0,0 @@ -const calculateStateTransitionFee = require('../../../lib/stateTransition/fee/calculateStateTransitionFee'); - -const getIdentityCreateTransitionFixture = require('../../../lib/test/fixtures/getIdentityCreateTransitionFixture'); -const IdentityPublicKey = require('../../../lib/identity/IdentityPublicKey'); -const ReadOperation = require('../../../lib/stateTransition/fee/operations/ReadOperation'); -const PreCalculatedOperation = require('../../../lib/stateTransition/fee/operations/PreCalculatedOperation'); -const DummyFeeResult = require('../../../lib/stateTransition/fee/DummyFeeResult'); - -describe('calculateStateTransitionFee', () => { - let stateTransition; - - beforeEach(async () => { - const privateKey = 'af432c476f65211f45f48f1d42c9c0b497e56696aa1736b40544ef1a496af837'; - - stateTransition = getIdentityCreateTransitionFixture(); - await stateTransition.signByPrivateKey(privateKey, IdentityPublicKey.TYPES.ECDSA_SECP256K1); - }); - - // TODO: Must be more comprehensive. After we settle all factors and formula. - it('should calculate fee based on executed operations', () => { - const executionContext = stateTransition.getExecutionContext(); - - executionContext.addOperation( - new ReadOperation(10), - new PreCalculatedOperation(new DummyFeeResult(12, 12)), - ); - - const result = calculateStateTransitionFee(stateTransition); - - expect(result).to.equal(17088); - }); -}); diff --git a/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js b/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js new file mode 100644 index 00000000000..db331266c3a --- /dev/null +++ b/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js @@ -0,0 +1,135 @@ +const calculateStateTransitionFeeFactory = require('../../../lib/stateTransition/fee/calculateStateTransitionFeeFactory'); + +const getIdentityCreateTransitionFixture = require('../../../lib/test/fixtures/getIdentityCreateTransitionFixture'); +const IdentityPublicKey = require('../../../lib/identity/IdentityPublicKey'); + +const createStateRepositoryMock = require('../../../lib/test/mocks/createStateRepositoryMock'); + +describe('calculateStateTransitionFeeFactory', () => { + let stateTransition; + let calculateStateTransitionFee; + let stateRepositoryMock; + let calculateOperationFeesMock; + + beforeEach(async function beforeEach() { + const privateKey = 'af432c476f65211f45f48f1d42c9c0b497e56696aa1736b40544ef1a496af837'; + + stateTransition = getIdentityCreateTransitionFixture(); + await stateTransition.signByPrivateKey(privateKey, IdentityPublicKey.TYPES.ECDSA_SECP256K1); + + stateRepositoryMock = createStateRepositoryMock(this.sinonSandbox); + + calculateOperationFeesMock = this.sinonSandbox.stub(); + + calculateStateTransitionFee = calculateStateTransitionFeeFactory( + stateRepositoryMock, + calculateOperationFeesMock, + ); + }); + + it('should throw an error if more than two identities have refunds', async () => { + const calculatedOperationsFeesResult = { + storageFee: 10000, + processingFee: 1000, + feeRefunds: [ + { + identifier: stateTransition.getOwnerId().toBuffer(), + creditsPerEpoch: { + 0: -100, + 1: -50, + }, + }, + { + identifier: stateTransition.getOwnerId().toBuffer(), + creditsPerEpoch: { + 1: -50, + }, + }, + ], + }; + + calculateOperationFeesMock.returns(calculatedOperationsFeesResult); + + try { + await calculateStateTransitionFee(stateTransition); + + expect.fail('should fail'); + } catch (e) { + expect(e.message).to.equals('State Transition removed bytes from several identities that is not defined by protocol'); + } + }); + + it('should throw an error if refunded identity is not owner of state transition', async () => { + const calculatedOperationsFeesResult = { + storageFee: 10000, + processingFee: 1000, + feeRefunds: [ + { + identifier: Buffer.alloc(32, 1), + creditsPerEpoch: { + 0: -100, + 1: -50, + }, + }, + ], + }; + + calculateOperationFeesMock.returns(calculatedOperationsFeesResult); + + try { + await calculateStateTransitionFee(stateTransition); + + expect.fail('should fail'); + } catch (e) { + expect(e.message).to.equals('State Transition removed bytes from different identity'); + } + }); + + it('should calculate fee based on executed operations', async () => { + const storageFee = 10000; + const processingFee = 1000; + const feeRefundsSum = -450 - 995 - 400 - 400; + const total = storageFee + processingFee + feeRefundsSum; + + const calculatedOperationsFeesResult = { + storageFee, + processingFee, + feeRefunds: [ + { + identifier: stateTransition.getOwnerId().toBuffer(), + creditsPerEpoch: { + 0: -1000, + 1: -500, + }, + }, + ], + }; + + calculateOperationFeesMock.returns(calculatedOperationsFeesResult); + + stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers + .onCall(0).resolves([-995, 400]); + + stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers + .onCall(1).resolves([-450, 400]); + + const result = await calculateStateTransitionFee(stateTransition); + + expect(result).to.equal(total); + + expect(stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers) + .to.have.been.calledWithExactly(-1000, 0); + + expect(stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers) + .to.have.been.calledWithExactly(-500, 1); + + const lastCalculatedFeeDetails = stateTransition.getExecutionContext() + .getLastCalculatedFeeDetails(); + + expect(lastCalculatedFeeDetails).to.be.deep.equal({ + ...calculatedOperationsFeesResult, + feeRefundsSum, + total, + }); + }); +}); diff --git a/packages/js-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/applyDocumentsBatchTransitionFactory.spec.js b/packages/js-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/applyDocumentsBatchTransitionFactory.spec.js index 785305dfc75..5448631415a 100644 --- a/packages/js-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/applyDocumentsBatchTransitionFactory.spec.js +++ b/packages/js-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/applyDocumentsBatchTransitionFactory.spec.js @@ -66,7 +66,7 @@ describe('applyDocumentsBatchTransitionFactory', () => { stateRepositoryMock = createStateRepositoryMock(this.sinonSandbox); stateRepositoryMock.fetchDataContract.resolves(dataContract); - stateRepositoryMock.fetchLatestPlatformBlockTime.returns(blockTimeMs); + stateRepositoryMock.fetchLatestPlatformBlockTime.resolves(blockTimeMs); fetchDocumentsMock = this.sinonSandbox.stub(); fetchDocumentsMock.resolves([ diff --git a/packages/js-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/validation/state/validateDocumentsBatchTransitionStateFactory.spec.js b/packages/js-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/validation/state/validateDocumentsBatchTransitionStateFactory.spec.js index 9550778afdd..2d89ee86034 100644 --- a/packages/js-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/validation/state/validateDocumentsBatchTransitionStateFactory.spec.js +++ b/packages/js-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/validation/state/validateDocumentsBatchTransitionStateFactory.spec.js @@ -68,7 +68,7 @@ describe('validateDocumentsBatchTransitionStateFactory', () => { blockTime = Date.now(); - stateRepositoryMock.fetchLatestPlatformBlockTime.returns(blockTime); + stateRepositoryMock.fetchLatestPlatformBlockTime.resolves(blockTime); fetchDocumentsMock = this.sinonSandbox.stub().resolves([]); diff --git a/packages/js-drive/lib/dpp/CachedStateRepositoryDecorator.js b/packages/js-drive/lib/dpp/CachedStateRepositoryDecorator.js index eb519c4277f..7086f6dea8b 100644 --- a/packages/js-drive/lib/dpp/CachedStateRepositoryDecorator.js +++ b/packages/js-drive/lib/dpp/CachedStateRepositoryDecorator.js @@ -279,11 +279,25 @@ class CachedStateRepositoryDecorator { /** * Returns block time * - * @returns {number} + * @returns {Promise} */ - fetchLatestPlatformBlockTime() { + async fetchLatestPlatformBlockTime() { return this.stateRepository.fetchLatestPlatformBlockTime(); } + + /** + * Calculates storage fee to epochs distribution amount and leftovers + * + * @param {number} storageFee + * @param {number} startEpochIndex + * @returns {Promise<[number, number]>} + */ + async calculateStorageFeeDistributionAmountAndLeftovers(storageFee, startEpochIndex) { + return this.stateRepository.calculateStorageFeeDistributionAmountAndLeftovers( + storageFee, + startEpochIndex, + ); + } } module.exports = CachedStateRepositoryDecorator; diff --git a/packages/js-drive/lib/dpp/DriveStateRepository.js b/packages/js-drive/lib/dpp/DriveStateRepository.js index 78475bd430c..0816509c321 100644 --- a/packages/js-drive/lib/dpp/DriveStateRepository.js +++ b/packages/js-drive/lib/dpp/DriveStateRepository.js @@ -1,3 +1,5 @@ +const { calculateStorageFeeDistributionAmountAndLeftovers } = require('@dashevo/rs-drive'); + const { TYPES } = require('@dashevo/dpp/lib/identity/IdentityPublicKey'); const ReadOperation = require('@dashevo/dpp/lib/stateTransition/fee/operations/ReadOperation'); @@ -435,9 +437,9 @@ class DriveStateRepository { /** * Fetch the latest platform block time * - * @return {number} + * @return {Promise} */ - fetchLatestPlatformBlockTime() { + async fetchLatestPlatformBlockTime() { const timeMs = this.blockExecutionContext.getTimeMs(); if (!timeMs) { @@ -541,6 +543,27 @@ class DriveStateRepository { ); } + /** + * Calculates storage fee to epochs distribution amount and leftovers + * + * @param {number} storageFee + * @param {number} startEpochIndex + * @returns {Promise<[number, number]>} + */ + async calculateStorageFeeDistributionAmountAndLeftovers(storageFee, startEpochIndex) { + const epochInfo = this.blockExecutionContext.getEpochInfo(); + + if (!epochInfo) { + throw new Error('epoch info is not set'); + } + + return calculateStorageFeeDistributionAmountAndLeftovers( + storageFee, + startEpochIndex, + epochInfo.currentEpochIndex, + ); + } + /** * @private * @param {StateTransitionExecutionContext} [executionContext] diff --git a/packages/js-drive/lib/dpp/LoggedStateRepositoryDecorator.js b/packages/js-drive/lib/dpp/LoggedStateRepositoryDecorator.js index 3ad734dc24f..591e580409e 100644 --- a/packages/js-drive/lib/dpp/LoggedStateRepositoryDecorator.js +++ b/packages/js-drive/lib/dpp/LoggedStateRepositoryDecorator.js @@ -498,13 +498,13 @@ class LoggedStateRepositoryDecorator { /** * Returns block time * - * @returns {number} + * @returns {Promise} */ - fetchLatestPlatformBlockTime() { + async fetchLatestPlatformBlockTime() { let response; try { - response = this.stateRepository.fetchLatestPlatformBlockTime(); + response = await this.stateRepository.fetchLatestPlatformBlockTime(); } finally { this.log('fetchLatestPlatformBlockTime', { }, response); } @@ -549,6 +549,26 @@ class LoggedStateRepositoryDecorator { this.log('enqueueWithdrawalTransaction', { index, transactionBytes }, response); } } + + /** + * Calculates storage fee to epochs distribution amount and leftovers + * + * @param {number} storageFee + * @param {number} startEpochIndex + * @returns {Promise<[number, number]>} + */ + async calculateStorageFeeDistributionAmountAndLeftovers(storageFee, startEpochIndex) { + let response; + + try { + response = await this.stateRepository.calculateStorageFeeDistributionAmountAndLeftovers( + storageFee, + startEpochIndex, + ); + } finally { + this.log('calculateStorageFeeDistributionAmountAndLeftovers', { storageFee, startEpochIndex }, response); + } + } } module.exports = LoggedStateRepositoryDecorator; diff --git a/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js index bd7ab744c29..064db776dab 100644 --- a/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js @@ -141,17 +141,13 @@ describe('deliverTxFactory', () => { ); expect(proposalBlockExecutionContextMock.addDataContract).to.not.be.called(); - const stateTransitionFee = documentsBatchTransitionFixture.calculateFee(); - - // TODO: enable once fee calculation is done - // expect(stateRepositoryMock.fetchIdentity).to.be.calledOnceWith( - // documentsBatchTransitionFixture.getOwnerId(), - // ); + expect(stateRepositoryMock.fetchIdentity).to.be.calledOnceWith( + documentsBatchTransitionFixture.getOwnerId(), + ); - identity.reduceBalance(stateTransitionFee); + identity.reduceBalance(stateTransitionExecutionContextMock.getLastCalculatedFeeDetails().total); - // TODO: enable once fee calculation is done - // expect(stateRepositoryMock.updateIdentity).to.be.calledOnceWith(identity); + expect(stateRepositoryMock.updateIdentity).to.be.calledOnceWith(identity); }); it('should apply a DataContractCreateTransition, add it to block execution state and return ResponseDeliverTx', async () => { diff --git a/packages/js-drive/test/unit/dpp/LoggedStateRepositoryDecorator.spec.js b/packages/js-drive/test/unit/dpp/LoggedStateRepositoryDecorator.spec.js index 67fe69fe309..81d9e3da579 100644 --- a/packages/js-drive/test/unit/dpp/LoggedStateRepositoryDecorator.spec.js +++ b/packages/js-drive/test/unit/dpp/LoggedStateRepositoryDecorator.spec.js @@ -693,7 +693,7 @@ describe('LoggedStateRepositoryDecorator', () => { it('should call logger with proper params', async () => { const response = {}; - stateRepositoryMock.fetchLatestPlatformBlockTime.returns(response); + stateRepositoryMock.fetchLatestPlatformBlockTime.resolves(response); await loggedStateRepositoryDecorator.fetchLatestPlatformBlockTime(); diff --git a/packages/rs-drive-nodejs/Drive.js b/packages/rs-drive-nodejs/Drive.js index d612048bc90..f543741b0e6 100644 --- a/packages/rs-drive-nodejs/Drive.js +++ b/packages/rs-drive-nodejs/Drive.js @@ -25,6 +25,7 @@ const { abciBlockBegin, abciBlockEnd, abciAfterFinalizeBlock, + calculateStorageFeeDistributionAmountAndLeftovers, } = require('neon-load-or-build')({ dir: __dirname, }); @@ -32,7 +33,7 @@ const { const GroveDB = require('./GroveDB'); const FeeResult = require('./FeeResult'); -const { appendStackAsync } = require('./appendStack'); +const { appendStackAsync, appendStack } = require('./appendStack'); const decodeProtocolEntity = decodeProtocolEntityFactory(); @@ -61,6 +62,10 @@ const abciBlockBeginAsync = appendStackAsync(promisify(abciBlockBegin)); const abciBlockEndAsync = appendStackAsync(promisify(abciBlockEnd)); const abciAfterFinalizeBlockAsync = appendStackAsync(promisify(abciAfterFinalizeBlock)); +const calculateStorageFeeDistributionAmountAndLeftoversWithStack = appendStack( + calculateStorageFeeDistributionAmountAndLeftovers, +); + // Wrapper class for the boxed `Drive` for idiomatic JavaScript usage class Drive { /** @@ -465,6 +470,9 @@ class Drive { } } +// eslint-disable-next-line max-len +Drive.calculateStorageFeeDistributionAmountAndLeftovers = calculateStorageFeeDistributionAmountAndLeftoversWithStack; + /** * @typedef RawBlockInfo * @property {number} height diff --git a/packages/rs-drive-nodejs/src/fee/mod.rs b/packages/rs-drive-nodejs/src/fee/mod.rs new file mode 100644 index 00000000000..8762a89dc1e --- /dev/null +++ b/packages/rs-drive-nodejs/src/fee/mod.rs @@ -0,0 +1,40 @@ +use drive::fee::credits::SignedCredits; +use drive::fee::epoch::distribution::calculate_storage_fee_distribution_amount_and_leftovers; +use drive::fee::epoch::EpochIndex; +use neon::prelude::*; + +pub mod result; + +pub fn js_calculate_storage_fee_distribution_amount_and_leftovers( + mut cx: FunctionContext, +) -> JsResult { + let js_storage_fees = cx.argument::(0)?; + let storage_fees = js_storage_fees.value(&mut cx) as SignedCredits; + + let js_start_epoch_index = cx.argument::(1)?; + + let start_epoch_index = EpochIndex::try_from(js_start_epoch_index.value(&mut cx) as i64) + .or_else(|_| cx.throw_range_error("`startEpochIndex` must fit in u16"))?; + + let js_skip_up_to_epoch_index = cx.argument::(2)?; + let skip_up_to_epoch_index = + EpochIndex::try_from(js_skip_up_to_epoch_index.value(&mut cx) as i64) + .or_else(|_| cx.throw_range_error("`startEpochIndex` must fit in u16"))?; + + let (amount, leftovers) = calculate_storage_fee_distribution_amount_and_leftovers( + storage_fees, + start_epoch_index, + skip_up_to_epoch_index, + ) + .or_else(|e| cx.throw_error(e.to_string()))?; + + let js_array = cx.empty_array(); + + let js_amount = cx.number(amount as f64); + let js_leftovers = cx.number(leftovers as f64); + + js_array.set(&mut cx, 0, js_amount)?; + js_array.set(&mut cx, 0, js_leftovers)?; + + Ok(js_array) +} diff --git a/packages/rs-drive-nodejs/src/fee_result.rs b/packages/rs-drive-nodejs/src/fee/result.rs similarity index 100% rename from packages/rs-drive-nodejs/src/fee_result.rs rename to packages/rs-drive-nodejs/src/fee/result.rs diff --git a/packages/rs-drive-nodejs/src/lib.rs b/packages/rs-drive-nodejs/src/lib.rs index 430a7b61daf..1201e5e024b 100644 --- a/packages/rs-drive-nodejs/src/lib.rs +++ b/packages/rs-drive-nodejs/src/lib.rs @@ -1,10 +1,10 @@ mod converter; -mod fee_result; +mod fee; use std::num::ParseIntError; use std::{option::Option::None, path::Path, sync::mpsc, thread}; -use crate::fee_result::FeeResultWrapper; +use crate::fee::result::FeeResultWrapper; use drive::dpp::identity::Identity; use drive::drive::batch::GroveDbOpBatch; use drive::drive::config::DriveConfig; @@ -21,6 +21,7 @@ use drive_abci::abci::messages::{ Serializable, }; use drive_abci::platform::Platform; +use fee::js_calculate_storage_fee_distribution_amount_and_leftovers; use neon::prelude::*; type PlatformCallback = Box FnOnce(&'a Platform, TransactionArg, &Channel) + Send>; @@ -2379,5 +2380,10 @@ fn main(mut cx: ModuleContext) -> NeonResult<()> { cx.export_function("feeResultCreate", FeeResultWrapper::create)?; cx.export_function("feeResultGetRefunds", FeeResultWrapper::get_fee_refunds)?; + cx.export_function( + "calculateStorageFeeDistributionAmountAndLeftovers", + js_calculate_storage_fee_distribution_amount_and_leftovers, + )?; + Ok(()) } diff --git a/packages/rs-drive/src/fee/epoch/distribution.rs b/packages/rs-drive/src/fee/epoch/distribution.rs index 1553197aecd..b6a568f49ed 100644 --- a/packages/rs-drive/src/fee/epoch/distribution.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -71,7 +71,7 @@ pub fn distribute_storage_fee_to_epochs_collection( storage_fee: SignedCredits, start_epoch_index: EpochIndex, skip_up_to_epoch_index: Option, -) -> Result { +) -> Result { distribution_storage_fee_to_epochs_map( storage_fee, start_epoch_index, @@ -97,14 +97,30 @@ pub fn distribute_storage_fee_to_epochs_collection( ) } -/// Calculates leftovers from distribution storage fees to epochs -pub fn calculate_distribution_storage_fee_to_epochs_leftovers( +type DistributionAmount = SignedCredits; +type DistributionLeftovers = SignedCredits; + +/// Calculates leftovers and amount of credits by distributing storage fees to epochs +pub fn calculate_storage_fee_distribution_amount_and_leftovers( storage_fee: SignedCredits, start_epoch_index: EpochIndex, -) -> Result { - // To do not mess `distribution_storage_fee_to_epochs_map` arguments better to pass dummy - // function. Rust optimizer reduce it and won't call anything. - distribution_storage_fee_to_epochs_map(storage_fee, start_epoch_index, |_, _| Ok(())) + skip_up_to_epoch_index: EpochIndex, +) -> Result<(DistributionAmount, DistributionLeftovers), Error> { + let mut skipped_amount = 0; + + let leftovers = distribution_storage_fee_to_epochs_map( + storage_fee, + start_epoch_index, + |epoch_index, epoch_fee_share| { + if epoch_index < skip_up_to_epoch_index { + skipped_amount += epoch_fee_share; + } + + Ok(()) + }, + )?; + + Ok((storage_fee - skipped_amount - leftovers, leftovers)) } /// Distributes storage fees to epochs and call function for each epoch. @@ -113,7 +129,7 @@ fn distribution_storage_fee_to_epochs_map( storage_fee: SignedCredits, start_epoch_index: EpochIndex, mut f: F, -) -> Result +) -> Result where F: FnMut(EpochIndex, SignedCredits) -> Result<(), Error>, { @@ -463,21 +479,24 @@ mod tests { } } - mod calculate_distribution_storage_fee_to_epochs_leftovers { + mod calculate_storage_fee_to_epochs_distribution_amount_and_leftovers { use super::*; #[test] - fn should_calculate_leftovers() { + fn should_calculate_amount_and_leftovers() { let storage_fee = 10000; - let leftovers = calculate_distribution_storage_fee_to_epochs_leftovers( + let (amount, leftovers) = calculate_storage_fee_distribution_amount_and_leftovers( storage_fee, GENESIS_EPOCH_INDEX, + 2, ) .expect("should distribute storage fee"); - // check leftover + let first_two_epochs_amount = 50; + assert_eq!(leftovers, 400); + assert_eq!(amount, storage_fee - leftovers - first_two_epochs_amount); } } } diff --git a/packages/wasm-dpp/test/integration/identity/stateTransition/IdentityUpdateTransition/validation/state/validateIdentityUpdateTransitionStateFactory.spec.js b/packages/wasm-dpp/test/integration/identity/stateTransition/IdentityUpdateTransition/validation/state/validateIdentityUpdateTransitionStateFactory.spec.js index add0fd34861..b43d9e792fe 100644 --- a/packages/wasm-dpp/test/integration/identity/stateTransition/IdentityUpdateTransition/validation/state/validateIdentityUpdateTransitionStateFactory.spec.js +++ b/packages/wasm-dpp/test/integration/identity/stateTransition/IdentityUpdateTransition/validation/state/validateIdentityUpdateTransitionStateFactory.spec.js @@ -36,7 +36,7 @@ describe('validateIdentityUpdateTransitionStateFactory', () => { blockTime = Date.now(); - stateRepositoryMock.fetchLatestPlatformBlockTime.returns(blockTime); + stateRepositoryMock.fetchLatestPlatformBlockTime.resolves(blockTime); validateIdentityUpdateTransitionState = validateIdentityUpdateTransitionStateFactory( stateRepositoryMock, diff --git a/packages/wasm-dpp/test/integration/stateTransition/AbstractStateTransitionIdentitySigned.spec.js b/packages/wasm-dpp/test/integration/stateTransition/AbstractStateTransitionIdentitySigned.spec.js index 58b279cc2b6..26f393f4722 100644 --- a/packages/wasm-dpp/test/integration/stateTransition/AbstractStateTransitionIdentitySigned.spec.js +++ b/packages/wasm-dpp/test/integration/stateTransition/AbstractStateTransitionIdentitySigned.spec.js @@ -1,7 +1,6 @@ const { PrivateKey, crypto: { Hash } } = require('@dashevo/dashcore-lib'); const crypto = require('crypto'); -const calculateStateTransitionFee = require('@dashevo/dpp/lib/stateTransition/fee/calculateStateTransitionFee'); const StateTransitionMock = require('@dashevo/dpp/lib/test/mocks/StateTransitionMock'); const IdentityPublicKey = require('@dashevo/dpp/lib/identity/IdentityPublicKey'); diff --git a/packages/wasm-dpp/test/integration/stateTransition/StateTransitionFacade.spec.js b/packages/wasm-dpp/test/integration/stateTransition/StateTransitionFacade.spec.js index d8bc224a9b6..d0a93bef855 100644 --- a/packages/wasm-dpp/test/integration/stateTransition/StateTransitionFacade.spec.js +++ b/packages/wasm-dpp/test/integration/stateTransition/StateTransitionFacade.spec.js @@ -71,7 +71,7 @@ describe('StateTransitionFacade', () => { stateRepositoryMock = createStateRepositoryMock(this.sinonSandbox); stateRepositoryMock.fetchIdentity.resolves(identity); - stateRepositoryMock.fetchLatestPlatformBlockTime.returns(blockTime); + stateRepositoryMock.fetchLatestPlatformBlockTime.resolves(blockTime); dpp = new DashPlatformProtocol({ stateRepository: stateRepositoryMock, diff --git a/packages/wasm-dpp/test/integration/stateTransition/calculateStateTransitionFee.spec.js b/packages/wasm-dpp/test/integration/stateTransition/calculateStateTransitionFee.spec.js deleted file mode 100644 index 3cc4af3d859..00000000000 --- a/packages/wasm-dpp/test/integration/stateTransition/calculateStateTransitionFee.spec.js +++ /dev/null @@ -1,32 +0,0 @@ -const calculateStateTransitionFee = require('@dashevo/dpp/lib/stateTransition/fee/calculateStateTransitionFee'); - -const getIdentityCreateTransitionFixture = require('@dashevo/dpp/lib/test/fixtures/getIdentityCreateTransitionFixture'); -const IdentityPublicKey = require('@dashevo/dpp/lib/identity/IdentityPublicKey'); -const ReadOperation = require('@dashevo/dpp/lib/stateTransition/fee/operations/ReadOperation'); -const PreCalculatedOperation = require('@dashevo/dpp/lib/stateTransition/fee/operations/PreCalculatedOperation'); -const DummyFeeResult = require('@dashevo/dpp/lib/stateTransition/fee/DummyFeeResult'); - -describe('calculateStateTransitionFee', () => { - let stateTransition; - - beforeEach(async () => { - const privateKey = 'af432c476f65211f45f48f1d42c9c0b497e56696aa1736b40544ef1a496af837'; - - stateTransition = getIdentityCreateTransitionFixture(); - await stateTransition.signByPrivateKey(privateKey, IdentityPublicKey.TYPES.ECDSA_SECP256K1); - }); - - // TODO: Must be more comprehensive. After we settle all factors and formula. - it('should calculate fee based on executed operations', () => { - const executionContext = stateTransition.getExecutionContext(); - - executionContext.addOperation( - new ReadOperation(10), - new PreCalculatedOperation(new DummyFeeResult(12, 12)), - ); - - const result = calculateStateTransitionFee(stateTransition); - - expect(result).to.equal(17088); - }); -}); diff --git a/packages/wasm-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js b/packages/wasm-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js new file mode 100644 index 00000000000..25f730104e9 --- /dev/null +++ b/packages/wasm-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js @@ -0,0 +1,135 @@ +const calculateStateTransitionFeeFactory = require('@dashevo/dpp/lib/stateTransition/fee/calculateStateTransitionFeeFactory'); + +const getIdentityCreateTransitionFixture = require('@dashevo/dpp/lib/test/fixtures/getIdentityCreateTransitionFixture'); +const IdentityPublicKey = require('@dashevo/dpp/lib/identity/IdentityPublicKey'); + +const createStateRepositoryMock = require('@dashevo/dpp/lib/test/mocks/createStateRepositoryMock'); + +describe('calculateStateTransitionFeeFactory', () => { + let stateTransition; + let calculateStateTransitionFee; + let stateRepositoryMock; + let calculateOperationFeesMock; + + beforeEach(async function beforeEach() { + const privateKey = 'af432c476f65211f45f48f1d42c9c0b497e56696aa1736b40544ef1a496af837'; + + stateTransition = getIdentityCreateTransitionFixture(); + await stateTransition.signByPrivateKey(privateKey, IdentityPublicKey.TYPES.ECDSA_SECP256K1); + + stateRepositoryMock = createStateRepositoryMock(this.sinonSandbox); + + calculateOperationFeesMock = this.sinonSandbox.stub(); + + calculateStateTransitionFee = calculateStateTransitionFeeFactory( + stateRepositoryMock, + calculateOperationFeesMock, + ); + }); + + it('should throw an error if more than two identities have refunds', async () => { + const calculatedOperationsFeesResult = { + storageFee: 10000, + processingFee: 1000, + feeRefunds: [ + { + identifier: stateTransition.getOwnerId().toBuffer(), + creditsPerEpoch: { + 0: -100, + 1: -50, + }, + }, + { + identifier: stateTransition.getOwnerId().toBuffer(), + creditsPerEpoch: { + 1: -50, + }, + }, + ], + }; + + calculateOperationFeesMock.returns(calculatedOperationsFeesResult); + + try { + await calculateStateTransitionFee(stateTransition); + + expect.fail('should fail'); + } catch (e) { + expect(e.message).to.equals('State Transition removed bytes from several identities that is not defined by protocol'); + } + }); + + it('should throw an error if refunded identity is not owner of state transition', async () => { + const calculatedOperationsFeesResult = { + storageFee: 10000, + processingFee: 1000, + feeRefunds: [ + { + identifier: Buffer.alloc(32, 1), + creditsPerEpoch: { + 0: -100, + 1: -50, + }, + }, + ], + }; + + calculateOperationFeesMock.returns(calculatedOperationsFeesResult); + + try { + await calculateStateTransitionFee(stateTransition); + + expect.fail('should fail'); + } catch (e) { + expect(e.message).to.equals('State Transition removed bytes from different identity'); + } + }); + + it('should calculate fee based on executed operations', async () => { + const storageFee = 10000; + const processingFee = 1000; + const feeRefundsSum = -450 - 995 - 400 - 400; + const total = storageFee + processingFee + feeRefundsSum; + + const calculatedOperationsFeesResult = { + storageFee, + processingFee, + feeRefunds: [ + { + identifier: stateTransition.getOwnerId().toBuffer(), + creditsPerEpoch: { + 0: -1000, + 1: -500, + }, + }, + ], + }; + + calculateOperationFeesMock.returns(calculatedOperationsFeesResult); + + stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers + .onCall(0).resolves([-995, 400]); + + stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers + .onCall(1).resolves([-450, 400]); + + const result = await calculateStateTransitionFee(stateTransition); + + expect(result).to.equal(total); + + expect(stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers) + .to.have.been.calledWithExactly(-1000, 0); + + expect(stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers) + .to.have.been.calledWithExactly(-500, 1); + + const lastCalculatedFeeDetails = stateTransition.getExecutionContext() + .getLastCalculatedFeeDetails(); + + expect(lastCalculatedFeeDetails).to.be.deep.equal({ + ...calculatedOperationsFeesResult, + feeRefundsSum, + total, + }); + }); +}); diff --git a/packages/wasm-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/applyDocumentsBatchTransitionFactory.spec.js b/packages/wasm-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/applyDocumentsBatchTransitionFactory.spec.js index 26de93e6a46..269e8ab12c4 100644 --- a/packages/wasm-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/applyDocumentsBatchTransitionFactory.spec.js +++ b/packages/wasm-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/applyDocumentsBatchTransitionFactory.spec.js @@ -66,7 +66,7 @@ describe('applyDocumentsBatchTransitionFactory', () => { stateRepositoryMock = createStateRepositoryMock(this.sinonSandbox); stateRepositoryMock.fetchDataContract.resolves(dataContract); - stateRepositoryMock.fetchLatestPlatformBlockTime.returns(blockTimeMs); + stateRepositoryMock.fetchLatestPlatformBlockTime.resolves(blockTimeMs); fetchDocumentsMock = this.sinonSandbox.stub(); fetchDocumentsMock.resolves([ diff --git a/packages/wasm-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/validation/state/validateDocumentsBatchTransitionStateFactory.spec.js b/packages/wasm-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/validation/state/validateDocumentsBatchTransitionStateFactory.spec.js index bb85c5793d5..9dbd1474f86 100644 --- a/packages/wasm-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/validation/state/validateDocumentsBatchTransitionStateFactory.spec.js +++ b/packages/wasm-dpp/test/unit/document/stateTransition/DocumetsBatchTransition/validation/state/validateDocumentsBatchTransitionStateFactory.spec.js @@ -68,7 +68,7 @@ describe('validateDocumentsBatchTransitionStateFactory', () => { blockTime = Date.now(); - stateRepositoryMock.fetchLatestPlatformBlockTime.returns(blockTime); + stateRepositoryMock.fetchLatestPlatformBlockTime.resolves(blockTime); fetchDocumentsMock = this.sinonSandbox.stub().resolves([]); From ddd897121c8692355e2e0c6883d012f5e8644f63 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 21 Dec 2022 00:32:30 +0800 Subject: [PATCH 28/54] tests: remove calculateFees method --- .../AbstractStateTransitionIdentitySigned.spec.js | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/wasm-dpp/test/integration/stateTransition/AbstractStateTransitionIdentitySigned.spec.js b/packages/wasm-dpp/test/integration/stateTransition/AbstractStateTransitionIdentitySigned.spec.js index 26f393f4722..ce7ac9acbd4 100644 --- a/packages/wasm-dpp/test/integration/stateTransition/AbstractStateTransitionIdentitySigned.spec.js +++ b/packages/wasm-dpp/test/integration/stateTransition/AbstractStateTransitionIdentitySigned.spec.js @@ -493,14 +493,4 @@ describe('AbstractStateTransitionIdentitySigned', () => { expect(stateTransition.signaturePublicKeyId).to.equal(signaturePublicKeyId); }); }); - - describe('#calculateFee', () => { - it('should calculate fee', () => { - const result = stateTransition.calculateFee(); - - const fee = calculateStateTransitionFee(stateTransition); - - expect(result).to.equal(fee); - }); - }); }); From 2823d1c9bf660c529cef07bcb822a5f629621402 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 21 Dec 2022 17:46:37 +0800 Subject: [PATCH 29/54] fix: calculateStorageFeeDistributionAmountAndLeftovers --- packages/rs-drive-nodejs/src/fee/mod.rs | 2 +- packages/rs-drive-nodejs/test/Drive.spec.js | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/rs-drive-nodejs/src/fee/mod.rs b/packages/rs-drive-nodejs/src/fee/mod.rs index 8762a89dc1e..02b48f5887a 100644 --- a/packages/rs-drive-nodejs/src/fee/mod.rs +++ b/packages/rs-drive-nodejs/src/fee/mod.rs @@ -34,7 +34,7 @@ pub fn js_calculate_storage_fee_distribution_amount_and_leftovers( let js_leftovers = cx.number(leftovers as f64); js_array.set(&mut cx, 0, js_amount)?; - js_array.set(&mut cx, 0, js_leftovers)?; + js_array.set(&mut cx, 1, js_leftovers)?; Ok(js_array) } diff --git a/packages/rs-drive-nodejs/test/Drive.spec.js b/packages/rs-drive-nodejs/test/Drive.spec.js index 450433282dd..8d7f3ed96a8 100644 --- a/packages/rs-drive-nodejs/test/Drive.spec.js +++ b/packages/rs-drive-nodejs/test/Drive.spec.js @@ -622,4 +622,18 @@ describe('Drive', () => { }); }); }); + + describe('.calculateStorageFeeDistributionAmountAndLeftovers', () => { + it('should calculate amount and leftovers', () => { + const result = Drive.calculateStorageFeeDistributionAmountAndLeftovers(1000, 1, 2); + + expect(result).to.be.an.instanceOf(Array); + expect(result).to.be.lengthOf(2); + + const [amount, leftovers] = result; + + expect(amount).to.equals(558); + expect(leftovers).to.equals(440); + }); + }) }); From 5937029be05c09da3dfbc0010342016bc76c30b1 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 21 Dec 2022 18:03:43 +0800 Subject: [PATCH 30/54] style: add semi --- packages/rs-drive-nodejs/test/Drive.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rs-drive-nodejs/test/Drive.spec.js b/packages/rs-drive-nodejs/test/Drive.spec.js index 8d7f3ed96a8..cd994839510 100644 --- a/packages/rs-drive-nodejs/test/Drive.spec.js +++ b/packages/rs-drive-nodejs/test/Drive.spec.js @@ -635,5 +635,5 @@ describe('Drive', () => { expect(amount).to.equals(558); expect(leftovers).to.equals(440); }); - }) + }); }); From f8b99fc80855633f468fcdf6f6074213a0157237 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 21 Dec 2022 20:58:17 +0700 Subject: [PATCH 31/54] temp --- Cargo.lock | 385 ++++++------------ packages/rs-drive/Cargo.toml | 6 +- .../drive/fee_pools/pending_epoch_updates.rs | 14 +- .../rs-drive/src/fee/epoch/distribution.rs | 2 +- packages/rs-drive/src/query/mod.rs | 5 +- 5 files changed, 144 insertions(+), 268 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 837f286feb1..40f578f256b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.17.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" dependencies = [ "gimli", ] @@ -55,9 +55,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" +checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] name = "array_tool" @@ -79,9 +79,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "async-trait" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" +checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" dependencies = [ "proc-macro2", "quote", @@ -94,20 +94,11 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] -[[package]] -name = "autocfg" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" -dependencies = [ - "autocfg 1.1.0", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -116,9 +107,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" dependencies = [ "addr2line", "cc", @@ -200,6 +191,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bitcoin_hashes" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90064b8dee6815a6470d60bad07bbbaee885c0e12d04177138fa3291a01b7bc4" + [[package]] name = "bitflags" version = "1.3.2" @@ -261,7 +258,7 @@ dependencies = [ "group", "hkdf", "pairing", - "rand_core 0.6.4", + "rand_core", "rayon", "sha2 0.9.9", "subtle", @@ -278,7 +275,7 @@ dependencies = [ "ff", "group", "pairing", - "rand_core 0.6.4", + "rand_core", "subtle", ] @@ -427,7 +424,7 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" dependencies = [ "camino", "cargo-platform", - "semver 1.0.14", + "semver 1.0.16", "serde", "serde_json", ] @@ -440,9 +437,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.77" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" +checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" dependencies = [ "jobserver", ] @@ -532,15 +529,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags", -] - [[package]] name = "codespan-reporting" version = "0.11.1" @@ -586,7 +574,7 @@ dependencies = [ [[package]] name = "costs" version = "0.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=6864631af1042e2f61ea9533ec72d95204df41f0#6864631af1042e2f61ea9533ec72d95204df41f0" +source = "git+https://github.com/dashpay/grovedb?branch=develop#3ef80bcd381d42d089681b1297e1137f4f5dbb67" dependencies = [ "integer-encoding", "intmap", @@ -665,7 +653,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ - "autocfg 1.1.0", + "autocfg", "cfg-if 0.1.10", "crossbeam-utils 0.7.2", "lazy_static", @@ -680,7 +668,7 @@ version = "0.9.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" dependencies = [ - "autocfg 1.1.0", + "autocfg", "cfg-if 1.0.0", "crossbeam-utils 0.8.14", "memoffset 0.7.1", @@ -693,7 +681,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ - "autocfg 1.1.0", + "autocfg", "cfg-if 0.1.10", "lazy_static", ] @@ -751,9 +739,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.83" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf" +checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" dependencies = [ "cc", "cxxbridge-flags", @@ -763,9 +751,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.83" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39" +checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" dependencies = [ "cc", "codespan-reporting", @@ -778,15 +766,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.83" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12" +checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" [[package]] name = "cxxbridge-macro" -version = "1.0.83" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6" +checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" dependencies = [ "proc-macro2", "quote", @@ -796,11 +784,11 @@ dependencies = [ [[package]] name = "dashcore" version = "0.29.1" -source = "git+https://github.com/dashevo/rust-dashcore?branch=master#6092d584487918c5330fec93152e4df06a714067" +source = "git+https://github.com/dashevo/rust-dashcore?branch=master#0cf3939e25a66b3cfdb5ba354fb4437d6b0d9d03" dependencies = [ "anyhow", "bech32", - "bitcoin_hashes", + "bitcoin_hashes 0.10.0", "core2", "hashbrown 0.8.2", "hex", @@ -812,11 +800,11 @@ dependencies = [ [[package]] name = "dashcore" version = "0.29.1" -source = "git+https://github.com/dashpay/rust-dashcore?branch=master#6092d584487918c5330fec93152e4df06a714067" +source = "git+https://github.com/dashpay/rust-dashcore?branch=master#0cf3939e25a66b3cfdb5ba354fb4437d6b0d9d03" dependencies = [ "anyhow", "bech32", - "bitcoin_hashes", + "bitcoin_hashes 0.10.0", "core2", "hashbrown 0.8.2", "hex", @@ -882,7 +870,7 @@ dependencies = [ "log", "mockall", "num_enum", - "rand 0.8.5", + "rand", "regex", "serde", "serde-big-array", @@ -919,7 +907,7 @@ dependencies = [ "itertools", "moka", "nohash-hasher", - "rand 0.8.5", + "rand", "rand_distr", "rust_decimal", "rust_decimal_macros", @@ -942,7 +930,7 @@ dependencies = [ "dashcore 0.29.1 (git+https://github.com/dashpay/rust-dashcore?branch=master)", "drive", "hex", - "rand 0.8.5", + "rand", "rust_decimal", "rust_decimal_macros", "serde", @@ -990,18 +978,18 @@ checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" [[package]] name = "enum-map" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5a56d54c8dd9b3ad34752ed197a4eb2a6601bc010808eb097a04a58ae4c43e1" +checksum = "50c25992259941eb7e57b936157961b217a4fc8597829ddef0596d6c3cd86e1a" dependencies = [ "enum-map-derive", ] [[package]] name = "enum-map-derive" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9045e2676cd5af83c3b167d917b0a5c90a4d8e266e2683d6631b235c457fc27" +checksum = "2a4da76b3b6116d758c7ba93f7ec6a35d2e2cf24feda76c6e38a375f4d5c59f2" dependencies = [ "proc-macro2", "quote", @@ -1078,7 +1066,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" dependencies = [ "bitvec", - "rand_core 0.6.4", + "rand_core", "subtle", ] @@ -1122,12 +1110,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - [[package]] name = "funty" version = "2.0.0" @@ -1248,9 +1230,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.26.2" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" +checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793" [[package]] name = "glob" @@ -1265,14 +1247,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" dependencies = [ "ff", - "rand_core 0.6.4", + "rand_core", "subtle", ] [[package]] name = "grovedb" version = "0.7.1" -source = "git+https://github.com/dashpay/grovedb?rev=6864631af1042e2f61ea9533ec72d95204df41f0#6864631af1042e2f61ea9533ec72d95204df41f0" +source = "git+https://github.com/dashpay/grovedb?branch=develop#3ef80bcd381d42d089681b1297e1137f4f5dbb67" dependencies = [ "bincode", "costs", @@ -1304,7 +1286,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b62f79061a0bc2e046024cb7ba44b08419ed238ecbd9adbd787434b9e8c25" dependencies = [ "ahash 0.3.8", - "autocfg 1.1.0", + "autocfg", ] [[package]] @@ -1340,6 +1322,15 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + [[package]] name = "hex" version = "0.4.3" @@ -1412,7 +1403,7 @@ version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ - "autocfg 1.1.0", + "autocfg", "hashbrown 0.12.3", ] @@ -1466,9 +1457,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" [[package]] name = "jemalloc-sys" @@ -1544,7 +1535,7 @@ dependencies = [ "fancy-regex", "fraction", "iso8601", - "itoa 1.0.4", + "itoa 1.0.5", "lazy_static", "memchr", "num-cmp", @@ -1630,9 +1621,9 @@ dependencies = [ [[package]] name = "link-cplusplus" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" dependencies = [ "cc", ] @@ -1643,7 +1634,7 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ - "autocfg 1.1.0", + "autocfg", "scopeguard", ] @@ -1683,7 +1674,7 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" dependencies = [ - "autocfg 1.1.0", + "autocfg", ] [[package]] @@ -1692,13 +1683,13 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" dependencies = [ - "autocfg 1.1.0", + "autocfg", ] [[package]] name = "merk" version = "0.7.1" -source = "git+https://github.com/dashpay/grovedb?rev=6864631af1042e2f61ea9533ec72d95204df41f0#6864631af1042e2f61ea9533ec72d95204df41f0" +source = "git+https://github.com/dashpay/grovedb?branch=develop#3ef80bcd381d42d089681b1297e1137f4f5dbb67" dependencies = [ "blake3", "byteorder", @@ -1711,7 +1702,7 @@ dependencies = [ "integer-encoding", "jemallocator", "num_cpus", - "rand 0.8.5", + "rand", "storage", "thiserror", "time 0.3.17", @@ -1732,9 +1723,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.5.4" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" dependencies = [ "adler", ] @@ -1897,7 +1888,7 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-integer", "num-traits", ] @@ -1908,7 +1899,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-integer", "num-traits", ] @@ -1925,7 +1916,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-traits", ] @@ -1944,7 +1935,7 @@ version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-traits", ] @@ -1954,7 +1945,7 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-integer", "num-traits", ] @@ -1965,7 +1956,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-bigint 0.2.6", "num-integer", "num-traits", @@ -1977,7 +1968,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" dependencies = [ - "autocfg 1.1.0", + "autocfg", "num-bigint 0.4.3", "num-integer", "num-traits", @@ -1989,17 +1980,17 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ - "autocfg 1.1.0", + "autocfg", "libm", ] [[package]] name = "num_cpus" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] @@ -2026,9 +2017,9 @@ dependencies = [ [[package]] name = "object" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +checksum = "239da7f290cfa979f43f85a8efeee9a8a76d0827c356d37f9d3d7254d6b537fb" dependencies = [ "memchr", ] @@ -2223,9 +2214,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" dependencies = [ "unicode-ident", ] @@ -2279,9 +2270,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -2292,25 +2283,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" -[[package]] -name = "rand" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -dependencies = [ - "autocfg 0.1.8", - "libc", - "rand_chacha 0.1.1", - "rand_core 0.4.2", - "rand_hc", - "rand_isaac", - "rand_jitter", - "rand_os", - "rand_pcg", - "rand_xorshift", - "winapi", -] - [[package]] name = "rand" version = "0.8.5" @@ -2318,18 +2290,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.3.1", + "rand_chacha", + "rand_core", ] [[package]] @@ -2339,24 +2301,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", + "rand_core", ] -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - [[package]] name = "rand_core" version = "0.6.4" @@ -2373,69 +2320,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand 0.8.5", -] - -[[package]] -name = "rand_hc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_isaac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_jitter" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" -dependencies = [ - "libc", - "rand_core 0.4.2", - "winapi", -] - -[[package]] -name = "rand_os" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" -dependencies = [ - "cloudabi", - "fuchsia-cprng", - "libc", - "rand_core 0.4.2", - "rdrand", - "winapi", -] - -[[package]] -name = "rand_pcg" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.4.2", -] - -[[package]] -name = "rand_xorshift" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -dependencies = [ - "rand_core 0.3.1", + "rand", ] [[package]] @@ -2469,15 +2354,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "redox_syscall" version = "0.2.16" @@ -2585,7 +2461,7 @@ dependencies = [ "byteorder", "bytes", "num-traits", - "rand 0.8.5", + "rand", "rkyv", "serde", "serde_json", @@ -2615,15 +2491,15 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustversion" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" +checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" [[package]] name = "ryu" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] name = "same-file" @@ -2651,9 +2527,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" +checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" [[package]] name = "seahash" @@ -2663,18 +2539,21 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "secp256k1" -version = "0.22.1" -source = "git+https://github.com/rust-bitcoin/rust-secp256k1?rev=f7cae46fc7733522cb84ef1b1ee1d1ed0cec2fd3#f7cae46fc7733522cb84ef1b1ee1d1ed0cec2fd3" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9512ffd81e3a3503ed401f79c33168b9148c75038956039166cd750eaa037c3" dependencies = [ - "rand 0.6.5", + "bitcoin_hashes 0.11.0", + "rand", "secp256k1-sys", "serde", ] [[package]] name = "secp256k1-sys" -version = "0.5.0" -source = "git+https://github.com/rust-bitcoin/rust-secp256k1?rev=f7cae46fc7733522cb84ef1b1ee1d1ed0cec2fd3#f7cae46fc7733522cb84ef1b1ee1d1ed0cec2fd3" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83080e2c2fc1006e625be82e5d1eb6a43b7fd9578b617fcc55814daf286bba4b" dependencies = [ "cc", ] @@ -2690,9 +2569,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" +checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" dependencies = [ "serde", ] @@ -2705,9 +2584,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91" +checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" dependencies = [ "serde_derive", ] @@ -2744,9 +2623,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e" +checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" dependencies = [ "proc-macro2", "quote", @@ -2755,21 +2634,21 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" dependencies = [ "indexmap", - "itoa 1.0.4", + "itoa 1.0.5", "ryu", "serde", ] [[package]] name = "serde_repr" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe39d9fbb0ebf5eb2c7cb7e2a47e4f462fad1379f1166b8ae49ad9eae89a7ca" +checksum = "9a5ec9fa74a20ebbe5d9ac23dac1fc96ba0ecfe9f50f2843b52e537b10fbcb4e" dependencies = [ "proc-macro2", "quote", @@ -2836,7 +2715,7 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" dependencies = [ - "autocfg 1.1.0", + "autocfg", ] [[package]] @@ -2867,7 +2746,7 @@ dependencies = [ [[package]] name = "storage" version = "0.2.0" -source = "git+https://github.com/dashpay/grovedb?rev=6864631af1042e2f61ea9533ec72d95204df41f0#6864631af1042e2f61ea9533ec72d95204df41f0" +source = "git+https://github.com/dashpay/grovedb?branch=develop#3ef80bcd381d42d089681b1297e1137f4f5dbb67" dependencies = [ "blake3", "costs", @@ -2912,9 +2791,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -3018,18 +2897,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", @@ -3104,7 +2983,7 @@ version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" dependencies = [ - "autocfg 1.1.0", + "autocfg", "bytes", "libc", "memchr", @@ -3131,9 +3010,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" dependencies = [ "serde", ] @@ -3176,9 +3055,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-normalization" @@ -3254,7 +3133,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "visualize" version = "0.1.0" -source = "git+https://github.com/dashpay/grovedb?rev=6864631af1042e2f61ea9533ec72d95204df41f0#6864631af1042e2f61ea9533ec72d95204df41f0" +source = "git+https://github.com/dashpay/grovedb?branch=develop#3ef80bcd381d42d089681b1297e1137f4f5dbb67" dependencies = [ "hex", "itertools", diff --git a/packages/rs-drive/Cargo.toml b/packages/rs-drive/Cargo.toml index a69b34ac68b..634bd0a466a 100644 --- a/packages/rs-drive/Cargo.toml +++ b/packages/rs-drive/Cargo.toml @@ -36,15 +36,15 @@ rust_decimal_macros = "1.25.0" [dependencies.grovedb] git = "https://github.com/dashpay/grovedb" -rev = "6864631af1042e2f61ea9533ec72d95204df41f0" +branch = "develop" [dependencies.storage] git = "https://github.com/dashpay/grovedb" -rev = "6864631af1042e2f61ea9533ec72d95204df41f0" +branch = "develop" [dependencies.costs] git = "https://github.com/dashpay/grovedb" -rev = "6864631af1042e2f61ea9533ec72d95204df41f0" +branch = "develop" [dev-dependencies] criterion = "0.3.5" diff --git a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs index 0c3d61f36b7..e1d8dff5021 100644 --- a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs +++ b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs @@ -76,15 +76,11 @@ impl Drive { )) })?); - let Element::SumItem(..) = element else { - return Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates credits must be sum items"))); - }; - - let credits: SignedCredits = element.sum_value().ok_or( - Error::Drive(DriveError::CorruptedCodeExecution("pending updates credits must have value") - ))?; - - Ok((epoch_index, credits)) + if let Element::SumItem(credits, _) = element { + Ok((epoch_index, credits)) + } else { + Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates credits must be sum items"))) + } }).collect::>() } diff --git a/packages/rs-drive/src/fee/epoch/distribution.rs b/packages/rs-drive/src/fee/epoch/distribution.rs index b6a568f49ed..9420bd8044c 100644 --- a/packages/rs-drive/src/fee/epoch/distribution.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -484,7 +484,7 @@ mod tests { #[test] fn should_calculate_amount_and_leftovers() { - let storage_fee = 10000; + let storage_fee = 1000; let (amount, leftovers) = calculate_storage_fee_distribution_amount_and_leftovers( storage_fee, diff --git a/packages/rs-drive/src/query/mod.rs b/packages/rs-drive/src/query/mod.rs index a678192791a..e75499a8e70 100644 --- a/packages/rs-drive/src/query/mod.rs +++ b/packages/rs-drive/src/query/mod.rs @@ -1256,8 +1256,9 @@ impl<'a> DriveQuery<'a> { for (_, value, _) in key_value_elements.iter_mut() { let element = Element::deserialize(value).unwrap(); match element { - Element::Item(val, _) | Element::SumItem(val, _) => values.push(val), - Element::Tree(..) | Element::SumTree(..) | Element::Reference(..) => { + Element::Item(val, _) => values.push(val), + | Element::SumItem(val, _) => values.push(val.to_be_bytes().to_vec()), + Element::Tree(..) | Element::SumTree(..) | Element::Reference(..) => { return Err(Error::GroveDB(GroveError::InvalidQuery( "path query should only point to items: got trees", ))); From a01f86ed87a2e79a94f3ba1a5c447281e1209035 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 21 Dec 2022 22:03:39 +0700 Subject: [PATCH 32/54] small fixes --- packages/rs-drive-abci/src/abci/handlers.rs | 6 +- packages/rs-drive-abci/src/abci/messages.rs | 4 +- .../fee_pools/distribute_storage_pool.rs | 17 +++--- .../execution/fee_pools/process_block_fees.rs | 8 +-- packages/rs-drive-nodejs/src/lib.rs | 4 +- packages/rs-drive/src/drive/fee_pools/mod.rs | 10 ++-- .../drive/fee_pools/pending_epoch_updates.rs | 56 ++++++++---------- packages/rs-drive/src/fee/credits.rs | 31 +++++----- .../rs-drive/src/fee/epoch/distribution.rs | 57 +++++++++++-------- packages/rs-drive/src/fee/result/refunds.rs | 26 ++++----- .../rs-drive/tests/query_tests_history.rs | 6 +- 11 files changed, 110 insertions(+), 115 deletions(-) diff --git a/packages/rs-drive-abci/src/abci/handlers.rs b/packages/rs-drive-abci/src/abci/handlers.rs index 940bac27abd..1ef7ae3830a 100644 --- a/packages/rs-drive-abci/src/abci/handlers.rs +++ b/packages/rs-drive-abci/src/abci/handlers.rs @@ -185,7 +185,7 @@ mod tests { use chrono::{Duration, Utc}; use drive::common::helpers::identities::create_test_masternode_identities; use drive::drive::batch::GroveDbOpBatch; - use drive::fee::epoch::SignedCreditsPerEpoch; + use drive::fee::epoch::CreditsPerEpoch; use rust_decimal::prelude::ToPrimitive; use std::ops::Div; @@ -361,7 +361,7 @@ mod tests { fees: BlockFees { storage_fee: storage_fees_per_block, processing_fee: 1600, - fee_refunds: SignedCreditsPerEpoch::from_iter([(0, -100)]), + fee_refunds: CreditsPerEpoch::from_iter([(0, 100)]), }, }; @@ -517,7 +517,7 @@ mod tests { fees: BlockFees { storage_fee: storage_fees_per_block, processing_fee: 1600, - fee_refunds: SignedCreditsPerEpoch::from_iter([(0, -100)]), + fee_refunds: CreditsPerEpoch::from_iter([(0, 100)]), }, }; diff --git a/packages/rs-drive-abci/src/abci/messages.rs b/packages/rs-drive-abci/src/abci/messages.rs index 3ae8df54f78..a58c7428bd6 100644 --- a/packages/rs-drive-abci/src/abci/messages.rs +++ b/packages/rs-drive-abci/src/abci/messages.rs @@ -37,7 +37,7 @@ use crate::error::serialization::SerializationError; use crate::error::Error; use crate::execution::fee_pools::epoch::EpochInfo; use crate::execution::fee_pools::process_block_fees::ProcessedBlockFeesResult; -use drive::fee::epoch::SignedCreditsPerEpoch; +use drive::fee::epoch::CreditsPerEpoch; use serde::{Deserialize, Serialize}; /// A struct for handling chain initialization requests @@ -94,7 +94,7 @@ pub struct BlockFees { /// Storage fee pub storage_fee: u64, /// Fee refunds - pub fee_refunds: SignedCreditsPerEpoch, + pub fee_refunds: CreditsPerEpoch, } impl BlockFees { diff --git a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs index 5260c19c9a7..26dac161a05 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs @@ -39,7 +39,7 @@ use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; use drive::fee::credits::{Creditable, Credits}; use drive::fee::epoch::distribution::distribute_storage_fee_to_epochs_collection; -use drive::fee::epoch::{EpochIndex, SignedCreditsPerEpoch}; +use drive::fee::epoch::{CreditsPerEpoch, EpochIndex, SignedCreditsPerEpoch}; use drive::grovedb::TransactionArg; /// Leftovers in result of divisions and rounding after storage fee distribution to epochs @@ -63,7 +63,8 @@ impl Platform { // Distribute from storage distribution pool let leftovers = distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, - storage_distribution_fees.to_signed()?, + storage_distribution_fees, + false, current_epoch_index, None, )?; @@ -76,6 +77,7 @@ impl Platform { distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, credits, + true, epoch_index, Some(current_epoch_index), )?; @@ -167,12 +169,8 @@ mod tests { // Add pending refunds - let refunds = SignedCreditsPerEpoch::from_iter([ - (0, -10000), - (1, -15000), - (2, -20000), - (3, -25000), - ]); + let refunds = + CreditsPerEpoch::from_iter([(0, 10000), (1, 15000), (2, 20000), (3, 25000)]); add_update_pending_epoch_storage_pool_update_operations(&mut batch, refunds.clone()) .expect("should update pending updates"); @@ -219,12 +217,13 @@ mod tests { let leftovers = distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, credits, + false, epoch_index, None, ) .expect("should distribute refunds"); - let already_paid_epochs = current_epoch_index as i64 - epoch_index as i64; + let already_paid_epochs = current_epoch_index as u64 - epoch_index as u64; let already_paid_credits = if already_paid_epochs > 0 { credits_per_epochs diff --git a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs index 00c31f74b3d..d590e620d6c 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs @@ -236,7 +236,7 @@ mod tests { mod helpers { use super::*; - use drive::fee::epoch::SignedCreditsPerEpoch; + use drive::fee::epoch::CreditsPerEpoch; /// Process and validate an epoch change pub fn process_and_validate_epoch_change( @@ -293,7 +293,7 @@ mod tests { let block_fees = BlockFees { storage_fee: 1000000000, processing_fee: 10000, - fee_refunds: SignedCreditsPerEpoch::from_iter([(0, -10000)]), + fee_refunds: CreditsPerEpoch::from_iter([(0, -10000)]), }; let mut batch = GroveDbOpBatch::new(); @@ -427,7 +427,7 @@ mod tests { mod helpers { use super::*; - use drive::fee::epoch::SignedCreditsPerEpoch; + use drive::fee::epoch::CreditsPerEpoch; /// Process and validate block fees pub fn process_and_validate_block_fees( @@ -458,7 +458,7 @@ mod tests { let block_fees = BlockFees { storage_fee: 1000, processing_fee: 10000, - fee_refunds: SignedCreditsPerEpoch::from_iter([(epoch_index, -100)]), + fee_refunds: CreditsPerEpoch::from_iter([(epoch_index, -100)]), }; let distribute_storage_pool_result = platform diff --git a/packages/rs-drive-nodejs/src/lib.rs b/packages/rs-drive-nodejs/src/lib.rs index 1201e5e024b..c087ee909ca 100644 --- a/packages/rs-drive-nodejs/src/lib.rs +++ b/packages/rs-drive-nodejs/src/lib.rs @@ -11,7 +11,7 @@ use drive::drive::config::DriveConfig; use drive::drive::flags::StorageFlags; use drive::error::Error; use drive::fee::credits::SignedCredits; -use drive::fee::epoch::SignedCreditsPerEpoch; +use drive::fee::epoch::CreditsPerEpoch; use drive::fee_pools::epochs::Epoch; use drive::grovedb::{PathQuery, Transaction}; use drive::query::TransactionArg; @@ -2040,7 +2040,7 @@ impl PlatformWrapper { let js_fee_refunds: Handle = js_fees.get(&mut cx, "feeRefunds")?; - let mut fee_refunds: SignedCreditsPerEpoch = Default::default(); + let mut fee_refunds: CreditsPerEpoch = Default::default(); for js_epoch_index_value in js_fee_refunds .get_own_property_names(&mut cx)? diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index 2d1df94f278..143898aa554 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -31,8 +31,8 @@ use crate::drive::batch::GroveDbOpBatch; use crate::drive::{Drive, RootTree}; use crate::error::drive::DriveError; use crate::error::Error; -use crate::fee::credits::{Creditable, SignedCredits}; -use crate::fee::epoch::SignedCreditsPerEpoch; +use crate::fee::credits::{Creditable, Credits, SignedCredits}; +use crate::fee::epoch::{CreditsPerEpoch, SignedCreditsPerEpoch}; use crate::fee::get_overflow_error; use crate::fee_pools::epochs::epoch_key_constants::KEY_POOL_STORAGE_FEES; use crate::fee_pools::epochs::{paths, Epoch}; @@ -137,7 +137,7 @@ impl Drive { .get(i) .map_or(0u64.to_vec_bytes(), |c| c.to_owned()); - let existing_storage_fee = SignedCredits::from_vec_bytes(encoded_existing_storage_fee)?; + let existing_storage_fee = Credits::from_vec_bytes(encoded_existing_storage_fee)?.to_signed()?; let credits_to_update = existing_storage_fee.checked_add(credits).ok_or_else(|| { get_overflow_error("can't add credits to existing epoch pool storage fee") @@ -217,8 +217,8 @@ mod tests { .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - let credits_to_epochs: SignedCreditsPerEpoch = (GENESIS_EPOCH_INDEX..TO_EPOCH_INDEX) - .map(|epoch_index| (epoch_index, epoch_index as SignedCredits)) + let credits_to_epochs: SignedCreditsPerEpoch = (GENESIS_EPOCH_INDEX..TO_EPOCH_INDEX).enumerate() + .map(|(credits, epoch_index)| (epoch_index, credits as SignedCredits)) .collect(); let mut batch = GroveDbOpBatch::new(); diff --git a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs index e1d8dff5021..7fdbf60a232 100644 --- a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs +++ b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs @@ -43,7 +43,7 @@ use crate::drive::Drive; use crate::error::drive::DriveError; use crate::error::Error; use crate::fee::credits::{Creditable, SignedCredits}; -use crate::fee::epoch::SignedCreditsPerEpoch; +use crate::fee::epoch::CreditsPerEpoch; use crate::fee::get_overflow_error; use grovedb::query_result_type::QueryResultType; use grovedb::{Element, PathQuery, Query, TransactionArg}; @@ -53,7 +53,7 @@ impl Drive { pub fn fetch_pending_updates( &self, transaction: TransactionArg, - ) -> Result { + ) -> Result { let mut query = Query::new(); query.insert_all(); @@ -77,20 +77,20 @@ impl Drive { })?); if let Element::SumItem(credits, _) = element { - Ok((epoch_index, credits)) + Ok((epoch_index, credits.to_unsigned())) } else { Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates credits must be sum items"))) } - }).collect::>() + }).collect::>() } /// Fetches existing pending epoch pool updates using specified epochs /// and returns merged result pub fn fetch_and_merge_with_existing_pending_epoch_storage_pool_updates( &self, - mut credits_per_epoch: SignedCreditsPerEpoch, + mut credits_per_epoch: CreditsPerEpoch, transaction: TransactionArg, - ) -> Result { + ) -> Result { if credits_per_epoch.is_empty() { return Ok(credits_per_epoch); } @@ -123,26 +123,18 @@ impl Drive { )) })?); - let Some(credits_to_update) = credits_per_epoch.get(&epoch_index) else { - return Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates should contain fetched epochs"))); - }; + let credits_to_update = credits_per_epoch.get(&epoch_index).ok_or( + Error::Drive(DriveError::CorruptedCodeExecution("pending updates should contain fetched epochs")))?; - let Element::SumItem(..) = element else { - return Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates credits must be sum items"))); - }; - - let existing_credits: SignedCredits = - element - .sum_value() - .ok_or(Error::Drive(DriveError::CorruptedCodeExecution( - "pending updates credits must have value", - )))?; - - let result_credits = credits_to_update - .checked_add(existing_credits) - .ok_or_else(|| get_overflow_error("pending updates credits overflow"))?; + if let Element::SumItem(credits, _) = element { + let result_credits = credits.to_unsigned() + .checked_add(*credits_to_update) + .ok_or_else(|| get_overflow_error("pending updates credits overflow"))?; - credits_per_epoch.insert(epoch_index, result_credits); + credits_per_epoch.insert(epoch_index, result_credits); + } else { + return Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates credits must be sum items"))); + } } Ok(credits_per_epoch) @@ -152,7 +144,7 @@ impl Drive { pub fn add_delete_pending_epoch_storage_pool_updates_except_specified_operations( &self, batch: &mut GroveDbOpBatch, - credits_per_epoch: &SignedCreditsPerEpoch, + credits_per_epoch: &CreditsPerEpoch, transaction: TransactionArg, ) -> Result<(), Error> { // TODO: Replace with key iterator @@ -192,7 +184,7 @@ impl Drive { /// Adds GroveDB batch operations to update pending epoch storage pool updates pub fn add_update_pending_epoch_storage_pool_update_operations( batch: &mut GroveDbOpBatch, - credits_per_epoch: SignedCreditsPerEpoch, + credits_per_epoch: CreditsPerEpoch, ) -> Result<(), Error> { for (epoch_index, credits) in credits_per_epoch { let epoch_index_key = epoch_index.to_be_bytes().to_vec(); @@ -222,7 +214,7 @@ mod tests { // Store initial set of pending updates let initial_pending_updates = - SignedCreditsPerEpoch::from_iter([(1, 15), (3, 25), (7, 95), (9, 100), (12, 120)]); + CreditsPerEpoch::from_iter([(1, 15), (3, 25), (7, 95), (9, 100), (12, 120)]); let mut batch = GroveDbOpBatch::new(); @@ -239,7 +231,7 @@ mod tests { // Fetch and merge let new_pending_updates = - SignedCreditsPerEpoch::from_iter([(1, 15), (3, 25), (30, 195), (41, 150)]); + CreditsPerEpoch::from_iter([(1, 15), (3, 25), (30, 195), (41, 150)]); let updated_pending_updates = drive .fetch_and_merge_with_existing_pending_epoch_storage_pool_updates( @@ -249,7 +241,7 @@ mod tests { .expect("should fetch and merge pending updates"); let expected_pending_updates = - SignedCreditsPerEpoch::from_iter([(1, 30), (3, 50), (30, 195), (41, 150)]); + CreditsPerEpoch::from_iter([(1, 30), (3, 50), (30, 195), (41, 150)]); assert_eq!(updated_pending_updates, expected_pending_updates); } @@ -268,7 +260,7 @@ mod tests { // Store initial set of pending updates let initial_pending_updates = - SignedCreditsPerEpoch::from_iter([(1, 15), (3, 25), (7, 95), (9, 100), (12, 120)]); + CreditsPerEpoch::from_iter([(1, 15), (3, 25), (7, 95), (9, 100), (12, 120)]); let mut batch = GroveDbOpBatch::new(); @@ -284,7 +276,7 @@ mod tests { // Delete existing pending updates expect specified pending updates - let new_pending_updates = SignedCreditsPerEpoch::from_iter([(1, 15), (3, 25)]); + let new_pending_updates = CreditsPerEpoch::from_iter([(1, 15), (3, 25)]); let mut batch = GroveDbOpBatch::new(); @@ -297,7 +289,7 @@ mod tests { .expect("should fetch and merge pending updates"); let expected_pending_updates = - SignedCreditsPerEpoch::from_iter([(7, 95), (9, 100), (12, 120)]); + CreditsPerEpoch::from_iter([(7, 95), (9, 100), (12, 120)]); assert_eq!(batch.len(), expected_pending_updates.len()); diff --git a/packages/rs-drive/src/fee/credits.rs b/packages/rs-drive/src/fee/credits.rs index 7444d29d2dc..56a66fb9116 100644 --- a/packages/rs-drive/src/fee/credits.rs +++ b/packages/rs-drive/src/fee/credits.rs @@ -39,6 +39,7 @@ // TODO: Should be moved to DPP when integration is done +use integer_encoding::VarInt; use crate::error::drive::DriveError; use crate::error::Error; use crate::fee::get_overflow_error; @@ -52,7 +53,7 @@ pub type Credits = u64; pub type SignedCredits = i64; /// Maximum value of credits -pub const MAX: Credits = SignedCredits::MAX as Credits; +pub const MAX_CREDITS: Credits = SignedCredits::MAX as Credits; /// Trait for signed and unsigned credits pub trait Creditable: Into { @@ -80,17 +81,15 @@ impl Creditable for Credits { } fn from_vec_bytes(vec: Vec) -> Result { - Ok(Self::from_be_bytes(vec.as_slice().try_into().map_err( - |_| { - Error::Drive(DriveError::CorruptedSerialization( - "pending updates epoch index for must be u16", - )) - }, - )?)) + Self::decode_var(vec.as_slice()).map(|(n, s)| n).ok_or( + Error::Drive(DriveError::CorruptedSerialization( + "pending updates epoch index for must be u16", + )) + ) } fn to_vec_bytes(&self) -> Vec { - self.to_be_bytes().to_vec() + self.encode_var_vec() } } impl Creditable for SignedCredits { @@ -103,16 +102,14 @@ impl Creditable for SignedCredits { } fn from_vec_bytes(vec: Vec) -> Result { - Ok(Self::from_be_bytes(vec.as_slice().try_into().map_err( - |_| { - Error::Drive(DriveError::CorruptedSerialization( - "pending updates epoch index for must be u16", - )) - }, - )?)) + Self::decode_var(vec.as_slice()).map(|(n, s)| n).ok_or( + Error::Drive(DriveError::CorruptedSerialization( + "pending updates epoch index for must be u16", + )) + ) } fn to_vec_bytes(&self) -> Vec { - self.to_be_bytes().to_vec() + self.encode_var_vec() } } diff --git a/packages/rs-drive/src/fee/epoch/distribution.rs b/packages/rs-drive/src/fee/epoch/distribution.rs index 9420bd8044c..a715f1a109f 100644 --- a/packages/rs-drive/src/fee/epoch/distribution.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -38,10 +38,8 @@ use crate::error::fee::FeeError; use crate::error::Error; -use crate::fee::credits::SignedCredits; -use crate::fee::epoch::{ - EpochIndex, SignedCreditsPerEpoch, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS, -}; +use crate::fee::credits::{Creditable, Credits, SignedCredits}; +use crate::fee::epoch::{EpochIndex, CreditsPerEpoch, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS, SignedCreditsPerEpoch}; use crate::fee::get_overflow_error; use rust_decimal::prelude::*; use rust_decimal::Decimal; @@ -64,19 +62,20 @@ pub const FEE_DISTRIBUTION_TABLE: [Decimal; PERPETUAL_STORAGE_YEARS as usize] = dec!(0.00325), dec!(0.00275), dec!(0.00225), dec!(0.00175), dec!(0.00125), ]; -/// Distributes storage fees to epochs into `SignedCreditsPerEpoch` and returns leftovers +/// Distributes storage fees to epochs into `CreditsPerEpoch` and returns leftovers /// It skips epochs up to specified `skip_up_to_epoch_index` pub fn distribute_storage_fee_to_epochs_collection( credits_per_epochs: &mut SignedCreditsPerEpoch, - storage_fee: SignedCredits, + storage_fee: Credits, + remove_instead: bool, start_epoch_index: EpochIndex, - skip_up_to_epoch_index: Option, + skip_distribution_until_epoch_index: Option, ) -> Result { distribution_storage_fee_to_epochs_map( storage_fee, start_epoch_index, |epoch_index, epoch_fee_share| { - if let Some(skip_epoch_index) = skip_up_to_epoch_index { + if let Some(skip_epoch_index) = skip_distribution_until_epoch_index { if epoch_index < skip_epoch_index { return Ok(()); } @@ -84,11 +83,16 @@ pub fn distribute_storage_fee_to_epochs_collection( let epoch_credits: SignedCredits = credits_per_epochs .get(&epoch_index) - .map_or(0, |i| i.to_owned()); + .map_or(0, |i| *i); - let result_storage_fee: SignedCredits = epoch_credits - .checked_add(epoch_fee_share) - .ok_or_else(|| get_overflow_error("storage fees are not fitting in a u64"))?; + let result_storage_fee: SignedCredits = if !remove_instead { // we add + epoch_credits + .checked_add(epoch_fee_share.to_signed()?) + } else { + epoch_credits + .checked_sub(epoch_fee_share.to_signed()?) + + }.ok_or_else(|| get_overflow_error("storage fees are not fitting in a u64"))?; credits_per_epochs.insert(epoch_index, result_storage_fee); @@ -97,12 +101,12 @@ pub fn distribute_storage_fee_to_epochs_collection( ) } -type DistributionAmount = SignedCredits; -type DistributionLeftovers = SignedCredits; +type DistributionAmount = Credits; +type DistributionLeftovers = Credits; /// Calculates leftovers and amount of credits by distributing storage fees to epochs pub fn calculate_storage_fee_distribution_amount_and_leftovers( - storage_fee: SignedCredits, + storage_fee: Credits, start_epoch_index: EpochIndex, skip_up_to_epoch_index: EpochIndex, ) -> Result<(DistributionAmount, DistributionLeftovers), Error> { @@ -126,12 +130,12 @@ pub fn calculate_storage_fee_distribution_amount_and_leftovers( /// Distributes storage fees to epochs and call function for each epoch. /// Returns leftovers fn distribution_storage_fee_to_epochs_map( - storage_fee: SignedCredits, + storage_fee: Credits, start_epoch_index: EpochIndex, mut f: F, ) -> Result where - F: FnMut(EpochIndex, SignedCredits) -> Result<(), Error>, + F: FnMut(EpochIndex, Credits) -> Result<(), Error>, { if storage_fee == 0 { return Ok(0); @@ -150,9 +154,9 @@ where let epoch_fee_share_dec = year_fee_share / epochs_per_year; - let epoch_fee_share: SignedCredits = epoch_fee_share_dec + let epoch_fee_share: Credits = epoch_fee_share_dec .floor() - .to_i64() + .to_u64() .ok_or_else(|| get_overflow_error("storage fees are not fitting in a u64"))?; let year_start_epoch_index = start_epoch_index + EPOCHS_PER_YEAR * year; @@ -175,7 +179,7 @@ where mod tests { use super::*; - use crate::fee::credits::{Creditable, MAX}; + use crate::fee::credits::{Creditable, MAX_CREDITS}; use crate::fee::epoch::GENESIS_EPOCH_INDEX; mod fee_distribution_table { @@ -210,6 +214,7 @@ mod tests { let leftovers = distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, 0, + false, GENESIS_EPOCH_INDEX, None, ) @@ -221,13 +226,14 @@ mod tests { #[test] fn should_distribute_max_credits_value_without_overflow() { - let storage_fee = MAX.to_signed().expect("should convert signed credits"); + let storage_fee = MAX_CREDITS; let mut credits_per_epochs = SignedCreditsPerEpoch::default(); let leftovers = distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, storage_fee, + false, GENESIS_EPOCH_INDEX, None, ) @@ -247,6 +253,7 @@ mod tests { let leftovers = distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, storage_fee, + false, current_epoch_index, None, ) @@ -257,7 +264,7 @@ mod tests { // compare them with reference table #[rustfmt::skip] - let reference_fees: [SignedCredits; PERPETUAL_STORAGE_EPOCHS as usize] = [ + let reference_fees: [Credits; PERPETUAL_STORAGE_EPOCHS as usize] = [ 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2300, 2300, @@ -347,6 +354,7 @@ mod tests { let leftovers = distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, storage_fee, + false, current_epoch_index, None, ) @@ -368,7 +376,7 @@ mod tests { #[test] fn should_add_to_collection_from_specific_epoch() { - let storage_fee: SignedCredits = 1000000; + let storage_fee: Credits = 1000000; let start_epoch_index: EpochIndex = 0; const SKIP_UP_TO_EPOCH_INDEX: EpochIndex = 42; @@ -378,6 +386,7 @@ mod tests { let leftovers = distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, storage_fee, + false, start_epoch_index, Some(SKIP_UP_TO_EPOCH_INDEX), ) @@ -484,7 +493,7 @@ mod tests { #[test] fn should_calculate_amount_and_leftovers() { - let storage_fee = 1000; + let storage_fee = 10000; let (amount, leftovers) = calculate_storage_fee_distribution_amount_and_leftovers( storage_fee, diff --git a/packages/rs-drive/src/fee/result/refunds.rs b/packages/rs-drive/src/fee/result/refunds.rs index 0940aaa34f5..eb1405b75bb 100644 --- a/packages/rs-drive/src/fee/result/refunds.rs +++ b/packages/rs-drive/src/fee/result/refunds.rs @@ -34,9 +34,9 @@ use crate::error::fee::FeeError; use crate::error::Error; -use crate::fee::credits::SignedCredits; +use crate::fee::credits::{Credits, SignedCredits}; use crate::fee::default_costs::STORAGE_DISK_USAGE_CREDIT_PER_BYTE; -use crate::fee::epoch::SignedCreditsPerEpoch; +use crate::fee::epoch::CreditsPerEpoch; use crate::fee::get_overflow_error; use bincode::Options; use costs::storage_cost::removal::{Identifier, StorageRemovalPerEpochByIdentifier}; @@ -45,11 +45,11 @@ use std::collections::btree_map::{IntoIter, Iter}; use std::collections::BTreeMap; /// Credits per Epoch by Identifier -pub type SignedCreditsPerEpochByIdentifier = BTreeMap; +pub type CreditsPerEpochByIdentifier = BTreeMap; /// Fee refunds to identities based on removed data from specific epochs #[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)] -pub struct FeeRefunds(pub SignedCreditsPerEpochByIdentifier); +pub struct FeeRefunds(pub CreditsPerEpochByIdentifier); impl FeeRefunds { /// Create fee refunds from GroveDB's StorageRemovalPerEpochByIdentifier @@ -66,18 +66,18 @@ impl FeeRefunds { // TODO We should use multipliers - let credits: SignedCredits = (bytes as i64) - .checked_mul(STORAGE_DISK_USAGE_CREDIT_PER_BYTE as i64) + let credits: Credits = (bytes as Credits) + .checked_mul(STORAGE_DISK_USAGE_CREDIT_PER_BYTE) .ok_or_else(|| { get_overflow_error("storage written bytes cost overflow") })?; - Ok((epoch_index, -credits)) + Ok((epoch_index, credits)) }) - .collect::>() + .collect::>() .map(|credits_per_epochs| (identifier, credits_per_epochs)) }) - .collect::>()?; + .collect::>()?; Ok(Self(refunds_per_epoch_by_identifier)) } @@ -98,7 +98,7 @@ impl FeeRefunds { }; combined.map(|c| (k, c)) }) - .collect::>()?; + .collect::>()?; intersection.into_iter().chain(int_map_b).collect() } else { int_map_b @@ -110,17 +110,17 @@ impl FeeRefunds { } /// Passthrough method for get - pub fn get(&self, key: &Identifier) -> Option<&SignedCreditsPerEpoch> { + pub fn get(&self, key: &Identifier) -> Option<&CreditsPerEpoch> { self.0.get(key) } /// Passthrough method for iteration - pub fn iter(&self) -> Iter { + pub fn iter(&self) -> Iter { self.0.iter() } /// Passthrough method for into iteration - pub fn into_iter(self) -> IntoIter { + pub fn into_iter(self) -> IntoIter { self.0.into_iter() } diff --git a/packages/rs-drive/tests/query_tests_history.rs b/packages/rs-drive/tests/query_tests_history.rs index cd02e400e31..566dd53b7d4 100644 --- a/packages/rs-drive/tests/query_tests_history.rs +++ b/packages/rs-drive/tests/query_tests_history.rs @@ -256,8 +256,7 @@ fn test_query_historical() { assert_eq!( root_hash.as_slice(), vec![ - 236, 38, 173, 17, 112, 26, 137, 231, 93, 215, 163, 48, 78, 158, 154, 228, 190, 205, - 158, 195, 38, 174, 108, 12, 64, 124, 158, 113, 169, 169, 109, 46 + 49, 205, 177, 218, 169, 224, 236, 206, 112, 34, 163, 112, 222, 73, 92, 82, 189, 120, 135, 32, 13, 65, 253, 139, 167, 209, 146, 1, 81, 127, 38, 61 ] ); @@ -1535,8 +1534,7 @@ fn test_query_historical() { assert_eq!( root_hash.as_slice(), vec![ - 221, 89, 95, 226, 135, 240, 233, 77, 132, 118, 139, 135, 96, 161, 143, 92, 83, 249, - 205, 183, 108, 18, 208, 11, 85, 178, 80, 107, 62, 248, 22, 73 + 200, 234, 81, 179, 120, 70, 117, 20, 202, 219, 197, 168, 20, 96, 55, 130, 62, 243, 181, 198, 88, 50, 225, 68, 205, 54, 191, 136, 37, 65, 113, 200 ] ); } From b9148f644d1adc2bb1f80722063866f959ebfa02 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 22 Dec 2022 06:10:34 +0800 Subject: [PATCH 33/54] chore: switch to sum trees and refactoring --- Cargo.lock | 10 +- .../validation/public_keys_validator_spec.rs | 4 +- .../fee_pools/distribute_storage_pool.rs | 80 ++--- .../execution/fee_pools/fee_distribution.rs | 58 +++- .../execution/fee_pools/process_block_fees.rs | 19 +- packages/rs-drive-nodejs/src/converter.rs | 13 +- packages/rs-drive-nodejs/src/fee/mod.rs | 4 +- packages/rs-drive-nodejs/src/lib.rs | 4 +- packages/rs-drive/Cargo.toml | 6 +- .../src/drive/batch/grovedb_op_batch.rs | 6 + .../epochs/credit_distribution_pools.rs | 221 ++++++------- .../src/drive/fee_pools/epochs/proposers.rs | 143 ++++----- .../src/drive/fee_pools/epochs/start_block.rs | 196 +++++------- .../src/drive/fee_pools/epochs/start_time.rs | 111 +++---- packages/rs-drive/src/drive/fee_pools/mod.rs | 56 ++-- .../drive/fee_pools/pending_epoch_updates.rs | 51 +-- .../storage_fee_distribution_pool.rs | 68 ++-- .../src/drive/fee_pools/unpaid_epoch.rs | 88 +++--- packages/rs-drive/src/drive/initialization.rs | 4 +- packages/rs-drive/src/error/drive.rs | 4 + packages/rs-drive/src/error/fee.rs | 50 +-- packages/rs-drive/src/fee/credits.rs | 18 +- .../rs-drive/src/fee/epoch/distribution.rs | 281 +++++++++-------- packages/rs-drive/src/fee/op.rs | 2 - packages/rs-drive/src/fee/result/refunds.rs | 2 +- .../fee_pools/epochs/operations_factory.rs | 292 +++++++++--------- .../rs-drive/src/fee_pools/epochs/paths.rs | 5 +- packages/rs-drive/src/fee_pools/mod.rs | 47 +-- 28 files changed, 896 insertions(+), 947 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40f578f256b..c4ef50e5264 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -574,7 +574,7 @@ dependencies = [ [[package]] name = "costs" version = "0.0.0" -source = "git+https://github.com/dashpay/grovedb?branch=develop#3ef80bcd381d42d089681b1297e1137f4f5dbb67" +source = "git+https://github.com/dashpay/grovedb?rev=3ef80bcd381d42d089681b1297e1137f4f5dbb67#3ef80bcd381d42d089681b1297e1137f4f5dbb67" dependencies = [ "integer-encoding", "intmap", @@ -1254,7 +1254,7 @@ dependencies = [ [[package]] name = "grovedb" version = "0.7.1" -source = "git+https://github.com/dashpay/grovedb?branch=develop#3ef80bcd381d42d089681b1297e1137f4f5dbb67" +source = "git+https://github.com/dashpay/grovedb?rev=3ef80bcd381d42d089681b1297e1137f4f5dbb67#3ef80bcd381d42d089681b1297e1137f4f5dbb67" dependencies = [ "bincode", "costs", @@ -1689,7 +1689,7 @@ dependencies = [ [[package]] name = "merk" version = "0.7.1" -source = "git+https://github.com/dashpay/grovedb?branch=develop#3ef80bcd381d42d089681b1297e1137f4f5dbb67" +source = "git+https://github.com/dashpay/grovedb?rev=3ef80bcd381d42d089681b1297e1137f4f5dbb67#3ef80bcd381d42d089681b1297e1137f4f5dbb67" dependencies = [ "blake3", "byteorder", @@ -2746,7 +2746,7 @@ dependencies = [ [[package]] name = "storage" version = "0.2.0" -source = "git+https://github.com/dashpay/grovedb?branch=develop#3ef80bcd381d42d089681b1297e1137f4f5dbb67" +source = "git+https://github.com/dashpay/grovedb?rev=3ef80bcd381d42d089681b1297e1137f4f5dbb67#3ef80bcd381d42d089681b1297e1137f4f5dbb67" dependencies = [ "blake3", "costs", @@ -3133,7 +3133,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "visualize" version = "0.1.0" -source = "git+https://github.com/dashpay/grovedb?branch=develop#3ef80bcd381d42d089681b1297e1137f4f5dbb67" +source = "git+https://github.com/dashpay/grovedb?rev=3ef80bcd381d42d089681b1297e1137f4f5dbb67#3ef80bcd381d42d089681b1297e1137f4f5dbb67" dependencies = [ "hex", "itertools", diff --git a/packages/rs-dpp/src/tests/identity/validation/public_keys_validator_spec.rs b/packages/rs-dpp/src/tests/identity/validation/public_keys_validator_spec.rs index 0ca453a1028..6c8591e9b30 100644 --- a/packages/rs-dpp/src/tests/identity/validation/public_keys_validator_spec.rs +++ b/packages/rs-dpp/src/tests/identity/validation/public_keys_validator_spec.rs @@ -3,7 +3,7 @@ use serde_json::{json, Value}; use crate::consensus::ConsensusError; use crate::identity::validation::PublicKeysValidator; use crate::identity::validation::TPublicKeysValidator; -use crate::identity::{KeyType, Purpose, SecurityLevel}; +use crate::identity::{Purpose, SecurityLevel}; use crate::tests::fixtures::get_public_keys_validator; use crate::tests::utils::serde_set_ref; use crate::{assert_consensus_errors, NativeBlsModule}; @@ -414,7 +414,7 @@ pub fn should_return_invalid_result_if_key_data_is_not_a_valid_der() { ); assert_eq!( error.validation_error().as_ref().unwrap().message(), - "Key secp256k1 error: secp: malformed public key" + "Key secp256k1 error: malformed public key" ); } diff --git a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs index 26dac161a05..6b316b31768 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs @@ -33,17 +33,23 @@ //! storage fees from the distribution pool to the epoch pools. //! -use crate::error::execution::ExecutionError; use crate::error::Error; use crate::platform::Platform; use drive::drive::batch::GroveDbOpBatch; -use drive::fee::credits::{Creditable, Credits}; -use drive::fee::epoch::distribution::distribute_storage_fee_to_epochs_collection; -use drive::fee::epoch::{CreditsPerEpoch, EpochIndex, SignedCreditsPerEpoch}; +use drive::fee::credits::Credits; +use drive::fee::epoch::distribution::{ + distribute_refunds_to_epochs_collection, distribute_storage_fee_to_epochs_collection, +}; +use drive::fee::epoch::{EpochIndex, SignedCreditsPerEpoch}; use drive::grovedb::TransactionArg; -/// Leftovers in result of divisions and rounding after storage fee distribution to epochs -pub type DistributionLeftoverCredits = Credits; +/// Result of storage fee distribution +pub struct DistributionStorageFeeResult { + /// Leftovers in result of divisions and rounding after storage fee distribution to epochs + pub leftovers: Credits, + /// A number of epochs which had refunded + pub refunded_epochs_count: usize, +} impl Platform { /// Adds operations to the GroveDB op batch which calculate and distribute storage fees @@ -53,7 +59,7 @@ impl Platform { current_epoch_index: EpochIndex, transaction: TransactionArg, batch: &mut GroveDbOpBatch, - ) -> Result { + ) -> Result { let storage_distribution_fees = self .drive .get_aggregate_storage_fees_from_distribution_pool(transaction)?; @@ -64,22 +70,22 @@ impl Platform { let leftovers = distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, storage_distribution_fees, - false, current_epoch_index, - None, )?; // Deduct refunds since epoch where data was removed skipping previous (already paid or pay-in-progress) epochs. // Leftovers are ignored since they already deducted from Identity's refund amount + let refunds = self.drive.fetch_pending_updates(transaction)?; + let refunded_epochs_count = refunds.len(); + // TODO Better to use iterator do not load everything into memory - for (epoch_index, credits) in self.drive.fetch_pending_updates(transaction)? { - distribute_storage_fee_to_epochs_collection( + for (epoch_index, credits) in refunds { + distribute_refunds_to_epochs_collection( &mut credits_per_epochs, credits, - true, epoch_index, - Some(current_epoch_index), + current_epoch_index, )?; } @@ -90,7 +96,10 @@ impl Platform { transaction, )?; - Ok(leftovers.to_unsigned()) + Ok(DistributionStorageFeeResult { + leftovers, + refunded_epochs_count, + }) } } @@ -103,12 +112,10 @@ mod tests { mod add_distribute_storage_fee_to_epochs_operations { use drive::drive::fee_pools::pending_epoch_updates::add_update_pending_epoch_storage_pool_update_operations; - use drive::fee::credits::SignedCredits; - use drive::fee::epoch::{GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; + use drive::fee::credits::Creditable; + use drive::fee::epoch::{CreditsPerEpoch, GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; use drive::fee_pools::epochs::Epoch; - use drive::fee_pools::{ - update_storage_fee_distribution_pool_operation, update_unpaid_epoch_index_operation, - }; + use drive::fee_pools::update_storage_fee_distribution_pool_operation; use super::*; @@ -127,7 +134,10 @@ mod tests { let mut batch = GroveDbOpBatch::new(); // Store distribution storage fees - batch.push(update_storage_fee_distribution_pool_operation(storage_pool)); + batch.push( + update_storage_fee_distribution_pool_operation(storage_pool) + .expect("should return operation"), + ); platform .drive @@ -136,7 +146,7 @@ mod tests { let mut batch = GroveDbOpBatch::new(); - let leftovers = platform + platform .add_distribute_storage_fee_to_epochs_operations( current_epoch_index, Some(&transaction), @@ -161,11 +171,16 @@ mod tests { // init additional epochs pools as it will be done in epoch_change for i in PERPETUAL_STORAGE_EPOCHS..=PERPETUAL_STORAGE_EPOCHS + current_epoch_index { let epoch = Epoch::new(i); - epoch.add_init_empty_operations(&mut batch); + epoch + .add_init_empty_operations(&mut batch) + .expect("should add init operations"); } // Store distribution storage fees - batch.push(update_storage_fee_distribution_pool_operation(storage_pool)); + batch.push( + update_storage_fee_distribution_pool_operation(storage_pool) + .expect("should add operation"), + ); // Add pending refunds @@ -182,7 +197,7 @@ mod tests { let mut batch = GroveDbOpBatch::new(); - let leftovers = platform + let result = platform .add_distribute_storage_fee_to_epochs_operations( current_epoch_index, Some(&transaction), @@ -196,7 +211,8 @@ mod tests { .expect("should apply batch"); // check leftover - assert_eq!(leftovers, 180); + assert_eq!(result.leftovers, 180); + assert_eq!(result.refunded_epochs_count, refunds.len()); // collect all the storage fee values of the 1000 epochs pools let storage_fees = get_storage_credits_for_distribution_for_epochs_in_range( @@ -207,9 +223,9 @@ mod tests { // Assert total distributed fees - let total_storage_pool_distribution = ((storage_pool - leftovers) * 2) as SignedCredits; + let total_storage_pool_distribution = (storage_pool - result.leftovers) * 2; - let total_refunds: SignedCredits = refunds + let total_refunds: Credits = refunds .into_iter() .map(|(epoch_index, credits)| { let mut credits_per_epochs = SignedCreditsPerEpoch::default(); @@ -217,9 +233,7 @@ mod tests { let leftovers = distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, credits, - false, epoch_index, - None, ) .expect("should distribute refunds"); @@ -229,7 +243,7 @@ mod tests { credits_per_epochs .into_iter() .take(already_paid_epochs as usize) - .map(|(_, credits)| credits) + .map(|(_, credits)| credits.to_unsigned()) .sum() } else { 0 @@ -239,11 +253,7 @@ mod tests { }) .sum(); - let total_distributed = storage_fees - .into_iter() - .sum::() - .to_signed() - .expect("shouldn't overflow"); + let total_distributed = storage_fees.into_iter().sum::(); assert_eq!( total_distributed, diff --git a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs index 590a50fdead..f5e19d96b7e 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs @@ -401,7 +401,7 @@ impl Platform { let total_processing_fees = epoch_processing_fees + block_fees.processing_fee; - batch.push(current_epoch.update_processing_fee_pool_operation(total_processing_fees)); + batch.push(current_epoch.update_processing_fee_pool_operation(total_processing_fees)?); // update storage fee pool let storage_distribution_credits_in_fee_pool = match cached_aggregated_storage_fees { @@ -415,7 +415,7 @@ impl Platform { batch.push(update_storage_fee_distribution_pool_operation( storage_distribution_credits_in_fee_pool + block_fees.storage_fee, - )); + )?); Ok(FeesInPools { processing_fees: total_processing_fees, @@ -477,7 +477,11 @@ mod tests { unpaid_epoch_tree_0.add_init_current_operations(1.0, 1, 1, &mut batch); - batch.push(unpaid_epoch_tree_0.update_processing_fee_pool_operation(10000)); + batch.push( + unpaid_epoch_tree_0 + .update_processing_fee_pool_operation(10000) + .expect("should add operation"), + ); let proposers_count = 100u16; @@ -546,7 +550,11 @@ mod tests { unpaid_epoch_tree_0.add_init_current_operations(1.0, 1, 1, &mut batch); - batch.push(unpaid_epoch_tree_0.update_processing_fee_pool_operation(10000)); + batch.push( + unpaid_epoch_tree_0 + .update_processing_fee_pool_operation(10000) + .expect("should add operation"), + ); let proposers_count = 100u16; @@ -630,7 +638,11 @@ mod tests { unpaid_epoch_tree_0.add_init_current_operations(1.0, 1, 1, &mut batch); - batch.push(unpaid_epoch_tree_0.update_processing_fee_pool_operation(10000)); + batch.push( + unpaid_epoch_tree_0 + .update_processing_fee_pool_operation(10000) + .expect("should add operation"), + ); let proposers_count = 200u16; @@ -725,9 +737,17 @@ mod tests { unpaid_epoch.add_init_current_operations(1.0, 1, 1, &mut batch); - batch.push(unpaid_epoch.update_processing_fee_pool_operation(processing_fees)); + batch.push( + unpaid_epoch + .update_processing_fee_pool_operation(processing_fees) + .expect("should add operation"), + ); - batch.push(unpaid_epoch.update_storage_fee_pool_operation(storage_fees)); + batch.push( + unpaid_epoch + .update_storage_fee_pool_operation(storage_fees) + .expect("should add operation"), + ); current_epoch.add_init_current_operations(1.0, 2, 2, &mut batch); @@ -810,9 +830,17 @@ mod tests { unpaid_epoch.add_init_current_operations(1.0, 1, 1, &mut batch); - batch.push(unpaid_epoch.update_processing_fee_pool_operation(processing_fees)); + batch.push( + unpaid_epoch + .update_processing_fee_pool_operation(processing_fees) + .expect("should add operation"), + ); - batch.push(unpaid_epoch.update_storage_fee_pool_operation(storage_fees)); + batch.push( + unpaid_epoch + .update_storage_fee_pool_operation(storage_fees) + .expect("should add operation"), + ); current_epoch.add_init_current_operations(1.0, 2, 2, &mut batch); @@ -1174,9 +1202,17 @@ mod tests { unpaid_epoch_tree.add_init_current_operations(1.0, 1, 1, &mut batch); - batch.push(unpaid_epoch_tree.update_processing_fee_pool_operation(processing_fees)); + batch.push( + unpaid_epoch_tree + .update_processing_fee_pool_operation(processing_fees) + .expect("should add operation"), + ); - batch.push(unpaid_epoch_tree.update_storage_fee_pool_operation(storage_fees)); + batch.push( + unpaid_epoch_tree + .update_storage_fee_pool_operation(storage_fees) + .expect("should add operation"), + ); next_epoch_tree.add_init_current_operations( 1.0, diff --git a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs index d590e620d6c..5e76534974a 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/process_block_fees.rs @@ -37,14 +37,13 @@ use std::option::Option::None; use drive::drive::batch::GroveDbOpBatch; use drive::drive::fee_pools::pending_epoch_updates::add_update_pending_epoch_storage_pool_update_operations; -use drive::fee::credits::Credits; use drive::fee_pools::epochs::Epoch; use drive::grovedb::TransactionArg; use crate::abci::messages::BlockFees; use crate::block::BlockInfo; use crate::error::Error; -use crate::execution::fee_pools::distribute_storage_pool::DistributionLeftoverCredits; +use crate::execution::fee_pools::distribute_storage_pool::DistributionStorageFeeResult; use crate::execution::fee_pools::epoch::EpochInfo; use crate::execution::fee_pools::fee_distribution::{FeesInPools, ProposersPayouts}; use crate::platform::Platform; @@ -70,6 +69,8 @@ pub struct ProcessedBlockFeesResult { pub fees_in_pools: FeesInPools, /// A struct with the number of proposers to be paid out and the last paid epoch index pub payouts: Option, + /// A number of epochs which had refunded + pub refunded_epochs_count: Option, } impl Platform { @@ -85,7 +86,7 @@ impl Platform { block_fees: &BlockFees, transaction: TransactionArg, batch: &mut GroveDbOpBatch, - ) -> Result, Error> { + ) -> Result, Error> { // init next thousandth empty epochs since last initiated let last_initiated_epoch_index = epoch_info .previous_epoch_index @@ -144,7 +145,7 @@ impl Platform { let mut batch = GroveDbOpBatch::new(); - let storage_fee_distribution_leftover_credits = if epoch_info.is_epoch_change { + let storage_fee_distribution_result = if epoch_info.is_epoch_change { self.add_process_epoch_change_operations( block_info, epoch_info, @@ -193,7 +194,9 @@ impl Platform { ¤t_epoch, &block_fees, // Add leftovers after storage fee pool distribution to the current block storage fees - storage_fee_distribution_leftover_credits, + storage_fee_distribution_result + .as_ref() + .map(|result| result.leftovers), transaction, &mut batch, )?; @@ -218,6 +221,8 @@ impl Platform { Ok(ProcessedBlockFeesResult { fees_in_pools, payouts, + refunded_epochs_count: storage_fee_distribution_result + .map(|result| result.refunded_epochs_count), }) } } @@ -293,7 +298,7 @@ mod tests { let block_fees = BlockFees { storage_fee: 1000000000, processing_fee: 10000, - fee_refunds: CreditsPerEpoch::from_iter([(0, -10000)]), + fee_refunds: CreditsPerEpoch::from_iter([(0, 10000)]), }; let mut batch = GroveDbOpBatch::new(); @@ -458,7 +463,7 @@ mod tests { let block_fees = BlockFees { storage_fee: 1000, processing_fee: 10000, - fee_refunds: CreditsPerEpoch::from_iter([(epoch_index, -100)]), + fee_refunds: CreditsPerEpoch::from_iter([(epoch_index, 100)]), }; let distribute_storage_pool_result = platform diff --git a/packages/rs-drive-nodejs/src/converter.rs b/packages/rs-drive-nodejs/src/converter.rs index eb76120c163..534460f10f2 100644 --- a/packages/rs-drive-nodejs/src/converter.rs +++ b/packages/rs-drive-nodejs/src/converter.rs @@ -134,10 +134,14 @@ pub fn element_to_js_object<'a, C: Context<'a>>( js_object.set(cx, "type", js_type_string)?; let maybe_js_value: Option> = match element { - Element::Item(item, _) | Element::SumItem(item, ..) => { + Element::Item(item, _) => { let js_buffer = JsBuffer::external(cx, item); Some(js_buffer.upcast()) } + Element::SumItem(number, _) => { + let js_number = cx.number(number as f64).upcast(); + Some(js_number) + } Element::Reference(reference, _, _) => { let reference = reference_to_dictionary(cx, reference)?; Some(reference) @@ -217,6 +221,13 @@ pub fn reference_to_dictionary<'a, C: Context<'a>>( js_object.set(cx, "type", js_type_name)?; js_object.set(cx, "key", js_key)?; } + ReferencePathType::RemovedCousinReference(path) => { + let js_type_name = cx.string("removedCousin"); + let js_path = nested_vecs_to_js(cx, path)?; + + js_object.set(cx, "type", js_type_name)?; + js_object.set(cx, "path", js_path)?; + } } Ok(js_object.upcast()) diff --git a/packages/rs-drive-nodejs/src/fee/mod.rs b/packages/rs-drive-nodejs/src/fee/mod.rs index 02b48f5887a..3b2f56cf8e2 100644 --- a/packages/rs-drive-nodejs/src/fee/mod.rs +++ b/packages/rs-drive-nodejs/src/fee/mod.rs @@ -1,4 +1,4 @@ -use drive::fee::credits::SignedCredits; +use drive::fee::credits::Credits; use drive::fee::epoch::distribution::calculate_storage_fee_distribution_amount_and_leftovers; use drive::fee::epoch::EpochIndex; use neon::prelude::*; @@ -9,7 +9,7 @@ pub fn js_calculate_storage_fee_distribution_amount_and_leftovers( mut cx: FunctionContext, ) -> JsResult { let js_storage_fees = cx.argument::(0)?; - let storage_fees = js_storage_fees.value(&mut cx) as SignedCredits; + let storage_fees = js_storage_fees.value(&mut cx) as Credits; let js_start_epoch_index = cx.argument::(1)?; diff --git a/packages/rs-drive-nodejs/src/lib.rs b/packages/rs-drive-nodejs/src/lib.rs index c087ee909ca..1fd1a67c958 100644 --- a/packages/rs-drive-nodejs/src/lib.rs +++ b/packages/rs-drive-nodejs/src/lib.rs @@ -10,7 +10,7 @@ use drive::drive::batch::GroveDbOpBatch; use drive::drive::config::DriveConfig; use drive::drive::flags::StorageFlags; use drive::error::Error; -use drive::fee::credits::SignedCredits; +use drive::fee::credits::Credits; use drive::fee::epoch::CreditsPerEpoch; use drive::fee_pools::epochs::Epoch; use drive::grovedb::{PathQuery, Transaction}; @@ -2054,7 +2054,7 @@ impl PlatformWrapper { .or_else(|e: ParseIntError| cx.throw_error(e.to_string()))?; let js_credits: Handle = js_fee_refunds.get(&mut cx, js_epoch_index)?; - let credits = js_credits.value(&mut cx) as SignedCredits; + let credits = js_credits.value(&mut cx) as Credits; fee_refunds.insert(epoch_index, credits); } diff --git a/packages/rs-drive/Cargo.toml b/packages/rs-drive/Cargo.toml index 634bd0a466a..6ef5ca3340d 100644 --- a/packages/rs-drive/Cargo.toml +++ b/packages/rs-drive/Cargo.toml @@ -36,15 +36,15 @@ rust_decimal_macros = "1.25.0" [dependencies.grovedb] git = "https://github.com/dashpay/grovedb" -branch = "develop" +rev = "3ef80bcd381d42d089681b1297e1137f4f5dbb67" [dependencies.storage] git = "https://github.com/dashpay/grovedb" -branch = "develop" +rev = "3ef80bcd381d42d089681b1297e1137f4f5dbb67" [dependencies.costs] git = "https://github.com/dashpay/grovedb" -branch = "develop" +rev = "3ef80bcd381d42d089681b1297e1137f4f5dbb67" [dev-dependencies] criterion = "0.3.5" diff --git a/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs b/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs index a45f971c2d2..1e24f953271 100644 --- a/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs +++ b/packages/rs-drive/src/drive/batch/grovedb_op_batch.rs @@ -78,6 +78,12 @@ impl GroveDbOpBatch { .push(GroveDbOp::insert_op(path, key, Element::empty_tree())) } + /// Adds an `Insert` operation with an empty sum tree at the specified path and key to a list of GroveDB ops. + pub fn add_insert_empty_sum_tree(&mut self, path: Vec>, key: Vec) { + self.operations + .push(GroveDbOp::insert_op(path, key, Element::empty_sum_tree())) + } + /// Adds an `Insert` operation with an empty tree with storage flags to a list of GroveDB ops. pub fn add_insert_empty_tree_with_flags( &mut self, diff --git a/packages/rs-drive/src/drive/fee_pools/epochs/credit_distribution_pools.rs b/packages/rs-drive/src/drive/fee_pools/epochs/credit_distribution_pools.rs index 59d45ae1cde..daa8be714ab 100644 --- a/packages/rs-drive/src/drive/fee_pools/epochs/credit_distribution_pools.rs +++ b/packages/rs-drive/src/drive/fee_pools/epochs/credit_distribution_pools.rs @@ -35,9 +35,9 @@ use grovedb::{Element, TransactionArg}; use crate::drive::Drive; -use crate::error::fee::FeeError; +use crate::error::drive::DriveError; use crate::error::Error; -use crate::fee::credits::Credits; +use crate::fee::credits::{Creditable, Credits}; use crate::fee::get_overflow_error; use crate::fee_pools::epochs::Epoch; @@ -60,19 +60,13 @@ impl Drive { .unwrap() .map_err(Error::GroveDB)?; - if let Element::Item(item, _) = element { - Ok(u64::from_be_bytes(item.as_slice().try_into().map_err( - |_| { - Error::Fee(FeeError::CorruptedStorageFeeInvalidItemLength( - "epochs storage fee is not u64", - )) - }, - )?)) - } else { - Err(Error::Fee(FeeError::CorruptedStorageFeeNotItem( + let Element::SumItem(item, _) = element else { + return Err(Error::Drive(DriveError::UnexpectedElementType( "epochs storage fee must be an item", ))) - } + }; + + Ok(item.to_unsigned()) } /// Gets the amount of processing fees to be distributed for the Epoch. @@ -91,19 +85,13 @@ impl Drive { .unwrap() .map_err(Error::GroveDB)?; - if let Element::Item(item, _) = element { - Ok(u64::from_be_bytes(item.as_slice().try_into().map_err( - |_| { - Error::Fee(FeeError::CorruptedProcessingFeeInvalidItemLength( - "epochs processing fee is not u64", - )) - }, - )?)) - } else { - Err(Error::Fee(FeeError::CorruptedProcessingFeeNotItem( + let Element::SumItem(credits, _) = element else { + return Err(Error::Drive(DriveError::UnexpectedElementType( "epochs processing fee must be an item", ))) - } + }; + + Ok(credits.to_unsigned()) } /// Gets the Fee Multiplier for the Epoch. @@ -122,19 +110,19 @@ impl Drive { .unwrap() .map_err(Error::GroveDB)?; - if let Element::Item(item, _) = element { - Ok(f64::from_be_bytes(item.as_slice().try_into().map_err( - |_| { - Error::Fee(FeeError::CorruptedMultiplierInvalidItemLength( - "epochs multiplier item have an invalid length", - )) - }, - )?)) - } else { - Err(Error::Fee(FeeError::CorruptedMultiplierNotItem( + let Element::Item(encoded_multiplier, _) = element else { + return Err(Error::Drive(DriveError::UnexpectedElementType( "epochs multiplier must be an item", ))) - } + }; + + Ok(f64::from_be_bytes( + encoded_multiplier.as_slice().try_into().map_err(|_| { + Error::Drive(DriveError::CorruptedSerialization( + "epochs multiplier must be f64", + )) + })?, + )) } /// Gets the total credits to be distributed for the Epoch. @@ -157,103 +145,89 @@ impl Drive { #[cfg(test)] mod tests { + use super::*; + use crate::common::helpers::setup::setup_drive_with_initial_state_structure; use crate::drive::batch::GroveDbOpBatch; - use crate::error; - use crate::error::fee::FeeError; - use crate::fee::credits::Credits; - use crate::fee_pools::epochs::epoch_key_constants; - use crate::fee_pools::epochs::Epoch; - use grovedb::Element; + use crate::fee_pools::epochs_root_tree_key_constants::KEY_STORAGE_FEE_POOL; mod get_epoch_storage_credits_for_distribution { - use crate::fee_pools::epochs_root_tree_key_constants::KEY_STORAGE_FEE_POOL; + use super::*; #[test] fn test_error_if_epoch_tree_is_not_initiated() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(7000); - - match drive.get_epoch_storage_credits_for_distribution(&epoch, Some(&transaction)) { - Ok(_) => assert!( - false, - "should not be able to get storage fee on uninit epochs pool" - ), - Err(e) => match e { - super::error::Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let epoch = Epoch::new(7000); + + let result = + drive.get_epoch_storage_credits_for_distribution(&epoch, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_))) + )); } #[test] fn test_error_if_value_has_invalid_length() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); drive .grove .insert( epoch.get_path(), KEY_STORAGE_FEE_POOL.as_slice(), - super::Element::Item(u128::MAX.to_be_bytes().to_vec(), None), + Element::Item(u128::MAX.to_be_bytes().to_vec(), None), None, Some(&transaction), ) .unwrap() .expect("should insert invalid data"); - match drive.get_epoch_storage_credits_for_distribution(&epoch, Some(&transaction)) { - Ok(_) => assert!(false, "should not be able to decode stored value"), - Err(e) => match e { - super::error::Error::Fee( - super::FeeError::CorruptedStorageFeeInvalidItemLength(_), - ) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = + drive.get_epoch_storage_credits_for_distribution(&epoch, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::Drive(DriveError::UnexpectedElementType(_))) + )); } } mod get_epoch_processing_credits_for_distribution { + use super::*; + #[test] - fn test_error_if_value_has_invalid_length() { - let drive = super::setup_drive_with_initial_state_structure(); + fn test_error_if_value_has_wrong_element_type() { + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); drive .grove .insert( epoch.get_path(), - super::epoch_key_constants::KEY_POOL_PROCESSING_FEES.as_slice(), - super::Element::Item(u128::MAX.to_be_bytes().to_vec(), None), + epoch_key_constants::KEY_POOL_PROCESSING_FEES.as_slice(), + Element::Item(u128::MAX.to_be_bytes().to_vec(), None), None, Some(&transaction), ) .unwrap() .expect("should insert invalid data"); - match drive.get_epoch_processing_credits_for_distribution(&epoch, Some(&transaction)) { - Ok(_) => assert!(false, "should not be able to decode stored value"), - Err(e) => match e { - super::error::Error::Fee( - super::FeeError::CorruptedProcessingFeeInvalidItemLength(_), - ) => { - assert!(true) - } - _ => assert!(false, "ivalid error type"), - }, - } + let result = + drive.get_epoch_processing_credits_for_distribution(&epoch, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::Drive(DriveError::UnexpectedElementType(_))) + )); } } @@ -269,9 +243,17 @@ mod tests { let mut batch = GroveDbOpBatch::new(); - batch.push(epoch.update_processing_fee_pool_operation(processing_fee)); + batch.push( + epoch + .update_processing_fee_pool_operation(processing_fee) + .expect("should add operation"), + ); - batch.push(epoch.update_storage_fee_pool_operation(storage_fee)); + batch.push( + epoch + .update_storage_fee_pool_operation(storage_fee) + .expect("should add operation"), + ); drive .grove_apply_batch(batch, false, Some(&transaction)) @@ -285,71 +267,64 @@ mod tests { } mod fee_multiplier { + use super::*; + #[test] fn test_error_if_epoch_tree_is_not_initiated() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(7000); - - match drive.get_epoch_fee_multiplier(&epoch, Some(&transaction)) { - Ok(_) => assert!( - false, - "should not be able to get multiplier on uninit epochs pool" - ), - Err(e) => match e { - super::error::Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let epoch = Epoch::new(7000); + + let result = drive.get_epoch_fee_multiplier(&epoch, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_))) + )); } #[test] fn test_error_if_value_has_invalid_length() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); drive .grove .insert( epoch.get_path(), - super::epoch_key_constants::KEY_FEE_MULTIPLIER.as_slice(), - super::Element::Item(u128::MAX.to_be_bytes().to_vec(), None), + epoch_key_constants::KEY_FEE_MULTIPLIER.as_slice(), + Element::Item(u128::MAX.to_be_bytes().to_vec(), None), None, Some(&transaction), ) .unwrap() .expect("should insert invalid data"); - match drive.get_epoch_fee_multiplier(&epoch, Some(&transaction)) { - Ok(_) => assert!(false, "should not be able to decode stored value"), - Err(e) => match e { - super::error::Error::Fee( - super::FeeError::CorruptedMultiplierInvalidItemLength(_), - ) => { - assert!(true) - } - _ => assert!(false, "ivalid error type"), - }, - } + let result = drive.get_epoch_fee_multiplier(&epoch, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::Drive(DriveError::CorruptedSerialization(_))) + )); } #[test] fn test_value_is_set() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); let multiplier = 42.0; - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); - epoch.add_init_empty_operations(&mut batch); + epoch + .add_init_empty_operations(&mut batch) + .expect("should add empty epoch operations"); epoch.add_init_current_operations(multiplier, 1, 1, &mut batch); diff --git a/packages/rs-drive/src/drive/fee_pools/epochs/proposers.rs b/packages/rs-drive/src/drive/fee_pools/epochs/proposers.rs index 69e225b14eb..0e0002ef604 100644 --- a/packages/rs-drive/src/drive/fee_pools/epochs/proposers.rs +++ b/packages/rs-drive/src/drive/fee_pools/epochs/proposers.rs @@ -37,7 +37,6 @@ use grovedb::{Element, PathQuery, Query, SizedQuery, TransactionArg}; use crate::drive::Drive; use crate::error::drive::DriveError; -use crate::error::fee::FeeError; use crate::error::Error; use crate::fee_pools::epochs::Epoch; @@ -55,19 +54,22 @@ impl Drive { .unwrap() .map_err(Error::GroveDB)?; - if let Element::Item(item, _) = element { - Ok(u64::from_be_bytes(item.as_slice().try_into().map_err( + let Element::Item(encoded_proposer_block_count, _) = element else { + return Err(Error::Drive(DriveError::UnexpectedElementType( + "epochs proposer block count must be an item", + ))); + }; + + let proposer_block_count = + u64::from_be_bytes(encoded_proposer_block_count.as_slice().try_into().map_err( |_| { - Error::Fee(FeeError::CorruptedProposerBlockCountItemLength( + Error::Drive(DriveError::CorruptedSerialization( "epochs proposer block count item have an invalid length", )) }, - )?)) - } else { - Err(Error::Fee(FeeError::CorruptedProposerBlockCountNotItem( - "epochs proposer block count must be an item", - ))) - } + )?); + + Ok(proposer_block_count) } /// Returns true if the Epoch's Proposers Tree is empty @@ -82,14 +84,12 @@ impl Drive { .unwrap() { Ok(result) => Ok(result), - Err(err) => match err { - grovedb::Error::PathNotFound(_) | grovedb::Error::PathParentLayerNotFound(_) => { - Ok(true) - } - _ => Err(Error::Drive(DriveError::CorruptedCodeExecution( - "internal grovedb error", - ))), - }, + Err(grovedb::Error::PathNotFound(_) | grovedb::Error::PathParentLayerNotFound(_)) => { + Ok(true) + } + Err(_) => Err(Error::Drive(DriveError::CorruptedCodeExecution( + "internal grovedb error", + ))), } } @@ -119,130 +119,123 @@ impl Drive { .0 .to_key_elements(); - let result = key_elements + let proposers = key_elements .into_iter() .map(|(pro_tx_hash, element)| { - if let Element::Item(item, _) = element { - let block_count = - u64::from_be_bytes(item.as_slice().try_into().map_err(|_| { - Error::Fee(FeeError::CorruptedProposerBlockCountItemLength( - "epochs proposer block count item have an invalid length", - )) - })?); - - Ok((pro_tx_hash, block_count)) - } else { - Err(Error::Fee(FeeError::CorruptedProposerBlockCountNotItem( + let Element::Item(encoded_block_count, _) = element else { + return Err(Error::Drive(DriveError::UnexpectedElementType( "epochs proposer block count must be an item", - ))) - } + ))); + }; + + let block_count = u64::from_be_bytes( + encoded_block_count.as_slice().try_into().map_err(|_| { + Error::Drive(DriveError::CorruptedSerialization( + "epochs proposer block count must be u64", + )) + })?, + ); + + Ok((pro_tx_hash, block_count)) }) - .collect::, u64)>, Error>>()?; + .collect::>()?; - Ok(result) + Ok(proposers) } } #[cfg(test)] mod tests { - use grovedb::Element; - - use crate::error::{self, fee::FeeError}; - + use super::*; use crate::common::helpers::setup::setup_drive_with_initial_state_structure; use crate::drive::batch::GroveDbOpBatch; - use crate::fee_pools::epochs::Epoch; mod get_epochs_proposer_block_count { + use super::*; #[test] fn test_error_if_value_has_invalid_length() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); let pro_tx_hash: [u8; 32] = rand::random(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); batch.push(epoch.init_proposers_tree_operation()); batch.add_insert( epoch.get_proposers_vec_path(), pro_tx_hash.to_vec(), - super::Element::Item(u128::MAX.to_be_bytes().to_vec(), None), + Element::Item(u128::MAX.to_be_bytes().to_vec(), None), ); drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - match drive.get_epochs_proposer_block_count(&epoch, &pro_tx_hash, Some(&transaction)) { - Ok(_) => assert!(false, "should not be able to decode stored value"), - Err(e) => match e { - super::error::Error::Fee( - super::FeeError::CorruptedProposerBlockCountItemLength(_), - ) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = + drive.get_epochs_proposer_block_count(&epoch, &pro_tx_hash, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::Drive(DriveError::CorruptedSerialization(_),)) + )); } #[test] fn test_error_if_epoch_tree_is_not_initiated() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); let pro_tx_hash: [u8; 32] = rand::random(); - let epoch = super::Epoch::new(7000); - - match drive.get_epochs_proposer_block_count(&epoch, &pro_tx_hash, Some(&transaction)) { - Ok(_) => assert!( - false, - "should not be able to get proposer block count on uninit epochs pool" - ), - Err(e) => match e { - super::error::Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let epoch = Epoch::new(7000); + + let result = + drive.get_epochs_proposer_block_count(&epoch, &pro_tx_hash, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_))) + )); } } mod is_epochs_proposers_tree_empty { + use super::*; + #[test] fn test_check_if_empty() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); let result = drive .is_epochs_proposers_tree_empty(&epoch, Some(&transaction)) .expect("should check if tree is empty"); - assert_eq!(result, true); + assert!(result); } } mod get_epoch_proposers { + use super::*; + #[test] fn test_value() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); let pro_tx_hash: [u8; 32] = rand::random(); let block_count = 42; - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); batch.push(epoch.init_proposers_tree_operation()); diff --git a/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs b/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs index c87c8094d9f..f957d9919c2 100644 --- a/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs +++ b/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs @@ -34,13 +34,13 @@ use crate::drive::fee_pools::pools_vec_path; use crate::drive::Drive; -use crate::error::fee::FeeError; +use crate::error::drive::DriveError; use crate::error::Error; +use crate::fee::epoch::EpochIndex; use crate::fee_pools::epochs::{paths, Epoch}; use grovedb::query_result_type::QueryResultType::QueryPathKeyElementTrioResultType; use grovedb::{Element, PathQuery, Query, SizedQuery, TransactionArg}; -use crate::fee_pools::epochs::epoch_key_constants; use crate::fee_pools::epochs::epoch_key_constants::KEY_START_BLOCK_HEIGHT; impl Drive { @@ -54,41 +54,47 @@ impl Drive { .grove .get( epoch_tree.get_path(), - epoch_key_constants::KEY_START_BLOCK_HEIGHT.as_slice(), + KEY_START_BLOCK_HEIGHT.as_slice(), transaction, ) .unwrap() .map_err(Error::GroveDB)?; - if let Element::Item(item, _) = element { - Ok(u64::from_be_bytes(item.as_slice().try_into().map_err( - |_| Error::Fee(FeeError::CorruptedStartBlockHeightItemLength()), - )?)) - } else { - Err(Error::Fee(FeeError::CorruptedStartBlockHeightNotItem())) - } + let Element::Item(encoded_start_block_height, _) = element else { + return Err(Error::Drive(DriveError::UnexpectedElementType("start block height must be an item"))); + }; + + let start_block_height = u64::from_be_bytes( + encoded_start_block_height + .as_slice() + .try_into() + .map_err(|_| { + Error::Drive(DriveError::CorruptedSerialization( + "start block height must be u64", + )) + })?, + ); + + Ok(start_block_height) } /// Returns the index and start block height of the first epoch between the two given. pub fn get_first_epoch_start_block_height_between_epochs( &self, - from_epoch_index: u16, - to_epoch_index: u16, + from_epoch_index: EpochIndex, + to_epoch_index: EpochIndex, transaction: TransactionArg, - ) -> Result, Error> { - let mut start_block_height_query = Query::new(); - start_block_height_query.insert_key(KEY_START_BLOCK_HEIGHT.to_vec()); - - let mut epochs_query = Query::new(); + ) -> Result, Error> { + let mut query = Query::new(); let from_epoch_key = paths::encode_epoch_index_key(from_epoch_index)?.to_vec(); let current_epoch_key = paths::encode_epoch_index_key(to_epoch_index)?.to_vec(); - epochs_query.insert_range_after_to_inclusive(from_epoch_key..=current_epoch_key); + query.insert_range_after_to_inclusive(from_epoch_key..=current_epoch_key); - epochs_query.set_subquery(start_block_height_query); + query.set_subquery_key(KEY_START_BLOCK_HEIGHT.to_vec()); - let sized_query = SizedQuery::new(epochs_query, Some(1), None); + let sized_query = SizedQuery::new(query, Some(1), None); let path_query = PathQuery::new(pools_vec_path(), sized_query); @@ -98,27 +104,28 @@ impl Drive { .unwrap() .map_err(Error::GroveDB)?; - if result_items.elements.is_empty() { + if result_items.len() == 0 { return Ok(None); } - let first_result = &result_items.to_path_key_elements()[0]; + let (path, _, element) = result_items.to_path_key_elements().remove(0); - let (path, _, element) = first_result; + let Element::Item(item, _) = element else { + return Err(Error::Drive(DriveError::UnexpectedElementType("start block must be an item"))); + }; - let next_start_block_height = if let Element::Item(item, _) = element { + let next_start_block_height = u64::from_be_bytes(item.as_slice().try_into().map_err(|_| { - Error::Fee(FeeError::CorruptedProposerBlockCountItemLength( - "item have an invalid length", + Error::Drive(DriveError::CorruptedSerialization( + "start block height must be u64", )) - })?) - } else { - return Err(Error::Fee(FeeError::CorruptedStartBlockHeightItemLength())); - }; + })?); let epoch_key = path .last() - .ok_or(Error::Fee(FeeError::CorruptedStartBlockHeightItemLength()))?; + .ok_or(Error::Drive(DriveError::CorruptedCodeExecution( + "epoch pool shouldn't have empty path", + )))?; let epoch_index = paths::decode_epoch_index_key(epoch_key.as_slice())?; @@ -128,124 +135,98 @@ impl Drive { #[cfg(test)] mod tests { + use super::*; use crate::common::helpers::setup::setup_drive_with_initial_state_structure; - use crate::fee_pools::epochs::epoch_key_constants; - use grovedb::Element; - - use crate::error; - use crate::error::fee::FeeError; - - use super::Epoch; mod get_epoch_start_block_height { - use crate::fee_pools::epochs::Epoch; + use super::*; #[test] fn test_error_if_epoch_tree_is_not_initiated() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); let non_initiated_epoch = Epoch::new(7000); - match drive.get_epoch_start_block_height(&non_initiated_epoch, Some(&transaction)) { - Ok(_) => assert!( - false, - "should not be able to get start block height on uninit epochs pool" - ), - Err(e) => match e { - super::error::Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = + drive.get_epoch_start_block_height(&non_initiated_epoch, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_))) + )); } #[test] fn test_error_if_value_is_not_set() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); + + let result = drive.get_epoch_start_block_height(&epoch, Some(&transaction)); - match drive.get_epoch_start_block_height(&epoch, Some(&transaction)) { - Ok(_) => assert!(false, "must be an error"), - Err(e) => match e { - super::error::Error::GroveDB(_) => assert!(true), - _ => assert!(false, "invalid error type"), - }, - } + assert!(matches!(result, Err(Error::GroveDB(_)))); } #[test] fn test_error_if_value_has_invalid_length() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); drive .grove .insert( epoch.get_path(), - super::epoch_key_constants::KEY_START_BLOCK_HEIGHT.as_slice(), - super::Element::Item(u128::MAX.to_be_bytes().to_vec(), None), + KEY_START_BLOCK_HEIGHT.as_slice(), + Element::Item(u128::MAX.to_be_bytes().to_vec(), None), None, Some(&transaction), ) .unwrap() .expect("should insert invalid data"); - match drive.get_epoch_start_block_height(&epoch, Some(&transaction)) { - Ok(_) => assert!(false, "should not be able to decode stored value"), - Err(e) => match e { - super::error::Error::Fee( - super::FeeError::CorruptedStartBlockHeightItemLength(), - ) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = drive.get_epoch_start_block_height(&epoch, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::Drive(DriveError::CorruptedSerialization(_))) + )); } #[test] fn test_error_if_element_has_invalid_type() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); drive .grove .insert( epoch.get_path(), - super::epoch_key_constants::KEY_START_BLOCK_HEIGHT.as_slice(), - super::Element::empty_tree(), + KEY_START_BLOCK_HEIGHT.as_slice(), + Element::empty_tree(), None, Some(&transaction), ) .unwrap() .expect("should insert invalid data"); - match drive.get_epoch_start_block_height(&epoch, Some(&transaction)) { - Ok(_) => assert!(false, "should not be able to decode stored value"), - Err(e) => match e { - super::error::Error::Fee( - super::FeeError::CorruptedStartBlockHeightNotItem(), - ) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = drive.get_epoch_start_block_height(&epoch, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::Drive(DriveError::UnexpectedElementType(_))) + )); } } mod get_first_epoch_start_block_height_between_epochs { - use crate::common::helpers::setup::setup_drive_with_initial_state_structure; + use super::*; use crate::drive::batch::GroveDbOpBatch; - use crate::fee_pools::epochs::Epoch; #[test] fn test_next_block_height() { @@ -260,7 +241,6 @@ mod tests { batch.push(epoch_tree_0.update_start_block_height_operation(1)); batch.push(epoch_tree_1.update_start_block_height_operation(2)); - // Apply proposers tree drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); @@ -269,13 +249,7 @@ mod tests { .get_first_epoch_start_block_height_between_epochs(0, 2, Some(&transaction)) .expect("should find next start_block_height"); - match next_epoch_start_block_height_option { - None => assert!(false, "should find start_block_height"), - Some((epoch_index, start_block_height)) => { - assert_eq!(epoch_index, 1); - assert_eq!(start_block_height, 2); - } - } + assert!(matches!(next_epoch_start_block_height_option, Some((1, 2)))); } #[test] @@ -287,10 +261,7 @@ mod tests { .get_first_epoch_start_block_height_between_epochs(0, 4, Some(&transaction)) .expect("should find next start_block_height"); - match next_epoch_start_block_height { - None => assert!(true), - Some(_) => assert!(false, "should not find any"), - } + assert!(next_epoch_start_block_height.is_none()); } #[test] @@ -306,7 +277,6 @@ mod tests { batch.push(epoch_tree_0.update_start_block_height_operation(1)); batch.push(epoch_tree_3.update_start_block_height_operation(3)); - // Apply proposers tree drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); @@ -315,10 +285,7 @@ mod tests { .get_first_epoch_start_block_height_between_epochs(0, 2, Some(&transaction)) .expect("should find next start_block_height"); - match next_epoch_start_block_height { - None => assert!(true), - Some(_) => assert!(false, "should not find any"), - } + assert!(next_epoch_start_block_height.is_none()); } #[test] @@ -334,7 +301,6 @@ mod tests { batch.push(epoch_tree_0.update_start_block_height_operation(1)); batch.push(epoch_tree_3.update_start_block_height_operation(2)); - // Apply proposers tree drive .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); @@ -343,13 +309,7 @@ mod tests { .get_first_epoch_start_block_height_between_epochs(0, 4, Some(&transaction)) .expect("should find next start_block_height"); - match next_epoch_start_block_height { - None => assert!(false), - Some((epoch_index, start_block_height)) => { - assert_eq!(epoch_index, 3); - assert_eq!(start_block_height, 2); - } - } + assert!(matches!(next_epoch_start_block_height, Some((3, 2)))); } } } diff --git a/packages/rs-drive/src/drive/fee_pools/epochs/start_time.rs b/packages/rs-drive/src/drive/fee_pools/epochs/start_time.rs index 38f8bfb194a..5db343933c0 100644 --- a/packages/rs-drive/src/drive/fee_pools/epochs/start_time.rs +++ b/packages/rs-drive/src/drive/fee_pools/epochs/start_time.rs @@ -33,12 +33,12 @@ //! use crate::drive::Drive; -use crate::error::fee::FeeError; +use crate::error::drive::DriveError; use crate::error::Error; use crate::fee_pools::epochs::Epoch; use grovedb::{Element, TransactionArg}; -use crate::fee_pools::epochs::epoch_key_constants; +use crate::fee_pools::epochs::epoch_key_constants::KEY_START_TIME; impl Drive { /// Returns the start time of the given Epoch. @@ -51,130 +51,113 @@ impl Drive { .grove .get( epoch_tree.get_path(), - epoch_key_constants::KEY_START_TIME.as_slice(), + KEY_START_TIME.as_slice(), transaction, ) .unwrap() .map_err(Error::GroveDB)?; - if let Element::Item(item, _) = element { - Ok(u64::from_be_bytes(item.as_slice().try_into().map_err( - |_| Error::Fee(FeeError::CorruptedStartTimeLength()), - )?)) - } else { - Err(Error::Fee(FeeError::CorruptedStartTimeNotItem())) - } + let Element::Item(encoded_start_time, _) = element else { + return Err(Error::Drive(DriveError::UnexpectedElementType("start time must be an item"))) + }; + + let start_time = + u64::from_be_bytes(encoded_start_time.as_slice().try_into().map_err(|_| { + Error::Drive(DriveError::CorruptedSerialization("start time must be u64")) + })?); + + Ok(start_time) } } #[cfg(test)] mod tests { use crate::common::helpers::setup::setup_drive_with_initial_state_structure; - use grovedb::Element; - use crate::error; - use crate::error::fee::FeeError; - - use super::Epoch; + use super::*; mod get_epoch_start_time { - use crate::fee_pools::epochs::epoch_key_constants::KEY_START_TIME; + use super::*; #[test] fn test_error_if_epoch_tree_is_not_initiated() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let non_initiated_epoch_tree = super::Epoch::new(7000); - - match drive.get_epoch_start_time(&non_initiated_epoch_tree, Some(&transaction)) { - Ok(_) => assert!( - false, - "should not be able to get start time on uninit epochs pool" - ), - Err(e) => match e { - super::error::Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let non_initiated_epoch_tree = Epoch::new(7000); + + let result = drive.get_epoch_start_time(&non_initiated_epoch_tree, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_))) + )); } #[test] fn test_error_if_value_is_not_set() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch_tree = super::Epoch::new(0); + let epoch_tree = Epoch::new(0); + + let result = drive.get_epoch_start_time(&epoch_tree, Some(&transaction)); - match drive.get_epoch_start_time(&epoch_tree, Some(&transaction)) { - Ok(_) => assert!(false, "must be an error"), - Err(e) => match e { - super::error::Error::GroveDB(_) => assert!(true), - _ => assert!(false, "invalid error type"), - }, - } + assert!(matches!(result, Err(Error::GroveDB(_)))); } #[test] fn test_error_if_element_has_invalid_type() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); drive .grove .insert( epoch.get_path(), KEY_START_TIME.as_slice(), - super::Element::empty_tree(), + Element::empty_tree(), None, Some(&transaction), ) .unwrap() .expect("should insert invalid data"); - match drive.get_epoch_start_time(&epoch, Some(&transaction)) { - Ok(_) => assert!(false, "must be an error"), - Err(e) => match e { - super::error::Error::Fee(super::FeeError::CorruptedStartTimeNotItem()) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = drive.get_epoch_start_time(&epoch, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::Drive(DriveError::UnexpectedElementType(_))) + )); } #[test] fn test_error_if_value_has_invalid_length() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch_tree = super::Epoch::new(0); + let epoch_tree = Epoch::new(0); drive .grove .insert( epoch_tree.get_path(), KEY_START_TIME.as_slice(), - super::Element::Item(u128::MAX.to_be_bytes().to_vec(), None), + Element::Item(u128::MAX.to_be_bytes().to_vec(), None), None, Some(&transaction), ) .unwrap() .expect("should insert invalid data"); - match drive.get_epoch_start_time(&epoch_tree, Some(&transaction)) { - Ok(_) => assert!(false, "must be an error"), - Err(e) => match e { - super::error::Error::Fee(super::FeeError::CorruptedStartTimeLength()) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = drive.get_epoch_start_time(&epoch_tree, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::Drive(DriveError::CorruptedSerialization(_))) + )) } } } diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index 143898aa554..9744bf2ce84 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -31,14 +31,15 @@ use crate::drive::batch::GroveDbOpBatch; use crate::drive::{Drive, RootTree}; use crate::error::drive::DriveError; use crate::error::Error; -use crate::fee::credits::{Creditable, Credits, SignedCredits}; -use crate::fee::epoch::{CreditsPerEpoch, SignedCreditsPerEpoch}; +use crate::fee::credits::SignedCredits; +use crate::fee::epoch::{EpochIndex, SignedCreditsPerEpoch}; use crate::fee::get_overflow_error; use crate::fee_pools::epochs::epoch_key_constants::KEY_POOL_STORAGE_FEES; use crate::fee_pools::epochs::{paths, Epoch}; use crate::fee_pools::epochs_root_tree_key_constants::{ KEY_PENDING_POOL_UPDATES, KEY_STORAGE_FEE_POOL, }; +use grovedb::query_result_type::QueryResultType; use grovedb::{Element, PathQuery, Query, TransactionArg}; use itertools::Itertools; @@ -104,7 +105,7 @@ impl Drive { let max_encoded_epoch_index = paths::encode_epoch_index_key(max_epoch_index.to_owned())?.to_vec(); - if max_epoch_index - min_epoch_index + 1 != credits_per_epochs.len() as u16 { + if max_epoch_index - min_epoch_index + 1 != credits_per_epochs.len() as EpochIndex { return Err(Error::Drive(DriveError::CorruptedCodeExecution( "gaps in credits per epoch are not supported", ))); @@ -119,36 +120,47 @@ impl Drive { epochs_query.set_subquery(storage_fee_pool_query); - let (storage_fee_pools, _) = self + let (storage_fee_pools_result, _) = self .grove - .query( + .query_raw( &PathQuery::new_unsized(pools_vec_path(), epochs_query), + QueryResultType::QueryElementResultType, transaction, ) .unwrap() .map_err(Error::GroveDB)?; + let storage_fee_pools = storage_fee_pools_result.to_elements(); + for (i, (epoch_index, credits)) in credits_per_epochs .into_iter() .sorted_by_key(|x| x.0) .enumerate() { - let encoded_existing_storage_fee = storage_fee_pools - .get(i) - .map_or(0u64.to_vec_bytes(), |c| c.to_owned()); - - let existing_storage_fee = Credits::from_vec_bytes(encoded_existing_storage_fee)?.to_signed()?; + let existing_storage_fee: SignedCredits = match storage_fee_pools.get(i) { + Some(Element::SumItem(storage_fee, _)) => *storage_fee, + None => 0, + Some(_) => { + return Err(Error::Drive(DriveError::UnexpectedElementType( + "epoch storage pools must be sum items", + ))) + } + }; let credits_to_update = existing_storage_fee.checked_add(credits).ok_or_else(|| { get_overflow_error("can't add credits to existing epoch pool storage fee") })?; - let element = Element::new_item(credits_to_update.to_unsigned().to_vec_bytes()); + if credits_to_update < 0 { + return Err(Error::Drive(DriveError::CorruptedCodeExecution( + "epoch storage pool went bellow zero", + ))); + } batch.add_insert( Epoch::new(epoch_index).get_vec_path(), KEY_POOL_STORAGE_FEES.to_vec(), - element, + Element::new_sum_item(credits_to_update), ); } @@ -164,7 +176,6 @@ mod tests { mod add_update_epoch_storage_fee_pools_operations { use super::*; - use crate::fee::credits::Credits; use crate::fee::epoch::{EpochIndex, GENESIS_EPOCH_INDEX}; use grovedb::batch::{GroveDbOp, Op}; @@ -198,15 +209,14 @@ mod tests { // Store initial epoch storage pool values let operations = (GENESIS_EPOCH_INDEX..TO_EPOCH_INDEX) .into_iter() - .map(|epoch_index| { - let credits = 10 - epoch_index as Credits; - - let element = Element::new_item(credits.to_unsigned().to_vec_bytes()); + .enumerate() + .map(|(i, epoch_index)| { + let credits = 10 - i as SignedCredits; GroveDbOp::insert_op( Epoch::new(epoch_index).get_vec_path(), KEY_POOL_STORAGE_FEES.to_vec(), - element, + Element::new_sum_item(credits), ) }) .collect(); @@ -217,7 +227,8 @@ mod tests { .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - let credits_to_epochs: SignedCreditsPerEpoch = (GENESIS_EPOCH_INDEX..TO_EPOCH_INDEX).enumerate() + let credits_to_epochs: SignedCreditsPerEpoch = (GENESIS_EPOCH_INDEX..TO_EPOCH_INDEX) + .enumerate() .map(|(credits, epoch_index)| (epoch_index, credits as SignedCredits)) .collect(); @@ -238,16 +249,13 @@ mod tests { assert_eq!( operation.path.to_path(), - Epoch::new(i as u16).get_vec_path() + Epoch::new(i as EpochIndex).get_vec_path() ); - let Op::Insert{ element: Element::Item (encoded_credits, _)} = operation.op else { + let Op::Insert{ element: Element::SumItem (credits, _)} = operation.op else { panic!("invalid operation"); }; - let credits = - Credits::from_vec_bytes(encoded_credits).expect("should decide credits"); - assert_eq!(credits, 10); } } diff --git a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs index 7fdbf60a232..f4efa29b423 100644 --- a/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs +++ b/packages/rs-drive/src/drive/fee_pools/pending_epoch_updates.rs @@ -42,7 +42,7 @@ use crate::drive::fee_pools::pools_pending_updates_path; use crate::drive::Drive; use crate::error::drive::DriveError; use crate::error::Error; -use crate::fee::credits::{Creditable, SignedCredits}; +use crate::fee::credits::Creditable; use crate::fee::epoch::CreditsPerEpoch; use crate::fee::get_overflow_error; use grovedb::query_result_type::QueryResultType; @@ -68,20 +68,26 @@ impl Drive { .unwrap() .map_err(Error::GroveDB)?; - query_result.to_key_elements().into_iter().map(|(epoch_index_key, element)| { - let epoch_index = - u16::from_be_bytes(epoch_index_key.as_slice().try_into().map_err(|_| { - Error::Drive(DriveError::CorruptedSerialization( - "epoch index for pending pool updates must be i64", - )) - })?); - - if let Element::SumItem(credits, _) = element { - Ok((epoch_index, credits.to_unsigned())) - } else { - Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates credits must be sum items"))) - } - }).collect::>() + query_result + .to_key_elements() + .into_iter() + .map(|(epoch_index_key, element)| { + let epoch_index = + u16::from_be_bytes(epoch_index_key.as_slice().try_into().map_err(|_| { + Error::Drive(DriveError::CorruptedSerialization( + "epoch index for pending pool updates must be i64", + )) + })?); + + if let Element::SumItem(credits, _) = element { + Ok((epoch_index, credits.to_unsigned())) + } else { + Err(Error::Drive(DriveError::CorruptedCodeExecution( + "pending updates credits must be sum items", + ))) + } + }) + .collect::>() } /// Fetches existing pending epoch pool updates using specified epochs @@ -123,17 +129,18 @@ impl Drive { )) })?); - let credits_to_update = credits_per_epoch.get(&epoch_index).ok_or( - Error::Drive(DriveError::CorruptedCodeExecution("pending updates should contain fetched epochs")))?; + let existing_credits = credits_per_epoch.get_mut(&epoch_index).ok_or(Error::Drive( + DriveError::CorruptedCodeExecution("pending updates should contain fetched epochs"), + ))?; if let Element::SumItem(credits, _) = element { - let result_credits = credits.to_unsigned() - .checked_add(*credits_to_update) + *existing_credits = existing_credits + .checked_add(credits.to_unsigned()) .ok_or_else(|| get_overflow_error("pending updates credits overflow"))?; - - credits_per_epoch.insert(epoch_index, result_credits); } else { - return Err(Error::Drive(DriveError::CorruptedCodeExecution("pending updates credits must be sum items"))); + return Err(Error::Drive(DriveError::CorruptedCodeExecution( + "pending updates credits must be sum items", + ))); } } diff --git a/packages/rs-drive/src/drive/fee_pools/storage_fee_distribution_pool.rs b/packages/rs-drive/src/drive/fee_pools/storage_fee_distribution_pool.rs index 8a880687fdf..d5610f36b34 100644 --- a/packages/rs-drive/src/drive/fee_pools/storage_fee_distribution_pool.rs +++ b/packages/rs-drive/src/drive/fee_pools/storage_fee_distribution_pool.rs @@ -32,11 +32,11 @@ use crate::drive::fee_pools::pools_path; use crate::drive::Drive; +use crate::error::drive::DriveError; use grovedb::{Element, TransactionArg}; -use crate::error::fee::FeeError; use crate::error::Error; -use crate::fee::credits::Credits; +use crate::fee::credits::{Creditable, Credits}; use crate::fee_pools::epochs_root_tree_key_constants::KEY_STORAGE_FEE_POOL; impl Drive { @@ -50,21 +50,10 @@ impl Drive { .get(pools_path(), KEY_STORAGE_FEE_POOL.as_slice(), transaction) .unwrap() { - Ok(element) => { - if let Element::Item(item, _) = element { - let fee = u64::from_be_bytes(item.as_slice().try_into().map_err(|_| { - Error::Fee(FeeError::CorruptedStorageFeePoolInvalidItemLength( - "fee pools storage fee pool is not i64", - )) - })?); - - Ok(fee) - } else { - Err(Error::Fee(FeeError::CorruptedStorageFeePoolNotItem( - "fee pools storage fee pool must be an item", - ))) - } - } + Ok(Element::SumItem(credits, _)) => Ok(credits.to_unsigned()), + Ok(_) => Err(Error::Drive(DriveError::UnexpectedElementType( + "fee pools storage fee pool must be sum item", + ))), Err(grovedb::Error::PathKeyNotFound(_)) => Ok(0), Err(e) => Err(Error::GroveDB(e)), } @@ -73,32 +62,27 @@ impl Drive { #[cfg(test)] mod tests { + use super::*; + + use crate::common::helpers::setup::{setup_drive, setup_drive_with_initial_state_structure}; + mod get_aggregate_storage_fees_from_distribution_pool { - use crate::common::helpers::setup::{ - setup_drive, setup_drive_with_initial_state_structure, - }; + use super::*; use crate::drive::batch::GroveDbOpBatch; use crate::drive::fee_pools::pools_vec_path; - use crate::error::fee::FeeError; - use crate::error::Error; - use crate::fee_pools::epochs_root_tree_key_constants::KEY_STORAGE_FEE_POOL; - use grovedb::Element; #[test] fn test_error_if_epoch_is_not_initiated() { let drive = setup_drive(None); let transaction = drive.grove.start_transaction(); - match drive.get_aggregate_storage_fees_from_distribution_pool(Some(&transaction)) { - Ok(_) => assert!( - false, - "should not be able to get genesis time on uninit fee pools" - ), - Err(e) => match e { - Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => assert!(true), - _ => assert!(false, "invalid error type"), - }, - } + let result = + drive.get_aggregate_storage_fees_from_distribution_pool(Some(&transaction)); + + assert!(matches!( + result, + Err(Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_))) + )); } #[test] @@ -118,15 +102,13 @@ mod tests { .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - match drive.get_aggregate_storage_fees_from_distribution_pool(Some(&transaction)) { - Ok(_) => assert!(false, "should not be able to decode stored value"), - Err(e) => match e { - Error::Fee(FeeError::CorruptedStorageFeePoolInvalidItemLength(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = + drive.get_aggregate_storage_fees_from_distribution_pool(Some(&transaction)); + + assert!(matches!( + result, + Err(Error::Drive(DriveError::UnexpectedElementType(_))), + )); } #[test] diff --git a/packages/rs-drive/src/drive/fee_pools/unpaid_epoch.rs b/packages/rs-drive/src/drive/fee_pools/unpaid_epoch.rs index 74f292c4ec9..c298ea7b448 100644 --- a/packages/rs-drive/src/drive/fee_pools/unpaid_epoch.rs +++ b/packages/rs-drive/src/drive/fee_pools/unpaid_epoch.rs @@ -32,7 +32,7 @@ use crate::drive::fee_pools::pools_path; use crate::drive::Drive; -use crate::error::fee::FeeError; +use crate::error::drive::DriveError; use crate::error::Error; use crate::fee::epoch::EpochIndex; use crate::fee_pools::epochs_root_tree_key_constants::KEY_UNPAID_EPOCH_INDEX; @@ -47,51 +47,43 @@ impl Drive { .unwrap() .map_err(Error::GroveDB)?; - if let Element::Item(item, _) = element { - Ok(u16::from_be_bytes(item.as_slice().try_into().map_err( - |_| { - Error::Fee(FeeError::CorruptedUnpaidEpochIndexItemLength( - "item have an invalid length", - )) - }, - )?)) - } else { - Err(Error::Fee(FeeError::CorruptedUnpaidEpochIndexNotItem( + let Element::Item(encoded_epoch_index, _) = element else { + return Err(Error::Drive(DriveError::UnexpectedElementType( "must be an item", - ))) - } + ))); + }; + + let epoch_index = + EpochIndex::from_be_bytes(encoded_epoch_index.as_slice().try_into().map_err(|_| { + Error::Drive(DriveError::CorruptedSerialization( + "item have an invalid length", + )) + })?); + + Ok(epoch_index) } } #[cfg(test)] mod tests { + use super::*; + + use crate::common::helpers::setup::{setup_drive, setup_drive_with_initial_state_structure}; + mod get_unpaid_epoch_index { - use crate::common::helpers::setup::{ - setup_drive, setup_drive_with_initial_state_structure, - }; - use crate::drive::fee_pools::pools_path; - use crate::error; - use crate::error::fee::FeeError; - use crate::fee_pools::epochs_root_tree_key_constants::KEY_UNPAID_EPOCH_INDEX; - use grovedb::Element; + use super::*; #[test] fn test_error_if_fee_pools_tree_is_not_initiated() { let drive = setup_drive(None); let transaction = drive.grove.start_transaction(); - match drive.get_unpaid_epoch_index(Some(&transaction)) { - Ok(_) => assert!( - false, - "should not be able to get unpaid epoch if fee pools tree is not initialized" - ), - Err(e) => match e { - error::Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = drive.get_unpaid_epoch_index(Some(&transaction)); + + assert!(matches!( + result, + Err(Error::GroveDB(grovedb::Error::PathParentLayerNotFound(_))) + )); } #[test] @@ -123,15 +115,12 @@ mod tests { .unwrap() .expect("should insert invalid data"); - match drive.get_unpaid_epoch_index(Some(&transaction)) { - Ok(_) => assert!(false, "must be an error"), - Err(e) => match e { - error::Error::Fee(FeeError::CorruptedUnpaidEpochIndexNotItem(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = drive.get_unpaid_epoch_index(Some(&transaction)); + + assert!(matches!( + result, + Err(Error::Drive(DriveError::UnexpectedElementType(_))) + )); } #[test] @@ -151,15 +140,12 @@ mod tests { .unwrap() .expect("should insert invalid data"); - match drive.get_unpaid_epoch_index(Some(&transaction)) { - Ok(_) => assert!(false, "must be an error"), - Err(e) => match e { - error::Error::Fee(FeeError::CorruptedUnpaidEpochIndexItemLength(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = drive.get_unpaid_epoch_index(Some(&transaction)); + + assert!(matches!( + result, + Err(Error::Drive(DriveError::CorruptedSerialization(_))) + )); } } } diff --git a/packages/rs-drive/src/drive/initialization.rs b/packages/rs-drive/src/drive/initialization.rs index d3a27f24e0a..0b4b823c192 100644 --- a/packages/rs-drive/src/drive/initialization.rs +++ b/packages/rs-drive/src/drive/initialization.rs @@ -52,14 +52,14 @@ impl Drive { batch.add_insert_empty_tree(vec![], vec![RootTree::SpentAssetLockTransactions as u8]); - batch.add_insert_empty_tree(vec![], vec![RootTree::Pools as u8]); + batch.add_insert_empty_sum_tree(vec![], vec![RootTree::Pools as u8]); batch.add_insert_empty_tree(vec![], vec![RootTree::Misc as u8]); add_initial_withdrawal_state_structure_operations(&mut batch); // initialize the pools with epochs - add_create_fee_pool_trees_operations(&mut batch); + add_create_fee_pool_trees_operations(&mut batch)?; self.grove_apply_batch(batch, false, transaction)?; diff --git a/packages/rs-drive/src/error/drive.rs b/packages/rs-drive/src/error/drive.rs index 7aafac9e87b..164a90a2e6c 100644 --- a/packages/rs-drive/src/error/drive.rs +++ b/packages/rs-drive/src/error/drive.rs @@ -103,4 +103,8 @@ pub enum DriveError { /// Error #[error("batch is empty")] BatchIsEmpty(), + + /// Error + #[error("unexpected element type: {0}")] + UnexpectedElementType(&'static str), } diff --git a/packages/rs-drive/src/error/fee.rs b/packages/rs-drive/src/error/fee.rs index 70c0cdaa896..2946f635f45 100644 --- a/packages/rs-drive/src/error/fee.rs +++ b/packages/rs-drive/src/error/fee.rs @@ -10,60 +10,14 @@ pub enum FeeError { #[error("corrupted estimated layer info missing error: {0}")] CorruptedEstimatedLayerInfoMissing(String), - /// Corrupted storage fee not an item error - #[error("corrupted storage fee not an item error: {0}")] - CorruptedStorageFeeNotItem(&'static str), - /// Corrupted storage fee invalid item length error - #[error("corrupted storage fee invalid item length error: {0}")] - CorruptedStorageFeeInvalidItemLength(&'static str), - /// Corrupted processing fee not an item error - #[error("corrupted processing fee not an item error: {0}")] - CorruptedProcessingFeeNotItem(&'static str), - /// Corrupted processing fee invalid item length error - #[error("corrupted processing fee invalid item length error: {0}")] - CorruptedProcessingFeeInvalidItemLength(&'static str), - /// Corrupted start time not an item error - #[error("corrupted start time not an item error")] - CorruptedStartTimeNotItem(), - /// Corrupted start time invalid item length error - #[error("corrupted start time invalid item length error")] - CorruptedStartTimeLength(), - /// Corrupted start block height not an item error - #[error("corrupted start block height not an item")] - CorruptedStartBlockHeightNotItem(), - /// Corrupted start block height invalid item length error - #[error("corrupted start block height invalid item length")] - CorruptedStartBlockHeightItemLength(), - /// Corrupted proposer block count not an item error - #[error("corrupted proposer block count not an item error: {0}")] - CorruptedProposerBlockCountNotItem(&'static str), - /// Corrupted proposer block count invalid item length error - #[error("corrupted proposer block count invalid item length error: {0}")] - CorruptedProposerBlockCountItemLength(&'static str), - /// Corrupted storage fee pool not an item error - #[error("corrupted storage fee pool not an item error: {0}")] - CorruptedStorageFeePoolNotItem(&'static str), - /// Corrupted storage fee pool invalid item length error - #[error("corrupted storage fee pool invalid item length error: {0}")] - CorruptedStorageFeePoolInvalidItemLength(&'static str), - /// Corrupted multiplier not an item error - #[error("corrupted multiplier not an item error: {0}")] - CorruptedMultiplierNotItem(&'static str), - /// Corrupted multiplier invalid item length error - #[error("corrupted multiplier invalid item length error: {0}")] - CorruptedMultiplierInvalidItemLength(&'static str), - /// Corrupted unpaid epoch index invalid item length error - #[error("corrupted unpaid epoch index invalid item length error: {0}")] - CorruptedUnpaidEpochIndexItemLength(&'static str), - /// Corrupted unpaid epoch index not an item error - #[error("corrupted unpaid epoch index not an item error: {0}")] - CorruptedUnpaidEpochIndexNotItem(&'static str), /// Corrupted code execution error #[error("corrupted removed bytes from identities serialization error: {0}")] CorruptedRemovedBytesFromIdentitiesSerialization(&'static str), + /// Corrupted code execution error #[error("corrupted code execution error: {0}")] CorruptedCodeExecution(&'static str), + /// Decimal conversion error #[error("decimal conversion error: {0}")] DecimalConversion(&'static str), diff --git a/packages/rs-drive/src/fee/credits.rs b/packages/rs-drive/src/fee/credits.rs index 56a66fb9116..b71d719c6ef 100644 --- a/packages/rs-drive/src/fee/credits.rs +++ b/packages/rs-drive/src/fee/credits.rs @@ -39,10 +39,10 @@ // TODO: Should be moved to DPP when integration is done -use integer_encoding::VarInt; use crate::error::drive::DriveError; use crate::error::Error; use crate::fee::get_overflow_error; +use integer_encoding::VarInt; use rust_decimal::Decimal; /// Credits type @@ -81,11 +81,11 @@ impl Creditable for Credits { } fn from_vec_bytes(vec: Vec) -> Result { - Self::decode_var(vec.as_slice()).map(|(n, s)| n).ok_or( - Error::Drive(DriveError::CorruptedSerialization( + Self::decode_var(vec.as_slice()) + .map(|(n, _)| n) + .ok_or(Error::Drive(DriveError::CorruptedSerialization( "pending updates epoch index for must be u16", - )) - ) + ))) } fn to_vec_bytes(&self) -> Vec { @@ -102,11 +102,11 @@ impl Creditable for SignedCredits { } fn from_vec_bytes(vec: Vec) -> Result { - Self::decode_var(vec.as_slice()).map(|(n, s)| n).ok_or( - Error::Drive(DriveError::CorruptedSerialization( + Self::decode_var(vec.as_slice()) + .map(|(n, _)| n) + .ok_or(Error::Drive(DriveError::CorruptedSerialization( "pending updates epoch index for must be u16", - )) - ) + ))) } fn to_vec_bytes(&self) -> Vec { diff --git a/packages/rs-drive/src/fee/epoch/distribution.rs b/packages/rs-drive/src/fee/epoch/distribution.rs index a715f1a109f..2e93f879b42 100644 --- a/packages/rs-drive/src/fee/epoch/distribution.rs +++ b/packages/rs-drive/src/fee/epoch/distribution.rs @@ -39,7 +39,9 @@ use crate::error::fee::FeeError; use crate::error::Error; use crate::fee::credits::{Creditable, Credits, SignedCredits}; -use crate::fee::epoch::{EpochIndex, CreditsPerEpoch, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS, SignedCreditsPerEpoch}; +use crate::fee::epoch::{ + EpochIndex, SignedCreditsPerEpoch, EPOCHS_PER_YEAR, PERPETUAL_STORAGE_YEARS, +}; use crate::fee::get_overflow_error; use rust_decimal::prelude::*; use rust_decimal::Decimal; @@ -62,37 +64,59 @@ pub const FEE_DISTRIBUTION_TABLE: [Decimal; PERPETUAL_STORAGE_YEARS as usize] = dec!(0.00325), dec!(0.00275), dec!(0.00225), dec!(0.00175), dec!(0.00125), ]; -/// Distributes storage fees to epochs into `CreditsPerEpoch` and returns leftovers -/// It skips epochs up to specified `skip_up_to_epoch_index` +type DistributionAmount = Credits; +type DistributionLeftovers = Credits; + +/// Distributes storage fees to epochs into `SignedCreditsPerEpoch` and returns leftovers pub fn distribute_storage_fee_to_epochs_collection( credits_per_epochs: &mut SignedCreditsPerEpoch, storage_fee: Credits, - remove_instead: bool, start_epoch_index: EpochIndex, - skip_distribution_until_epoch_index: Option, ) -> Result { distribution_storage_fee_to_epochs_map( storage_fee, start_epoch_index, |epoch_index, epoch_fee_share| { - if let Some(skip_epoch_index) = skip_distribution_until_epoch_index { - if epoch_index < skip_epoch_index { - return Ok(()); - } - } + let epoch_credits: SignedCredits = + credits_per_epochs.get(&epoch_index).map_or(0, |i| *i); + + let result_storage_fee: SignedCredits = epoch_credits + .checked_add(epoch_fee_share.to_signed()?) + .ok_or_else(|| { + get_overflow_error("updated epoch credits are not fitting to credits max size") + })?; + + credits_per_epochs.insert(epoch_index, result_storage_fee); - let epoch_credits: SignedCredits = credits_per_epochs - .get(&epoch_index) - .map_or(0, |i| *i); + Ok(()) + }, + ) +} + +/// Distributes refunds to epochs into `SignedCreditsPerEpoch` and returns leftovers +/// It skips epochs up to specified `skip_until_epoch_index` +pub fn distribute_refunds_to_epochs_collection( + credits_per_epochs: &mut SignedCreditsPerEpoch, + storage_fee: Credits, + start_epoch_index: EpochIndex, + skip_until_epoch_index: EpochIndex, +) -> Result { + distribution_storage_fee_to_epochs_map( + storage_fee, + start_epoch_index, + |epoch_index, epoch_fee_share| { + if epoch_index < skip_until_epoch_index { + return Ok(()); + } - let result_storage_fee: SignedCredits = if !remove_instead { // we add - epoch_credits - .checked_add(epoch_fee_share.to_signed()?) - } else { - epoch_credits - .checked_sub(epoch_fee_share.to_signed()?) + let epoch_credits: SignedCredits = + credits_per_epochs.get(&epoch_index).map_or(0, |i| *i); - }.ok_or_else(|| get_overflow_error("storage fees are not fitting in a u64"))?; + let result_storage_fee: SignedCredits = epoch_credits + .checked_sub(epoch_fee_share.to_signed()?) + .ok_or_else(|| { + get_overflow_error("updated epoch credits are not fitting to credits min size") + })?; credits_per_epochs.insert(epoch_index, result_storage_fee); @@ -101,9 +125,6 @@ pub fn distribute_storage_fee_to_epochs_collection( ) } -type DistributionAmount = Credits; -type DistributionLeftovers = Credits; - /// Calculates leftovers and amount of credits by distributing storage fees to epochs pub fn calculate_storage_fee_distribution_amount_and_leftovers( storage_fee: Credits, @@ -132,7 +153,7 @@ pub fn calculate_storage_fee_distribution_amount_and_leftovers( fn distribution_storage_fee_to_epochs_map( storage_fee: Credits, start_epoch_index: EpochIndex, - mut f: F, + mut map_function: F, ) -> Result where F: FnMut(EpochIndex, Credits) -> Result<(), Error>, @@ -162,7 +183,7 @@ where let year_start_epoch_index = start_epoch_index + EPOCHS_PER_YEAR * year; for epoch_index in year_start_epoch_index..year_start_epoch_index + EPOCHS_PER_YEAR { - f(epoch_index, epoch_fee_share)?; + map_function(epoch_index, epoch_fee_share)?; distribution_leftover_credits = distribution_leftover_credits .checked_sub(epoch_fee_share) @@ -181,6 +202,7 @@ mod tests { use crate::fee::credits::{Creditable, MAX_CREDITS}; use crate::fee::epoch::GENESIS_EPOCH_INDEX; + use crate::fee::epoch::PERPETUAL_STORAGE_EPOCHS; mod fee_distribution_table { use super::*; @@ -203,26 +225,28 @@ mod tests { } } - mod distribute_storage_fee_to_epochs_collection { + mod distribution_storage_fee_to_epochs_map { use super::*; - use crate::fee::epoch::PERPETUAL_STORAGE_EPOCHS; #[test] fn should_distribute_nothing_if_storage_fee_are_zero() { - let mut credits_per_epochs = SignedCreditsPerEpoch::default(); + let mut calls = 0; - let leftovers = distribute_storage_fee_to_epochs_collection( - &mut credits_per_epochs, - 0, - false, - GENESIS_EPOCH_INDEX, - None, - ) - .expect("should distribute storage fee"); + let leftovers = + distribution_storage_fee_to_epochs_map(0, GENESIS_EPOCH_INDEX, |_, _| { + calls += 1; + + Ok(()) + }) + .expect("should distribute storage fee"); - assert_eq!(credits_per_epochs.len(), 0); + assert_eq!(calls, 0); assert_eq!(leftovers, 0); } + } + + mod distribute_storage_fee_to_epochs_collection { + use super::*; #[test] fn should_distribute_max_credits_value_without_overflow() { @@ -233,9 +257,7 @@ mod tests { let leftovers = distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, storage_fee, - false, GENESIS_EPOCH_INDEX, - None, ) .expect("should distribute storage fee"); @@ -253,9 +275,7 @@ mod tests { let leftovers = distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, storage_fee, - false, current_epoch_index, - None, ) .expect("should distribute storage fee"); @@ -264,7 +284,7 @@ mod tests { // compare them with reference table #[rustfmt::skip] - let reference_fees: [Credits; PERPETUAL_STORAGE_EPOCHS as usize] = [ + let reference_fees: [SignedCredits; PERPETUAL_STORAGE_EPOCHS as usize] = [ 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2300, 2300, @@ -334,16 +354,13 @@ mod tests { ]; assert_eq!( - credits_per_epochs - .clone() - .into_values() - .collect::>(), + credits_per_epochs.clone().into_values().collect::>(), reference_fees ); let total_distributed: SignedCredits = credits_per_epochs.values().sum(); - assert_eq!(total_distributed + leftovers, storage_fee); + assert_eq!(total_distributed.to_unsigned() + leftovers, storage_fee); /* @@ -354,28 +371,28 @@ mod tests { let leftovers = distribute_storage_fee_to_epochs_collection( &mut credits_per_epochs, storage_fee, - false, current_epoch_index, - None, ) .expect("should distribute storage fee"); // assert that all the values doubled meaning that distribution is reproducible assert_eq!( - credits_per_epochs - .into_values() - .collect::>(), + credits_per_epochs.into_values().collect::>(), reference_fees .into_iter() .map(|val| val * 2) - .collect::>() + .collect::>() ); assert_eq!(leftovers, 180); } + } + + mod distribute_refunds_to_epochs_collection { + use super::*; #[test] - fn should_add_to_collection_from_specific_epoch() { + fn should_deduct_refunds_from_collection_since_specific_epoch() { let storage_fee: Credits = 1000000; let start_epoch_index: EpochIndex = 0; @@ -383,12 +400,11 @@ mod tests { let mut credits_per_epochs = SignedCreditsPerEpoch::default(); - let leftovers = distribute_storage_fee_to_epochs_collection( + let leftovers = distribute_refunds_to_epochs_collection( &mut credits_per_epochs, storage_fee, - false, start_epoch_index, - Some(SKIP_UP_TO_EPOCH_INDEX), + SKIP_UP_TO_EPOCH_INDEX, ) .expect("should distribute storage fee"); @@ -396,93 +412,100 @@ mod tests { assert_eq!(leftovers, 180); // compare them with reference table - #[rustfmt::skip] - let reference_fees: [SignedCredits; (PERPETUAL_STORAGE_EPOCHS - SKIP_UP_TO_EPOCH_INDEX) as usize] = [ - 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, 2300, - 2300, 2300, 2300, 2300, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, - 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2200, 2100, 2100, 2100, 2100, - 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, - 2100, 2100, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, - 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 1925, 1925, 1925, 1925, 1925, 1925, - 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, 1925, - 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, 1850, - 1850, 1850, 1850, 1850, 1850, 1850, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, - 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1775, 1700, 1700, - 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, 1700, - 1700, 1700, 1700, 1700, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, - 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1625, 1550, 1550, 1550, 1550, - 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, 1550, - 1550, 1550, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, - 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1475, 1425, 1425, 1425, 1425, 1425, 1425, - 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, 1425, - 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, 1375, - 1375, 1375, 1375, 1375, 1375, 1375, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, - 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1275, 1275, - 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, 1275, - 1275, 1275, 1275, 1275, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, - 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1225, 1175, 1175, 1175, 1175, - 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, 1175, - 1175, 1175, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, - 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1075, 1075, 1075, 1075, 1075, 1075, - 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, 1075, - 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, 1025, - 1025, 1025, 1025, 1025, 1025, 1025, 975, 975, 975, 975, 975, 975, 975, 975, 975, - 975, 975, 975, 975, 975, 975, 975, 975, 975, 975, 975, 937, 937, 937, 937, 937, - 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 937, 900, - 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, 900, - 900, 900, 900, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, 862, - 862, 862, 862, 862, 862, 862, 862, 825, 825, 825, 825, 825, 825, 825, 825, 825, - 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, 825, 787, 787, 787, 787, 787, - 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 787, 750, - 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, - 750, 750, 750, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, 712, - 712, 712, 712, 712, 712, 712, 712, 675, 675, 675, 675, 675, 675, 675, 675, 675, - 675, 675, 675, 675, 675, 675, 675, 675, 675, 675, 675, 637, 637, 637, 637, 637, - 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 637, 600, - 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, - 600, 600, 600, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, 562, - 562, 562, 562, 562, 562, 562, 562, 525, 525, 525, 525, 525, 525, 525, 525, 525, - 525, 525, 525, 525, 525, 525, 525, 525, 525, 525, 525, 487, 487, 487, 487, 487, - 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 487, 450, - 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, 450, - 450, 450, 450, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, 412, - 412, 412, 412, 412, 412, 412, 412, 375, 375, 375, 375, 375, 375, 375, 375, 375, - 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, 337, 337, 337, 337, 337, - 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 337, 300, - 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, - 300, 300, 300, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, - 262, 262, 262, 262, 262, 262, 262, 237, 237, 237, 237, 237, 237, 237, 237, 237, - 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, 212, 212, 212, 212, 212, - 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, 187, - 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, - 187, 187, 187, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, 162, - 162, 162, 162, 162, 162, 162, 162, 137, 137, 137, 137, 137, 137, 137, 137, 137, - 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 137, 112, 112, 112, 112, 112, - 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 112, 87, - 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 87, 62, - 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62, 62 + let reference_fees: [SignedCredits; + (PERPETUAL_STORAGE_EPOCHS - SKIP_UP_TO_EPOCH_INDEX) as usize] = [ + -2300, -2300, -2300, -2300, -2300, -2300, -2300, -2300, -2300, -2300, -2300, -2300, + -2300, -2300, -2300, -2300, -2300, -2300, -2200, -2200, -2200, -2200, -2200, -2200, + -2200, -2200, -2200, -2200, -2200, -2200, -2200, -2200, -2200, -2200, -2200, -2200, + -2200, -2200, -2100, -2100, -2100, -2100, -2100, -2100, -2100, -2100, -2100, -2100, + -2100, -2100, -2100, -2100, -2100, -2100, -2100, -2100, -2100, -2100, -2000, -2000, + -2000, -2000, -2000, -2000, -2000, -2000, -2000, -2000, -2000, -2000, -2000, -2000, + -2000, -2000, -2000, -2000, -2000, -2000, -1925, -1925, -1925, -1925, -1925, -1925, + -1925, -1925, -1925, -1925, -1925, -1925, -1925, -1925, -1925, -1925, -1925, -1925, + -1925, -1925, -1850, -1850, -1850, -1850, -1850, -1850, -1850, -1850, -1850, -1850, + -1850, -1850, -1850, -1850, -1850, -1850, -1850, -1850, -1850, -1850, -1775, -1775, + -1775, -1775, -1775, -1775, -1775, -1775, -1775, -1775, -1775, -1775, -1775, -1775, + -1775, -1775, -1775, -1775, -1775, -1775, -1700, -1700, -1700, -1700, -1700, -1700, + -1700, -1700, -1700, -1700, -1700, -1700, -1700, -1700, -1700, -1700, -1700, -1700, + -1700, -1700, -1625, -1625, -1625, -1625, -1625, -1625, -1625, -1625, -1625, -1625, + -1625, -1625, -1625, -1625, -1625, -1625, -1625, -1625, -1625, -1625, -1550, -1550, + -1550, -1550, -1550, -1550, -1550, -1550, -1550, -1550, -1550, -1550, -1550, -1550, + -1550, -1550, -1550, -1550, -1550, -1550, -1475, -1475, -1475, -1475, -1475, -1475, + -1475, -1475, -1475, -1475, -1475, -1475, -1475, -1475, -1475, -1475, -1475, -1475, + -1475, -1475, -1425, -1425, -1425, -1425, -1425, -1425, -1425, -1425, -1425, -1425, + -1425, -1425, -1425, -1425, -1425, -1425, -1425, -1425, -1425, -1425, -1375, -1375, + -1375, -1375, -1375, -1375, -1375, -1375, -1375, -1375, -1375, -1375, -1375, -1375, + -1375, -1375, -1375, -1375, -1375, -1375, -1325, -1325, -1325, -1325, -1325, -1325, + -1325, -1325, -1325, -1325, -1325, -1325, -1325, -1325, -1325, -1325, -1325, -1325, + -1325, -1325, -1275, -1275, -1275, -1275, -1275, -1275, -1275, -1275, -1275, -1275, + -1275, -1275, -1275, -1275, -1275, -1275, -1275, -1275, -1275, -1275, -1225, -1225, + -1225, -1225, -1225, -1225, -1225, -1225, -1225, -1225, -1225, -1225, -1225, -1225, + -1225, -1225, -1225, -1225, -1225, -1225, -1175, -1175, -1175, -1175, -1175, -1175, + -1175, -1175, -1175, -1175, -1175, -1175, -1175, -1175, -1175, -1175, -1175, -1175, + -1175, -1175, -1125, -1125, -1125, -1125, -1125, -1125, -1125, -1125, -1125, -1125, + -1125, -1125, -1125, -1125, -1125, -1125, -1125, -1125, -1125, -1125, -1075, -1075, + -1075, -1075, -1075, -1075, -1075, -1075, -1075, -1075, -1075, -1075, -1075, -1075, + -1075, -1075, -1075, -1075, -1075, -1075, -1025, -1025, -1025, -1025, -1025, -1025, + -1025, -1025, -1025, -1025, -1025, -1025, -1025, -1025, -1025, -1025, -1025, -1025, + -1025, -1025, -975, -975, -975, -975, -975, -975, -975, -975, -975, -975, -975, + -975, -975, -975, -975, -975, -975, -975, -975, -975, -937, -937, -937, -937, -937, + -937, -937, -937, -937, -937, -937, -937, -937, -937, -937, -937, -937, -937, -937, + -937, -900, -900, -900, -900, -900, -900, -900, -900, -900, -900, -900, -900, -900, + -900, -900, -900, -900, -900, -900, -900, -862, -862, -862, -862, -862, -862, -862, + -862, -862, -862, -862, -862, -862, -862, -862, -862, -862, -862, -862, -862, -825, + -825, -825, -825, -825, -825, -825, -825, -825, -825, -825, -825, -825, -825, -825, + -825, -825, -825, -825, -825, -787, -787, -787, -787, -787, -787, -787, -787, -787, + -787, -787, -787, -787, -787, -787, -787, -787, -787, -787, -787, -750, -750, -750, + -750, -750, -750, -750, -750, -750, -750, -750, -750, -750, -750, -750, -750, -750, + -750, -750, -750, -712, -712, -712, -712, -712, -712, -712, -712, -712, -712, -712, + -712, -712, -712, -712, -712, -712, -712, -712, -712, -675, -675, -675, -675, -675, + -675, -675, -675, -675, -675, -675, -675, -675, -675, -675, -675, -675, -675, -675, + -675, -637, -637, -637, -637, -637, -637, -637, -637, -637, -637, -637, -637, -637, + -637, -637, -637, -637, -637, -637, -637, -600, -600, -600, -600, -600, -600, -600, + -600, -600, -600, -600, -600, -600, -600, -600, -600, -600, -600, -600, -600, -562, + -562, -562, -562, -562, -562, -562, -562, -562, -562, -562, -562, -562, -562, -562, + -562, -562, -562, -562, -562, -525, -525, -525, -525, -525, -525, -525, -525, -525, + -525, -525, -525, -525, -525, -525, -525, -525, -525, -525, -525, -487, -487, -487, + -487, -487, -487, -487, -487, -487, -487, -487, -487, -487, -487, -487, -487, -487, + -487, -487, -487, -450, -450, -450, -450, -450, -450, -450, -450, -450, -450, -450, + -450, -450, -450, -450, -450, -450, -450, -450, -450, -412, -412, -412, -412, -412, + -412, -412, -412, -412, -412, -412, -412, -412, -412, -412, -412, -412, -412, -412, + -412, -375, -375, -375, -375, -375, -375, -375, -375, -375, -375, -375, -375, -375, + -375, -375, -375, -375, -375, -375, -375, -337, -337, -337, -337, -337, -337, -337, + -337, -337, -337, -337, -337, -337, -337, -337, -337, -337, -337, -337, -337, -300, + -300, -300, -300, -300, -300, -300, -300, -300, -300, -300, -300, -300, -300, -300, + -300, -300, -300, -300, -300, -262, -262, -262, -262, -262, -262, -262, -262, -262, + -262, -262, -262, -262, -262, -262, -262, -262, -262, -262, -262, -237, -237, -237, + -237, -237, -237, -237, -237, -237, -237, -237, -237, -237, -237, -237, -237, -237, + -237, -237, -237, -212, -212, -212, -212, -212, -212, -212, -212, -212, -212, -212, + -212, -212, -212, -212, -212, -212, -212, -212, -212, -187, -187, -187, -187, -187, + -187, -187, -187, -187, -187, -187, -187, -187, -187, -187, -187, -187, -187, -187, + -187, -162, -162, -162, -162, -162, -162, -162, -162, -162, -162, -162, -162, -162, + -162, -162, -162, -162, -162, -162, -162, -137, -137, -137, -137, -137, -137, -137, + -137, -137, -137, -137, -137, -137, -137, -137, -137, -137, -137, -137, -137, -112, + -112, -112, -112, -112, -112, -112, -112, -112, -112, -112, -112, -112, -112, -112, + -112, -112, -112, -112, -112, -87, -87, -87, -87, -87, -87, -87, -87, -87, -87, + -87, -87, -87, -87, -87, -87, -87, -87, -87, -87, -62, -62, -62, -62, -62, -62, + -62, -62, -62, -62, -62, -62, -62, -62, -62, -62, -62, -62, -62, -62, ]; - let skipped_reference_fees: [SignedCredits; SKIP_UP_TO_EPOCH_INDEX as usize] = [ + let skipped_reference_fees: [Credits; SKIP_UP_TO_EPOCH_INDEX as usize] = [ 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2500, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2400, 2300, 2300, ]; assert_eq!( - credits_per_epochs - .clone() - .into_values() - .collect::>(), + credits_per_epochs.clone().into_values().collect::>(), reference_fees ); let total_distributed: SignedCredits = credits_per_epochs.values().sum(); assert_eq!( - total_distributed + total_distributed.to_unsigned() + leftovers - + skipped_reference_fees.into_iter().sum::(), + + skipped_reference_fees.into_iter().sum::(), storage_fee ); } diff --git a/packages/rs-drive/src/fee/op.rs b/packages/rs-drive/src/fee/op.rs index b4ad4b65535..8036fd27bbd 100644 --- a/packages/rs-drive/src/fee/op.rs +++ b/packages/rs-drive/src/fee/op.rs @@ -41,11 +41,9 @@ use enum_map::Enum; use grovedb::batch::key_info::KeyInfo; use grovedb::batch::KeyInfoPath; use grovedb::{batch::GroveDbOp, Element}; -use std::collections::BTreeMap; use crate::drive::flags::StorageFlags; use crate::error::drive::DriveError; -use crate::error::fee::FeeError; use crate::error::Error; use crate::fee::default_costs::{ STORAGE_DISK_USAGE_CREDIT_PER_BYTE, STORAGE_LOAD_CREDIT_PER_BYTE, diff --git a/packages/rs-drive/src/fee/result/refunds.rs b/packages/rs-drive/src/fee/result/refunds.rs index eb1405b75bb..07012c9debe 100644 --- a/packages/rs-drive/src/fee/result/refunds.rs +++ b/packages/rs-drive/src/fee/result/refunds.rs @@ -34,7 +34,7 @@ use crate::error::fee::FeeError; use crate::error::Error; -use crate::fee::credits::{Credits, SignedCredits}; +use crate::fee::credits::Credits; use crate::fee::default_costs::STORAGE_DISK_USAGE_CREDIT_PER_BYTE; use crate::fee::epoch::CreditsPerEpoch; use crate::fee::get_overflow_error; diff --git a/packages/rs-drive/src/fee_pools/epochs/operations_factory.rs b/packages/rs-drive/src/fee_pools/epochs/operations_factory.rs index a0515141e67..0874681b6e5 100644 --- a/packages/rs-drive/src/fee_pools/epochs/operations_factory.rs +++ b/packages/rs-drive/src/fee_pools/epochs/operations_factory.rs @@ -36,11 +36,12 @@ use crate::drive::batch::GroveDbOpBatch; use crate::drive::fee_pools::pools_vec_path; use crate::drive::Drive; use crate::error::Error; +use crate::fee::credits::{Creditable, Credits}; use crate::fee_pools::epochs::epoch_key_constants::{ - KEY_FEE_MULTIPLIER, KEY_POOL_PROCESSING_FEES, KEY_POOL_STORAGE_FEES, KEY_START_BLOCK_HEIGHT, - KEY_START_TIME, + KEY_FEE_MULTIPLIER, KEY_POOL_PROCESSING_FEES, KEY_POOL_STORAGE_FEES, KEY_PROPOSERS, + KEY_START_BLOCK_HEIGHT, KEY_START_TIME, }; -use crate::fee_pools::epochs::{epoch_key_constants, Epoch}; +use crate::fee_pools::epochs::Epoch; use grovedb::batch::GroveDbOp; use grovedb::{Element, TransactionArg}; @@ -54,32 +55,37 @@ impl Epoch { transaction: TransactionArg, ) -> Result { // get current proposer's block count - let proposed_block_count = match cached_previous_block_count { - Some(block_count) => block_count, - None => drive + let proposed_block_count = if let Some(block_count) = cached_previous_block_count { + block_count + } else { + drive .get_epochs_proposer_block_count(self, proposer_pro_tx_hash, transaction) .or_else(|e| match e { Error::GroveDB(grovedb::Error::PathKeyNotFound(_)) => Ok(0u64), _ => Err(e), - })?, + })? }; - Ok(self - .update_proposer_block_count_operation(proposer_pro_tx_hash, proposed_block_count + 1)) + let operation = self + .update_proposer_block_count_operation(proposer_pro_tx_hash, proposed_block_count + 1); + + Ok(operation) } /// Adds to the groveDB op batch operations to insert an empty tree into the epoch pub fn add_init_empty_without_storage_operations(&self, batch: &mut GroveDbOpBatch) { - batch.add_insert_empty_tree(pools_vec_path(), self.key.to_vec()); + batch.add_insert_empty_sum_tree(pools_vec_path(), self.key.to_vec()); } /// Adds to the groveDB op batch operations to insert an empty tree into the epoch /// and sets the storage distribution pool to 0. - pub fn add_init_empty_operations(&self, batch: &mut GroveDbOpBatch) { + pub fn add_init_empty_operations(&self, batch: &mut GroveDbOpBatch) -> Result<(), Error> { self.add_init_empty_without_storage_operations(batch); // init storage fee item to 0 - batch.push(self.update_storage_fee_pool_operation(0)); + batch.push(self.update_storage_fee_pool_operation(0)?); + + Ok(()) } /// Adds to the groveDB op batch initialization operations for the epoch. @@ -136,12 +142,15 @@ impl Epoch { } /// Returns a groveDB op which updates the epoch processing credits for distribution. - pub fn update_processing_fee_pool_operation(&self, processing_fee: u64) -> GroveDbOp { - GroveDbOp::insert_op( + pub fn update_processing_fee_pool_operation( + &self, + processing_fee: Credits, + ) -> Result { + Ok(GroveDbOp::insert_op( self.get_vec_path(), KEY_POOL_PROCESSING_FEES.to_vec(), - Element::new_item(processing_fee.to_be_bytes().to_vec()), - ) + Element::new_sum_item(processing_fee.to_signed()?), + )) } /// Returns a groveDB op which deletes the epoch processing credits for distribution tree. @@ -150,12 +159,15 @@ impl Epoch { } /// Returns a groveDB op which updates the epoch storage credits for distribution. - pub fn update_storage_fee_pool_operation(&self, storage_fee: u64) -> GroveDbOp { - GroveDbOp::insert_op( + pub fn update_storage_fee_pool_operation( + &self, + storage_fee: Credits, + ) -> Result { + Ok(GroveDbOp::insert_op( self.get_vec_path(), KEY_POOL_STORAGE_FEES.to_vec(), - Element::new_item(storage_fee.to_be_bytes().to_vec()), - ) + Element::new_sum_item(storage_fee.to_signed()?), + )) } /// Returns a groveDB op which deletes the epoch storage credits for distribution tree. @@ -180,18 +192,14 @@ impl Epoch { pub fn init_proposers_tree_operation(&self) -> GroveDbOp { GroveDbOp::insert_op( self.get_vec_path(), - epoch_key_constants::KEY_PROPOSERS.to_vec(), + KEY_PROPOSERS.to_vec(), Element::empty_tree(), ) } /// Returns a groveDB op which deletes the epoch proposers tree. pub fn delete_proposers_tree_operation(&self) -> GroveDbOp { - GroveDbOp::delete_tree_op( - self.get_vec_path(), - epoch_key_constants::KEY_PROPOSERS.to_vec(), - false, - ) + GroveDbOp::delete_tree_op(self.get_vec_path(), KEY_PROPOSERS.to_vec(), false) } /// Adds a groveDB op to the batch which deletes the given epoch proposers from the proposers tree. @@ -208,22 +216,23 @@ impl Epoch { #[cfg(test)] mod tests { - use crate::common::helpers::setup::setup_drive_with_initial_state_structure; - use crate::drive::batch::GroveDbOpBatch; - use crate::fee_pools::epochs::Epoch; + use super::*; + use crate::common::helpers::setup::{setup_drive, setup_drive_with_initial_state_structure}; use chrono::Utc; mod increment_proposer_block_count_operation { + use super::*; + #[test] fn test_increment_block_count_to_1_if_proposers_tree_is_not_committed() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); let pro_tx_hash: [u8; 32] = rand::random(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); batch.push(epoch.init_proposers_tree_operation()); @@ -251,14 +260,14 @@ mod tests { #[test] fn test_existing_block_count_is_incremented() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); let pro_tx_hash: [u8; 32] = rand::random(); - let epoch = super::Epoch::new(1); + let epoch = Epoch::new(1); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); batch.push(epoch.init_proposers_tree_operation()); @@ -267,7 +276,7 @@ mod tests { .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); batch.push(epoch.update_proposer_block_count_operation(&pro_tx_hash, 1)); @@ -276,7 +285,7 @@ mod tests { .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); batch.push( epoch @@ -302,41 +311,41 @@ mod tests { } mod add_init_empty_operations { - use crate::common::helpers::setup::setup_drive; - use crate::error; + use super::*; #[test] fn test_error_if_fee_pools_not_initialized() { let drive = setup_drive(None); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(1042); + let epoch = Epoch::new(1042); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); - epoch.add_init_empty_operations(&mut batch); + epoch + .add_init_empty_operations(&mut batch) + .expect("should init empty epoch"); - match drive.grove_apply_batch(batch, false, Some(&transaction)) { - Ok(_) => assert!(false, "should not be able to init epochs without FeePools"), - Err(e) => match e { - error::Error::GroveDB(grovedb::Error::InvalidPath(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = drive.grove_apply_batch(batch, false, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::GroveDB(grovedb::Error::InvalidPath(_))) + )); } #[test] fn test_values_are_set() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(1042); + let epoch = Epoch::new(1042); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); - epoch.add_init_empty_operations(&mut batch); + epoch + .add_init_empty_operations(&mut batch) + .expect("should init empty epoch"); drive .grove_apply_batch(batch, false, Some(&transaction)) @@ -351,21 +360,24 @@ mod tests { } mod add_init_current_operations { + use super::*; #[test] fn test_values_are_set() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(1042); + let epoch = Epoch::new(1042); let multiplier = 42.0; let start_time = 1; let start_block_height = 2; - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); - epoch.add_init_empty_operations(&mut batch); + epoch + .add_init_empty_operations(&mut batch) + .expect("should init empty epoch"); epoch.add_init_current_operations( multiplier, @@ -409,17 +421,16 @@ mod tests { } mod add_mark_as_paid_operations { - use crate::error; - use crate::fee_pools::epochs::epoch_key_constants; + use super::*; #[test] fn test_values_are_deleted() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); epoch.add_init_current_operations(1.0, 2, 3, &mut batch); @@ -428,7 +439,7 @@ mod tests { .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); epoch.add_mark_as_paid_operations(&mut batch); @@ -436,56 +447,49 @@ mod tests { .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - match drive + let result = drive .grove .get( epoch.get_path(), - epoch_key_constants::KEY_PROPOSERS.as_slice(), + KEY_PROPOSERS.as_slice(), Some(&transaction), ) - .unwrap() - { - Ok(_) => assert!(false, "should not be able to get proposers"), - Err(e) => match e { - grovedb::Error::PathKeyNotFound(_) => assert!(true), - _ => assert!(false, "invalid error type"), - }, - } + .unwrap(); - match drive.get_epoch_processing_credits_for_distribution(&epoch, Some(&transaction)) { - Ok(_) => assert!(false, "should not be able to get processing fee"), - Err(e) => match e { - error::Error::GroveDB(grovedb::Error::PathKeyNotFound(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + assert!(matches!(result, Err(grovedb::Error::PathKeyNotFound(_)))); - match drive.get_epoch_storage_credits_for_distribution(&epoch, Some(&transaction)) { - Ok(_) => assert!(false, "should not be able to get storage fee"), - Err(e) => match e { - error::Error::GroveDB(grovedb::Error::PathKeyNotFound(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = + drive.get_epoch_processing_credits_for_distribution(&epoch, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::GroveDB(grovedb::Error::PathKeyNotFound(_))) + )); + + let result = + drive.get_epoch_storage_credits_for_distribution(&epoch, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::GroveDB(grovedb::Error::PathKeyNotFound(_))) + )); } } mod update_proposer_block_count { + use super::*; + #[test] fn test_value_is_set() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); let pro_tx_hash: [u8; 32] = rand::random(); let block_count = 42; - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); batch.push(epoch.init_proposers_tree_operation()); @@ -508,7 +512,7 @@ mod tests { let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch_tree = super::Epoch::new(0); + let epoch_tree = Epoch::new(0); let start_time_ms: u64 = Utc::now().timestamp_millis() as u64; @@ -550,41 +554,39 @@ mod tests { } mod update_epoch_processing_credits_for_distribution { - use crate::error; + use super::*; #[test] fn test_error_if_epoch_tree_is_not_initiated() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(7000); + let epoch = Epoch::new(7000); - let op = epoch.update_processing_fee_pool_operation(42); + let op = epoch + .update_processing_fee_pool_operation(42) + .expect("should return operation"); - match drive.grove_apply_operation(op, false, Some(&transaction)) { - Ok(_) => assert!( - false, - "should not be able to update processing fee on uninit epochs pool" - ), - Err(e) => match e { - error::Error::GroveDB(grovedb::Error::InvalidPath(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + let result = drive.grove_apply_operation(op, false, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::GroveDB(grovedb::Error::InvalidPath(_))) + )); } #[test] fn test_value_is_set() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); let processing_fee: u64 = 42; - let op = epoch.update_processing_fee_pool_operation(42); + let op = epoch + .update_processing_fee_pool_operation(42) + .expect("should return operation"); drive .grove_apply_operation(op, false, Some(&transaction)) @@ -599,41 +601,39 @@ mod tests { } mod update_epoch_storage_credits_for_distribution { - use crate::error; + use super::*; #[test] fn test_error_if_epoch_tree_is_not_initiated() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(7000); + let epoch = Epoch::new(7000); - let op = epoch.update_storage_fee_pool_operation(42); + let op = epoch + .update_storage_fee_pool_operation(42) + .expect("should return operation"); - match drive.grove_apply_operation(op, false, Some(&transaction)) { - Ok(_) => assert!( - false, - "should not be able to update storage fee on uninit epochs pool" - ), - Err(e) => match e { - error::Error::GroveDB(grovedb::Error::InvalidPath(_)) => { - assert!(true) - } - _ => assert!(false, "{}", format!("invalid error type {}", e)), - }, - } + let result = drive.grove_apply_operation(op, false, Some(&transaction)); + + assert!(matches!( + result, + Err(Error::GroveDB(grovedb::Error::InvalidPath(_))) + )); } #[test] fn test_value_is_set() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); let storage_fee = 42; - let op = epoch.update_storage_fee_pool_operation(storage_fee); + let op = epoch + .update_storage_fee_pool_operation(storage_fee) + .expect("should return operation"); drive .grove_apply_operation(op, false, Some(&transaction)) @@ -648,16 +648,16 @@ mod tests { } mod delete_proposers_tree { - use crate::fee_pools::epochs::epoch_key_constants::KEY_PROPOSERS; + use super::*; #[test] fn test_values_has_been_deleted() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); batch.push(epoch.init_proposers_tree_operation()); @@ -666,7 +666,7 @@ mod tests { .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); batch.push(epoch.delete_proposers_tree_operation()); @@ -693,14 +693,16 @@ mod tests { } mod delete_proposers { + use super::*; + #[test] fn test_values_are_being_deleted() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); - let epoch = super::Epoch::new(0); + let epoch = Epoch::new(0); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); batch.push(epoch.init_proposers_tree_operation()); @@ -711,7 +713,7 @@ mod tests { let pro_tx_hashes: Vec<[u8; 32]> = (0..10).map(|_| rand::random()).collect(); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); for pro_tx_hash in pro_tx_hashes.iter() { batch.push(epoch.update_proposer_block_count_operation(pro_tx_hash, 1)); @@ -746,7 +748,7 @@ mod tests { awaited_result.remove(0); awaited_result.remove(1); - let mut batch = super::GroveDbOpBatch::new(); + let mut batch = GroveDbOpBatch::new(); epoch.add_delete_proposers_operations(deleted_pro_tx_hashes, &mut batch); diff --git a/packages/rs-drive/src/fee_pools/epochs/paths.rs b/packages/rs-drive/src/fee_pools/epochs/paths.rs index 939865817de..51144acb7ff 100644 --- a/packages/rs-drive/src/fee_pools/epochs/paths.rs +++ b/packages/rs-drive/src/fee_pools/epochs/paths.rs @@ -33,6 +33,7 @@ //! use crate::drive::RootTree; +use crate::error::drive::DriveError; use crate::error::fee::FeeError; use crate::error::Error; use crate::fee_pools::epochs::epoch_key_constants; @@ -84,8 +85,8 @@ pub fn encode_epoch_index_key(index: u16) -> Result<[u8; 2], Error> { /// Decodes an epoch index key pub fn decode_epoch_index_key(epoch_key: &[u8]) -> Result { let index_with_offset = u16::from_be_bytes(epoch_key.try_into().map_err(|_| { - Error::Fee(FeeError::CorruptedProposerBlockCountItemLength( - "item have an invalid length", + Error::Drive(DriveError::CorruptedSerialization( + "epoch index must be u16", )) })?); diff --git a/packages/rs-drive/src/fee_pools/mod.rs b/packages/rs-drive/src/fee_pools/mod.rs index 184391a3376..0556f5fe3fe 100644 --- a/packages/rs-drive/src/fee_pools/mod.rs +++ b/packages/rs-drive/src/fee_pools/mod.rs @@ -29,7 +29,8 @@ use crate::drive::batch::GroveDbOpBatch; use crate::drive::fee_pools::pools_vec_path; -use crate::fee::credits::Credits; +use crate::error::Error; +use crate::fee::credits::{Creditable, Credits}; use crate::fee::epoch::{EpochIndex, GENESIS_EPOCH_INDEX, PERPETUAL_STORAGE_EPOCHS}; use crate::fee_pools::epochs::Epoch; use crate::fee_pools::epochs_root_tree_key_constants::{ @@ -44,9 +45,9 @@ pub mod epochs; pub mod epochs_root_tree_key_constants; /// Adds the operations to groveDB op batch to create the fee pool trees -pub fn add_create_fee_pool_trees_operations(batch: &mut GroveDbOpBatch) { +pub fn add_create_fee_pool_trees_operations(batch: &mut GroveDbOpBatch) -> Result<(), Error> { // Init storage credit pool - batch.push(update_storage_fee_distribution_pool_operation(0)); + batch.push(update_storage_fee_distribution_pool_operation(0)?); // Init next epoch to pay batch.push(update_unpaid_epoch_index_operation(GENESIS_EPOCH_INDEX)); @@ -57,22 +58,26 @@ pub fn add_create_fee_pool_trees_operations(batch: &mut GroveDbOpBatch) { // with 20 epochs per year that's 1000 epochs for i in GENESIS_EPOCH_INDEX..PERPETUAL_STORAGE_EPOCHS { let epoch = Epoch::new(i); - epoch.add_init_empty_operations(batch); + epoch.add_init_empty_operations(batch)?; } + + Ok(()) } /// Adds operations to batch to create pending pool updates tree pub fn add_create_pending_pool_updates_tree_operations(batch: &mut GroveDbOpBatch) { - batch.add_insert_empty_tree(pools_vec_path(), KEY_PENDING_POOL_UPDATES.to_vec()); + batch.add_insert_empty_sum_tree(pools_vec_path(), KEY_PENDING_POOL_UPDATES.to_vec()); } /// Updates the storage fee distribution pool with a new storage fee -pub fn update_storage_fee_distribution_pool_operation(storage_fee: Credits) -> GroveDbOp { - GroveDbOp::insert_op( +pub fn update_storage_fee_distribution_pool_operation( + storage_fee: Credits, +) -> Result { + Ok(GroveDbOp::insert_op( pools_vec_path(), KEY_STORAGE_FEE_POOL.to_vec(), - Element::new_item(storage_fee.to_be_bytes().to_vec()), - ) + Element::new_sum_item(storage_fee.to_signed()?), + )) } /// Updates the unpaid epoch index @@ -90,14 +95,13 @@ pub fn update_unpaid_epoch_index_operation(epoch_index: EpochIndex) -> GroveDbOp mod tests { use super::*; use crate::common::helpers::setup::setup_drive_with_initial_state_structure; - use crate::error::Error; mod add_create_fee_pool_trees_operations { use super::*; #[test] fn test_values_are_set() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); let storage_fee_pool = drive @@ -109,11 +113,11 @@ mod tests { #[test] fn test_epoch_trees_are_created() { - let drive = super::setup_drive_with_initial_state_structure(); + let drive = setup_drive_with_initial_state_structure(); let transaction = drive.grove.start_transaction(); for epoch_index in 0..1000 { - let epoch = super::Epoch::new(epoch_index); + let epoch = Epoch::new(epoch_index); let storage_fee = drive .get_epoch_storage_credits_for_distribution(&epoch, Some(&transaction)) @@ -122,14 +126,12 @@ mod tests { assert_eq!(storage_fee, 0); } - let epoch = super::Epoch::new(1000); // 1001th epochs pool + let epoch = Epoch::new(1000); // 1001th epochs pool - match drive.get_epoch_storage_credits_for_distribution(&epoch, Some(&transaction)) { - Ok(_) => unreachable!("must be an error"), - Err(e) => { - assert!(matches!(e, Error::GroveDB(..))); - } - } + let result = + drive.get_epoch_storage_credits_for_distribution(&epoch, Some(&transaction)); + + assert!(matches!(result, Err(Error::GroveDB(_)))); } } @@ -145,7 +147,10 @@ mod tests { let mut batch = GroveDbOpBatch::new(); - batch.push(update_storage_fee_distribution_pool_operation(storage_fee)); + batch.push( + update_storage_fee_distribution_pool_operation(storage_fee) + .expect("should return operation"), + ); drive .grove_apply_batch(batch, false, Some(&transaction)) From 3d856ba5b9868b3e0aec8b54830d7d4789cd2188 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 22 Dec 2022 06:18:04 +0800 Subject: [PATCH 34/54] refactor: add_update_epoch_storage_fee_pools_operations --- packages/rs-drive/src/drive/fee_pools/mod.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index 9744bf2ce84..10be2dd74ab 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -176,8 +176,9 @@ mod tests { mod add_update_epoch_storage_fee_pools_operations { use super::*; + use crate::fee::credits::Credits; use crate::fee::epoch::{EpochIndex, GENESIS_EPOCH_INDEX}; - use grovedb::batch::{GroveDbOp, Op}; + use grovedb::batch::Op; #[test] fn should_do_nothing_if_credits_per_epoch_are_empty() { @@ -211,15 +212,14 @@ mod tests { .into_iter() .enumerate() .map(|(i, epoch_index)| { - let credits = 10 - i as SignedCredits; + let credits = 10 - i as Credits; - GroveDbOp::insert_op( - Epoch::new(epoch_index).get_vec_path(), - KEY_POOL_STORAGE_FEES.to_vec(), - Element::new_sum_item(credits), - ) + let epoch = Epoch::new(epoch_index); + + epoch.update_storage_fee_pool_operation(credits) }) - .collect(); + .collect::>() + .expect("should add operations"); let batch = GroveDbOpBatch::from_operations(operations); From 0f43b3abbca4a12f5e19830f4a2e577eb2f39d38 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 22 Dec 2022 06:21:27 +0800 Subject: [PATCH 35/54] refactor: add_update_epoch_storage_fee_pools_sequence_operations --- packages/rs-drive/src/drive/fee_pools/mod.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index 10be2dd74ab..e3d413f4498 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -111,19 +111,16 @@ impl Drive { ))); } - let mut storage_fee_pool_query = Query::new(); - storage_fee_pool_query.insert_key(KEY_POOL_STORAGE_FEES.to_vec()); + let mut query = Query::new(); - let mut epochs_query = Query::new(); + query.insert_range_inclusive(min_encoded_epoch_index..=max_encoded_epoch_index); - epochs_query.insert_range_inclusive(min_encoded_epoch_index..=max_encoded_epoch_index); - - epochs_query.set_subquery(storage_fee_pool_query); + query.set_subquery_key(KEY_POOL_STORAGE_FEES.to_vec()); let (storage_fee_pools_result, _) = self .grove .query_raw( - &PathQuery::new_unsized(pools_vec_path(), epochs_query), + &PathQuery::new_unsized(pools_vec_path(), query), QueryResultType::QueryElementResultType, transaction, ) From 9919aa1946900c4a4471d2a15f0ff590ecbd319a Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 22 Dec 2022 06:26:35 +0800 Subject: [PATCH 36/54] refactor: small --- packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs b/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs index f957d9919c2..ef7333cbdf7 100644 --- a/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs +++ b/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs @@ -85,13 +85,12 @@ impl Drive { to_epoch_index: EpochIndex, transaction: TransactionArg, ) -> Result, Error> { - let mut query = Query::new(); - let from_epoch_key = paths::encode_epoch_index_key(from_epoch_index)?.to_vec(); let current_epoch_key = paths::encode_epoch_index_key(to_epoch_index)?.to_vec(); - query.insert_range_after_to_inclusive(from_epoch_key..=current_epoch_key); + let mut query = Query::new(); + query.insert_range_after_to_inclusive(from_epoch_key..=current_epoch_key); query.set_subquery_key(KEY_START_BLOCK_HEIGHT.to_vec()); let sized_query = SizedQuery::new(query, Some(1), None); From 92362a0470e7b97b1ad57d1f2b637f1833acfae9 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 22 Dec 2022 06:03:18 +0700 Subject: [PATCH 37/54] chore: using new grovedb v0.8.2 --- Cargo.lock | 18 +++++++++--------- packages/rs-drive/Cargo.toml | 6 +++--- packages/rs-drive/src/common/helpers/setup.rs | 5 ++++- .../rs-drive/src/drive/grove_operations.rs | 13 +++++++------ .../rs-drive/src/drive/object_size_info.rs | 10 +++++----- 5 files changed, 28 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c4ef50e5264..ba2ed10273d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -573,8 +573,8 @@ dependencies = [ [[package]] name = "costs" -version = "0.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=3ef80bcd381d42d089681b1297e1137f4f5dbb67#3ef80bcd381d42d089681b1297e1137f4f5dbb67" +version = "1.0.0" +source = "git+https://github.com/dashpay/grovedb?rev=1e92ea9875b4952e30142ac9353f4a12470eb9a3#1e92ea9875b4952e30142ac9353f4a12470eb9a3" dependencies = [ "integer-encoding", "intmap", @@ -1253,8 +1253,8 @@ dependencies = [ [[package]] name = "grovedb" -version = "0.7.1" -source = "git+https://github.com/dashpay/grovedb?rev=3ef80bcd381d42d089681b1297e1137f4f5dbb67#3ef80bcd381d42d089681b1297e1137f4f5dbb67" +version = "0.8.2" +source = "git+https://github.com/dashpay/grovedb?rev=1e92ea9875b4952e30142ac9353f4a12470eb9a3#1e92ea9875b4952e30142ac9353f4a12470eb9a3" dependencies = [ "bincode", "costs", @@ -1688,8 +1688,8 @@ dependencies = [ [[package]] name = "merk" -version = "0.7.1" -source = "git+https://github.com/dashpay/grovedb?rev=3ef80bcd381d42d089681b1297e1137f4f5dbb67#3ef80bcd381d42d089681b1297e1137f4f5dbb67" +version = "0.8.2" +source = "git+https://github.com/dashpay/grovedb?rev=1e92ea9875b4952e30142ac9353f4a12470eb9a3#1e92ea9875b4952e30142ac9353f4a12470eb9a3" dependencies = [ "blake3", "byteorder", @@ -2745,8 +2745,8 @@ dependencies = [ [[package]] name = "storage" -version = "0.2.0" -source = "git+https://github.com/dashpay/grovedb?rev=3ef80bcd381d42d089681b1297e1137f4f5dbb67#3ef80bcd381d42d089681b1297e1137f4f5dbb67" +version = "1.0.0" +source = "git+https://github.com/dashpay/grovedb?rev=1e92ea9875b4952e30142ac9353f4a12470eb9a3#1e92ea9875b4952e30142ac9353f4a12470eb9a3" dependencies = [ "blake3", "costs", @@ -3133,7 +3133,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "visualize" version = "0.1.0" -source = "git+https://github.com/dashpay/grovedb?rev=3ef80bcd381d42d089681b1297e1137f4f5dbb67#3ef80bcd381d42d089681b1297e1137f4f5dbb67" +source = "git+https://github.com/dashpay/grovedb?rev=1e92ea9875b4952e30142ac9353f4a12470eb9a3#1e92ea9875b4952e30142ac9353f4a12470eb9a3" dependencies = [ "hex", "itertools", diff --git a/packages/rs-drive/Cargo.toml b/packages/rs-drive/Cargo.toml index 6ef5ca3340d..3c37473b186 100644 --- a/packages/rs-drive/Cargo.toml +++ b/packages/rs-drive/Cargo.toml @@ -36,15 +36,15 @@ rust_decimal_macros = "1.25.0" [dependencies.grovedb] git = "https://github.com/dashpay/grovedb" -rev = "3ef80bcd381d42d089681b1297e1137f4f5dbb67" +rev = "1e92ea9875b4952e30142ac9353f4a12470eb9a3" [dependencies.storage] git = "https://github.com/dashpay/grovedb" -rev = "3ef80bcd381d42d089681b1297e1137f4f5dbb67" +rev = "1e92ea9875b4952e30142ac9353f4a12470eb9a3" [dependencies.costs] git = "https://github.com/dashpay/grovedb" -rev = "3ef80bcd381d42d089681b1297e1137f4f5dbb67" +rev = "1e92ea9875b4952e30142ac9353f4a12470eb9a3" [dev-dependencies] criterion = "0.3.5" diff --git a/packages/rs-drive/src/common/helpers/setup.rs b/packages/rs-drive/src/common/helpers/setup.rs index a1cb48e2e9f..24e9362aa4b 100644 --- a/packages/rs-drive/src/common/helpers/setup.rs +++ b/packages/rs-drive/src/common/helpers/setup.rs @@ -61,7 +61,10 @@ pub fn setup_drive(drive_config: Option) -> Drive { /// Sets up Drive with the initial state structure. pub fn setup_drive_with_initial_state_structure() -> Drive { - let drive = setup_drive(None); + let drive = setup_drive(Some(DriveConfig{ + batching_consistency_verification: true, + ..Default::default() + })); drive .create_initial_state_structure(None) .expect("should create root tree successfully"); diff --git a/packages/rs-drive/src/drive/grove_operations.rs b/packages/rs-drive/src/drive/grove_operations.rs index 68c9fde8a22..dc6f5726ac9 100644 --- a/packages/rs-drive/src/drive/grove_operations.rs +++ b/packages/rs-drive/src/drive/grove_operations.rs @@ -56,7 +56,7 @@ use crate::error::drive::DriveError; use crate::error::Error; use crate::fee::op::DriveOperation; use crate::fee::op::DriveOperation::CalculatedCostOperation; -use grovedb::operations::delete::DeleteOptions; +use grovedb::operations::delete::{DeleteOptions, DeleteUpTreeOptions}; use grovedb::operations::insert::InsertOptions; use grovedb::query_result_type::{QueryResultElements, QueryResultType}; use grovedb::Error as GroveError; @@ -285,6 +285,7 @@ impl Drive { allow_deleting_non_empty_trees: false, deleting_non_empty_trees_returns_error: true, base_root_storage_is_free: true, + validate_tree_at_path_exists: false, }; let cost_context = self.grove.delete(path, key, Some(options), transaction); push_drive_operation_result(cost_context, drive_operations) @@ -742,6 +743,7 @@ impl Drive { allow_deleting_non_empty_trees: false, deleting_non_empty_trees_returns_error: true, base_root_storage_is_free: true, + validate_tree_at_path_exists: false, }; let delete_operation = match apply_type { BatchDeleteApplyType::StatelessBatchDelete { @@ -763,7 +765,6 @@ impl Drive { path, key, &options, - true, is_known_to_be_subtree_with_sum, ¤t_batch_operations.operations, transaction, @@ -806,17 +807,17 @@ impl Drive { BatchDeleteUpTreeApplyType::StatefulBatchDelete { is_known_to_be_subtree_with_sum, } => { - let options = DeleteOptions { + let options = DeleteUpTreeOptions { allow_deleting_non_empty_trees: false, deleting_non_empty_trees_returns_error: true, base_root_storage_is_free: true, + validate_tree_at_path_exists: false, + stop_path_height, }; self.grove.delete_operations_for_delete_up_tree_while_empty( path.to_path_refs(), key, - stop_path_height, &options, - true, is_known_to_be_subtree_with_sum, current_batch_operations.operations, transaction, @@ -870,7 +871,7 @@ impl Drive { return Err(Error::Drive(DriveError::BatchIsEmpty())); } if self.config.batching_enabled { - //println!("batch {:#?}", ops); + // println!("batch {:#?}", ops); if self.config.batching_consistency_verification { let consistency_results = GroveDbOp::verify_consistency_of_operations(&ops.operations); diff --git a/packages/rs-drive/src/drive/object_size_info.rs b/packages/rs-drive/src/drive/object_size_info.rs index 605f5556e2d..8c2a70e1491 100644 --- a/packages/rs-drive/src/drive/object_size_info.rs +++ b/packages/rs-drive/src/drive/object_size_info.rs @@ -79,7 +79,7 @@ impl<'a, const N: usize> PathInfo<'a, N> { (*path_iterator).into_iter().map(|a| a.len() as u32).sum() } PathIterator(path_iterator) => path_iterator.iter().map(|a| a.len() as u32).sum(), - PathWithSizes(path_size) => path_size.iter().map(|a| a.len() as u32).sum(), + PathWithSizes(path_size) => path_size.iterator().map(|a| a.max_length() as u32).sum(), } } @@ -151,7 +151,7 @@ impl<'a> DriveKeyInfo<'a> { match self { Key(key) => key.len() as u32, KeyRef(key) => key.len() as u32, - KeySize(info) => info.len() as u32, + KeySize(info) => info.max_length() as u32, } } @@ -160,7 +160,7 @@ impl<'a> DriveKeyInfo<'a> { match self { Key(key) => key.is_empty(), KeyRef(key) => key.is_empty(), - KeySize(info) => info.len() == 0, + KeySize(info) => info.max_length() == 0, } } @@ -261,7 +261,7 @@ impl<'a, const N: usize> PathKeyInfo<'a, N> { (*path_iterator).iter().map(|a| a.len() as u32).sum::() + key.len() as u32 } PathKeySize(key_info_path, key_size) => { - key_info_path.iter().map(|a| a.len() as u32).sum::() + key_size.len() as u32 + key_info_path.iterator().map(|a| a.max_length() as u32).sum::() + key_size.max_length() as u32 } } } @@ -281,7 +281,7 @@ impl<'a, const N: usize> PathKeyInfo<'a, N> { PathFixedSizeKeyRef((path_iterator, key)) => { key.is_empty() && (*path_iterator).iter().all(|a| a.is_empty()) } - PathKeySize(path_info, key_info) => path_info.is_empty() && key_info.len() == 0, + PathKeySize(path_info, key_info) => path_info.is_empty() && key_info.max_length() == 0, } } From a46b07d00d9b8c8a622f84b4e15cb97df41d64f7 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 22 Dec 2022 06:07:23 +0700 Subject: [PATCH 38/54] revert: reverted unworking change" --- .../src/drive/fee_pools/epochs/start_block.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs b/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs index f957d9919c2..d37e539155b 100644 --- a/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs +++ b/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs @@ -85,16 +85,19 @@ impl Drive { to_epoch_index: EpochIndex, transaction: TransactionArg, ) -> Result, Error> { - let mut query = Query::new(); + let mut start_block_height_query = Query::new(); + start_block_height_query.insert_key(KEY_START_BLOCK_HEIGHT.to_vec()); + + let mut epochs_query = Query::new(); let from_epoch_key = paths::encode_epoch_index_key(from_epoch_index)?.to_vec(); let current_epoch_key = paths::encode_epoch_index_key(to_epoch_index)?.to_vec(); - query.insert_range_after_to_inclusive(from_epoch_key..=current_epoch_key); + epochs_query.insert_range_after_to_inclusive(from_epoch_key..=current_epoch_key); - query.set_subquery_key(KEY_START_BLOCK_HEIGHT.to_vec()); + epochs_query.set_subquery(start_block_height_query); - let sized_query = SizedQuery::new(query, Some(1), None); + let sized_query = SizedQuery::new(epochs_query, Some(1), None); let path_query = PathQuery::new(pools_vec_path(), sized_query); From f6845da19eb8548398aa096ce4956f97b002aa1c Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 22 Dec 2022 13:12:28 +0800 Subject: [PATCH 39/54] test: fix tests --- .../identity/validation/public_keys_validator_spec.rs | 2 +- .../src/execution/fee_pools/distribute_storage_pool.rs | 2 +- .../rs-drive/src/drive/fee_pools/epochs/start_block.rs | 2 +- packages/rs-drive/src/drive/fee_pools/mod.rs | 10 ++++++---- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/rs-dpp/src/tests/identity/validation/public_keys_validator_spec.rs b/packages/rs-dpp/src/tests/identity/validation/public_keys_validator_spec.rs index 6c8591e9b30..3f1a9c8aea1 100644 --- a/packages/rs-dpp/src/tests/identity/validation/public_keys_validator_spec.rs +++ b/packages/rs-dpp/src/tests/identity/validation/public_keys_validator_spec.rs @@ -3,7 +3,7 @@ use serde_json::{json, Value}; use crate::consensus::ConsensusError; use crate::identity::validation::PublicKeysValidator; use crate::identity::validation::TPublicKeysValidator; -use crate::identity::{Purpose, SecurityLevel}; +use crate::identity::{KeyType, Purpose, SecurityLevel}; use crate::tests::fixtures::get_public_keys_validator; use crate::tests::utils::serde_set_ref; use crate::{assert_consensus_errors, NativeBlsModule}; diff --git a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs index 6b316b31768..e064c404deb 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/distribute_storage_pool.rs @@ -257,7 +257,7 @@ mod tests { assert_eq!( total_distributed, - total_storage_pool_distribution + total_refunds + total_storage_pool_distribution - total_refunds ); } } diff --git a/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs b/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs index d37e539155b..7aa5d0d532e 100644 --- a/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs +++ b/packages/rs-drive/src/drive/fee_pools/epochs/start_block.rs @@ -107,7 +107,7 @@ impl Drive { .unwrap() .map_err(Error::GroveDB)?; - if result_items.len() == 0 { + if result_items.is_empty() { return Ok(None); } diff --git a/packages/rs-drive/src/drive/fee_pools/mod.rs b/packages/rs-drive/src/drive/fee_pools/mod.rs index e3d413f4498..3064d20ee1b 100644 --- a/packages/rs-drive/src/drive/fee_pools/mod.rs +++ b/packages/rs-drive/src/drive/fee_pools/mod.rs @@ -111,16 +111,18 @@ impl Drive { ))); } - let mut query = Query::new(); + let mut storage_fee_pool_query = Query::new(); + storage_fee_pool_query.insert_key(KEY_POOL_STORAGE_FEES.to_vec()); - query.insert_range_inclusive(min_encoded_epoch_index..=max_encoded_epoch_index); + let mut epochs_query = Query::new(); - query.set_subquery_key(KEY_POOL_STORAGE_FEES.to_vec()); + epochs_query.insert_range_inclusive(min_encoded_epoch_index..=max_encoded_epoch_index); + epochs_query.set_subquery(storage_fee_pool_query); let (storage_fee_pools_result, _) = self .grove .query_raw( - &PathQuery::new_unsized(pools_vec_path(), query), + &PathQuery::new_unsized(pools_vec_path(), epochs_query), QueryResultType::QueryElementResultType, transaction, ) From 0e3a53dcd278bba4bed63515148e1feea183dece Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 22 Dec 2022 13:27:18 +0800 Subject: [PATCH 40/54] test: fix JS tests --- .../fee/calculateStateTransitionFeeFactory.js | 5 ++--- .../calculateStateTransitionFeeFactory.spec.js | 16 ++++++++-------- .../js-drive/test/integration/fee/pools.spec.js | 6 +++--- .../prepareProposalHandlerFactory.spec.js | 8 ++++---- .../handlers/proposal/deliverTxFactory.spec.js | 12 ++++++------ .../handlers/proposal/endBlockFactory.spec.js | 4 ++-- .../proposal/processProposalFactory.spec.js | 8 ++++---- packages/rs-drive-nodejs/test/Drive.spec.js | 4 ++-- .../calculateStateTransitionFeeFactory.spec.js | 16 ++++++++-------- 9 files changed, 39 insertions(+), 40 deletions(-) diff --git a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFeeFactory.js b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFeeFactory.js index 0648fb987bd..cdb205de6fd 100644 --- a/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFeeFactory.js +++ b/packages/js-dpp/lib/stateTransition/fee/calculateStateTransitionFeeFactory.js @@ -41,12 +41,11 @@ function calculateStateTransitionFeeFactory(stateRepository, calculateOperationF const [amount, leftovers] = await stateRepository .calculateStorageFeeDistributionAmountAndLeftovers(credits, Number(epochIndex)); - return (await sum) + amount - leftovers; + return (await sum) + amount + leftovers; }, 0); } - // Fee refunds are negative - const total = (storageFee + processingFee + feeRefundsSum) + DEFAULT_USER_TIP; + const total = (storageFee + processingFee - feeRefundsSum) + DEFAULT_USER_TIP; executionContext.setLastCalculatedFeeDetails({ ...calculatedFees, diff --git a/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js b/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js index db331266c3a..6acb695ae70 100644 --- a/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js +++ b/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js @@ -88,8 +88,8 @@ describe('calculateStateTransitionFeeFactory', () => { it('should calculate fee based on executed operations', async () => { const storageFee = 10000; const processingFee = 1000; - const feeRefundsSum = -450 - 995 - 400 - 400; - const total = storageFee + processingFee + feeRefundsSum; + const feeRefundsSum = 450 + 995 + 400 + 400; + const total = storageFee + processingFee - feeRefundsSum; const calculatedOperationsFeesResult = { storageFee, @@ -98,8 +98,8 @@ describe('calculateStateTransitionFeeFactory', () => { { identifier: stateTransition.getOwnerId().toBuffer(), creditsPerEpoch: { - 0: -1000, - 1: -500, + 0: 1000, + 1: 500, }, }, ], @@ -108,20 +108,20 @@ describe('calculateStateTransitionFeeFactory', () => { calculateOperationFeesMock.returns(calculatedOperationsFeesResult); stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers - .onCall(0).resolves([-995, 400]); + .onCall(0).resolves([995, 400]); stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers - .onCall(1).resolves([-450, 400]); + .onCall(1).resolves([450, 400]); const result = await calculateStateTransitionFee(stateTransition); expect(result).to.equal(total); expect(stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers) - .to.have.been.calledWithExactly(-1000, 0); + .to.have.been.calledWithExactly(1000, 0); expect(stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers) - .to.have.been.calledWithExactly(-500, 1); + .to.have.been.calledWithExactly(500, 1); const lastCalculatedFeeDetails = stateTransition.getExecutionContext() .getLastCalculatedFeeDetails(); diff --git a/packages/js-drive/test/integration/fee/pools.spec.js b/packages/js-drive/test/integration/fee/pools.spec.js index 2b2ff900308..bca973cfbb8 100644 --- a/packages/js-drive/test/integration/fee/pools.spec.js +++ b/packages/js-drive/test/integration/fee/pools.spec.js @@ -105,11 +105,11 @@ describe('Fee Pools', () => { const blockEndRequest = { fees: { processingFee: 1000, - storageFee: 10000 + 15, + storageFee: 10000 - 15, feeRefunds: { - 1: -15, + 1: 15, }, - feeRefundsSum: -15, + feeRefundsSum: 15, }, }; diff --git a/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js index 33f306de540..c52e03cb2b0 100644 --- a/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js @@ -78,9 +78,9 @@ describe('prepareProposalHandlerFactory', () => { processingFee: 10, storageFee: 100, feeRefunds: { - 1: -15, + 1: 15, }, - feeRefundsSum: -15, + feeRefundsSum: 15, }, }); @@ -183,9 +183,9 @@ describe('prepareProposalHandlerFactory', () => { processingFee: 10 * 3, storageFee: 100 * 3, feeRefunds: { - 1: -15 * 3, + 1: 15 * 3, }, - feeRefundsSum: -15 * 3, + feeRefundsSum: 15 * 3, }, coreChainLockedHeight: request.coreChainLockedHeight, }, diff --git a/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js index 064db776dab..91e4e6793d9 100644 --- a/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js @@ -55,8 +55,8 @@ describe('deliverTxFactory', () => { stateTransitionExecutionContextMock.setLastCalculatedFeeDetails({ storageFee: 100, processingFee: 10, - feeRefunds: [{ identifier: Buffer.alloc(32), creditsPerEpoch: { 1: -15 } }], - feeRefundsSum: -15, + feeRefunds: [{ identifier: Buffer.alloc(32), creditsPerEpoch: { 1: 15 } }], + feeRefundsSum: 15, total: 95, }); @@ -124,9 +124,9 @@ describe('deliverTxFactory', () => { processingFee: 10, storageFee: 100, feeRefunds: { - 1: -15, + 1: 15, }, - feeRefundsSum: -15, + feeRefundsSum: 15, }, }); @@ -161,9 +161,9 @@ describe('deliverTxFactory', () => { processingFee: 10, storageFee: 100, feeRefunds: { - 1: -15, + 1: 15, }, - feeRefundsSum: -15, + feeRefundsSum: 15, }, }); diff --git a/packages/js-drive/test/unit/abci/handlers/proposal/endBlockFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/proposal/endBlockFactory.spec.js index 30360947c93..cbb0ca7a2cc 100644 --- a/packages/js-drive/test/unit/abci/handlers/proposal/endBlockFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/proposal/endBlockFactory.spec.js @@ -36,9 +36,9 @@ describe('endBlockFactory', () => { processingFee: 10, storageFee: 100, feeRefunds: { - 1: -15, + 1: 15, }, - feeRefundsSum: -15, + feeRefundsSum: 15, }; executionTimerMock = { diff --git a/packages/js-drive/test/unit/abci/handlers/proposal/processProposalFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/proposal/processProposalFactory.spec.js index b45d1d7ff0c..cda64a71068 100644 --- a/packages/js-drive/test/unit/abci/handlers/proposal/processProposalFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/proposal/processProposalFactory.spec.js @@ -70,9 +70,9 @@ describe('processProposalFactory', () => { processingFee: 10, storageFee: 100, feeRefunds: { - 1: -15, + 1: 15, }, - feeRefundsSum: -15, + feeRefundsSum: 15, }, }); @@ -155,9 +155,9 @@ describe('processProposalFactory', () => { processingFee: 10 * 3, storageFee: 100 * 3, feeRefunds: { - 1: -15 * 3, + 1: 15 * 3, }, - feeRefundsSum: -15 * 3, + feeRefundsSum: 15 * 3, }, coreChainLockedHeight: request.coreChainLockedHeight, }, diff --git a/packages/rs-drive-nodejs/test/Drive.spec.js b/packages/rs-drive-nodejs/test/Drive.spec.js index cd994839510..de74729aec5 100644 --- a/packages/rs-drive-nodejs/test/Drive.spec.js +++ b/packages/rs-drive-nodejs/test/Drive.spec.js @@ -570,8 +570,8 @@ describe('Drive', () => { storageFee: 100, processingFee: 100, feeRefunds: { - 1: -15, - 2: -16, + 1: 15, + 2: 16, }, }, }; diff --git a/packages/wasm-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js b/packages/wasm-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js index 25f730104e9..b62316f19a9 100644 --- a/packages/wasm-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js +++ b/packages/wasm-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js @@ -88,8 +88,8 @@ describe('calculateStateTransitionFeeFactory', () => { it('should calculate fee based on executed operations', async () => { const storageFee = 10000; const processingFee = 1000; - const feeRefundsSum = -450 - 995 - 400 - 400; - const total = storageFee + processingFee + feeRefundsSum; + const feeRefundsSum = 450 + 995 + 400 + 400; + const total = storageFee + processingFee - feeRefundsSum; const calculatedOperationsFeesResult = { storageFee, @@ -98,8 +98,8 @@ describe('calculateStateTransitionFeeFactory', () => { { identifier: stateTransition.getOwnerId().toBuffer(), creditsPerEpoch: { - 0: -1000, - 1: -500, + 0: 1000, + 1: 500, }, }, ], @@ -108,20 +108,20 @@ describe('calculateStateTransitionFeeFactory', () => { calculateOperationFeesMock.returns(calculatedOperationsFeesResult); stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers - .onCall(0).resolves([-995, 400]); + .onCall(0).resolves([995, 400]); stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers - .onCall(1).resolves([-450, 400]); + .onCall(1).resolves([450, 400]); const result = await calculateStateTransitionFee(stateTransition); expect(result).to.equal(total); expect(stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers) - .to.have.been.calledWithExactly(-1000, 0); + .to.have.been.calledWithExactly(1000, 0); expect(stateRepositoryMock.calculateStorageFeeDistributionAmountAndLeftovers) - .to.have.been.calledWithExactly(-500, 1); + .to.have.been.calledWithExactly(500, 1); const lastCalculatedFeeDetails = stateTransition.getExecutionContext() .getLastCalculatedFeeDetails(); From 7836912508c6832ebebb5748acd59a4d2f3b2b3e Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 22 Dec 2022 17:42:05 +0800 Subject: [PATCH 41/54] tests: addition fixes --- ...calculateStateTransitionFeeFactory.spec.js | 10 +- ...hronizeMasternodeIdentitiesFactory.spec.js | 16 +-- .../execution/fee_pools/fee_distribution.rs | 127 +++++++++--------- 3 files changed, 79 insertions(+), 74 deletions(-) diff --git a/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js b/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js index 6acb695ae70..398b26b3863 100644 --- a/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js +++ b/packages/js-dpp/test/integration/stateTransition/calculateStateTransitionFeeFactory.spec.js @@ -35,14 +35,14 @@ describe('calculateStateTransitionFeeFactory', () => { { identifier: stateTransition.getOwnerId().toBuffer(), creditsPerEpoch: { - 0: -100, - 1: -50, + 0: 100, + 1: 50, }, }, { identifier: stateTransition.getOwnerId().toBuffer(), creditsPerEpoch: { - 1: -50, + 1: 50, }, }, ], @@ -67,8 +67,8 @@ describe('calculateStateTransitionFeeFactory', () => { { identifier: Buffer.alloc(32, 1), creditsPerEpoch: { - 0: -100, - 1: -50, + 0: 100, + 1: 50, }, }, ], diff --git a/packages/js-drive/test/integration/identity/masternode/synchronizeMasternodeIdentitiesFactory.spec.js b/packages/js-drive/test/integration/identity/masternode/synchronizeMasternodeIdentitiesFactory.spec.js index 0236f177c64..76e142bcb74 100644 --- a/packages/js-drive/test/integration/identity/masternode/synchronizeMasternodeIdentitiesFactory.spec.js +++ b/packages/js-drive/test/integration/identity/masternode/synchronizeMasternodeIdentitiesFactory.spec.js @@ -367,7 +367,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { beforeEach(async function beforeEach() { coreHeight = 3; - firstSyncAppHash = '2b791519644290d49e47024885d37f66da9a278830ef1872e170191acae6d4b9'; + firstSyncAppHash = '5121fd70b91071bd1bec6bfd67763ed61c5797ce3256156e467bd455f1dc0227'; blockInfo = new BlockInfo(10, 0, 1668702100799); container = await createTestDIContainer(); @@ -695,7 +695,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { // Nothing happened - await expectDeterministicAppHash('b3470ce12f4f0a59cab7a24de9607b6cbfb0419ff9597f388c61e631372c9a25'); + await expectDeterministicAppHash('570b71849a365109f6190e3dbf02d1ff961529dc36deb5798f6aa57fdd23f2db'); // Core RPC should be called @@ -750,7 +750,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { expect(result.updatedEntities).to.have.lengthOf(0); expect(result.removedEntities).to.have.lengthOf(0); - await expectDeterministicAppHash('ddd75c33b08e43bfe6f437befdef22b30f1e806fa5d97b39138c1a110fdd8f43'); + await expectDeterministicAppHash('4bc6f9c7852e4db9a02c5fc517538cac779f49e4ad99926a278c5a3500c99e95'); // New masternode identity should be created @@ -834,7 +834,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { expect(result.updatedEntities).to.have.lengthOf(0); expect(result.removedEntities).to.have.lengthOf(1); - await expectDeterministicAppHash('57b1c71c3bd783030a7e4d2f17431ff1c632a6cf1c25bf2724b1505ee30dcc3a'); + await expectDeterministicAppHash('468ff1dc0db7bec9541bf036e6ab60ead37f679d9f4f958b46c7d1cc94a51c00'); // Masternode identity should stay @@ -902,7 +902,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { expect(result.updatedEntities).to.have.lengthOf(0); expect(result.removedEntities).to.have.lengthOf(1); - await expectDeterministicAppHash('57b1c71c3bd783030a7e4d2f17431ff1c632a6cf1c25bf2724b1505ee30dcc3a'); + await expectDeterministicAppHash('468ff1dc0db7bec9541bf036e6ab60ead37f679d9f4f958b46c7d1cc94a51c00'); const invalidMasternodeIdentifier = Identifier.from( Buffer.from(invalidSmlEntry.proRegTxHash, 'hex'), @@ -952,7 +952,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { expect(result.updatedEntities).to.have.lengthOf(3); expect(result.removedEntities).to.have.lengthOf(0); - await expectDeterministicAppHash('d32960fd6e1751d2c3b74574e6b5e23e2f6cac514e3d12a6011d920e0924cbed'); + await expectDeterministicAppHash('7395fde7492c722a2d4e119550e893b5961726849facc1c6f01272cb130de21d'); // Masternode identity should stay @@ -1027,7 +1027,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { await synchronizeMasternodeIdentities(coreHeight + 1, blockInfo); - await expectDeterministicAppHash('e6fec7d1664736157634e73f3dde7818394969323296dbfa880792c2ce472dd9'); + await expectDeterministicAppHash('1662fdf2f72f1082479b28044f8090dcac94a3a6210555a87e363a4d5a2ab407'); // Masternode identity should contain new public key @@ -1110,7 +1110,7 @@ describe('synchronizeMasternodeIdentitiesFactory', () => { await synchronizeMasternodeIdentities(coreHeight, blockInfo); - await expectDeterministicAppHash('934bb8c9dd4f33564ba8392942b2c3a7e5408dabc9d1c5405e8c28ac936ebcc3'); + await expectDeterministicAppHash('96dba6e9a051b795ce900fbfe17eda7f48c4f6b1dcf156a32f085c364ce537e9'); const votingIdentifier = createVotingIdentifier(smlFixture[0]); diff --git a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs index f5e19d96b7e..91e35932247 100644 --- a/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs +++ b/packages/rs-drive-abci/src/execution/fee_pools/fee_distribution.rs @@ -520,13 +520,13 @@ mod tests { .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - match proposer_payouts { - None => assert!(false, "proposers should be paid"), - Some(payouts) => { - assert_eq!(payouts.proposers_paid_count, 50); - assert_eq!(payouts.paid_epoch_index, 0); - } - } + assert!(matches!( + proposer_payouts, + Some(ProposersPayouts { + proposers_paid_count: 50, + paid_epoch_index: 0 + }) + )); } #[test] @@ -607,13 +607,13 @@ mod tests { .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - match proposer_payouts { - None => assert!(false, "proposers should be paid"), - Some(payouts) => { - assert_eq!(payouts.proposers_paid_count, 100); - assert_eq!(payouts.paid_epoch_index, 0); - } - } + assert!(matches!( + proposer_payouts, + Some(ProposersPayouts { + proposers_paid_count: 100, + paid_epoch_index: 0 + }) + )); } #[test] @@ -709,13 +709,13 @@ mod tests { .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - match proposer_payouts { - None => assert!(false, "proposers should be paid"), - Some(payouts) => { - assert_eq!(payouts.proposers_paid_count, 150); - assert_eq!(payouts.paid_epoch_index, 0); - } - } + assert!(matches!( + proposer_payouts, + Some(ProposersPayouts { + proposers_paid_count: 150, + paid_epoch_index: 0 + }) + )); } #[test] @@ -795,19 +795,18 @@ mod tests { assert_eq!(next_unpaid_epoch_index, current_epoch.index); // check we've removed proposers tree - match platform.drive.get_epochs_proposer_block_count( + let result = platform.drive.get_epochs_proposer_block_count( &unpaid_epoch, &proposers[0], Some(&transaction), - ) { - Ok(_) => assert!(false, "expect tree not exists"), - Err(e) => match e { - DriveError::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + ); + + assert!(matches!( + result, + Err(DriveError::GroveDB( + grovedb::Error::PathParentLayerNotFound(_) + )) + )); } #[test] @@ -872,13 +871,13 @@ mod tests { .grove_apply_batch(batch, false, Some(&transaction)) .expect("should apply batch"); - match proposer_payouts { - None => assert!(false, "proposers should be paid"), - Some(payouts) => { - assert_eq!(payouts.proposers_paid_count, proposers_count); - assert_eq!(payouts.paid_epoch_index, 0); - } - } + assert!(matches!( + proposer_payouts, + Some(ProposersPayouts { + proposers_paid_count: proposers_count, + paid_epoch_index: 0 + }) + )); // The Epoch 0 should still not marked as paid because proposers count == proposers limit let next_unpaid_epoch_index = platform @@ -914,19 +913,18 @@ mod tests { assert_eq!(next_unpaid_epoch_index, current_epoch.index); // check we've removed proposers tree - match platform.drive.get_epochs_proposer_block_count( + let result = platform.drive.get_epochs_proposer_block_count( &unpaid_epoch, &proposers[0], Some(&transaction), - ) { - Ok(_) => assert!(false, "expect tree not exists"), - Err(e) => match e { - DriveError::GroveDB(grovedb::Error::PathParentLayerNotFound(_)) => { - assert!(true) - } - _ => assert!(false, "invalid error type"), - }, - } + ); + + assert!(matches!( + result, + Err(DriveError::GroveDB( + grovedb::Error::PathParentLayerNotFound(_) + )) + )); } } @@ -1010,11 +1008,13 @@ mod tests { assert_eq!(unpaid_epoch.start_block_height, 1); assert_eq!(unpaid_epoch.end_block_height, 2); - let block_count = unpaid_epoch.block_count().expect("should calculate "); + let block_count = unpaid_epoch + .block_count() + .expect("should calculate block count"); assert_eq!(block_count, 1); } - None => assert!(false, "unpaid epoch should be present"), + None => unreachable!("unpaid epoch should be present"), } } @@ -1051,11 +1051,13 @@ mod tests { assert_eq!(unpaid_epoch.start_block_height, 1); assert_eq!(unpaid_epoch.end_block_height, 2); - let block_count = unpaid_epoch.block_count().expect("should calculate "); + let block_count = unpaid_epoch + .block_count() + .expect("should calculate block count"); assert_eq!(block_count, 1); } - None => assert!(false, "unpaid epoch should be present"), + None => unreachable!("unpaid epoch should be present"), } } @@ -1093,11 +1095,13 @@ mod tests { assert_eq!(unpaid_epoch.start_block_height, 1); assert_eq!(unpaid_epoch.end_block_height, 2); - let block_count = unpaid_epoch.block_count().expect("should calculate "); + let block_count = unpaid_epoch + .block_count() + .expect("should calculate block count"); assert_eq!(block_count, 1); } - None => assert!(false, "unpaid epoch should be present"), + None => unreachable!("unpaid epoch should be present"), } } @@ -1136,11 +1140,13 @@ mod tests { assert_eq!(unpaid_epoch.start_block_height, 1); assert_eq!(unpaid_epoch.end_block_height, 2); - let block_count = unpaid_epoch.block_count().expect("should calculate "); + let block_count = unpaid_epoch + .block_count() + .expect("should calculate block count"); assert_eq!(block_count, 1); } - None => assert!(false, "unpaid epoch should be present"), + None => unreachable!("unpaid epoch should be present"), } } @@ -1169,11 +1175,10 @@ mod tests { Some(&transaction), ); - match unpaid_epoch { - Ok(_) => assert!(false, "should return code execution error"), - Err(Error::Execution(ExecutionError::CorruptedCodeExecution(_))) => assert!(true), - Err(_) => assert!(false, "wrong error type"), - } + assert!(matches!( + unpaid_epoch, + Err(Error::Execution(ExecutionError::CorruptedCodeExecution(_))) + )); } } From 6f18d161db0ed687cea2fa0e9295cf1c57fede19 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 22 Dec 2022 19:39:43 +0800 Subject: [PATCH 42/54] fix: calculateOperationFees is not a function --- .../js-dpp/lib/stateTransition/StateTransitionFacade.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/js-dpp/lib/stateTransition/StateTransitionFacade.js b/packages/js-dpp/lib/stateTransition/StateTransitionFacade.js index 83f7552fd66..997f638aa1b 100644 --- a/packages/js-dpp/lib/stateTransition/StateTransitionFacade.js +++ b/packages/js-dpp/lib/stateTransition/StateTransitionFacade.js @@ -93,6 +93,7 @@ const identityJsonSchema = require('../../schema/identity/stateTransition/public const validatePublicKeySignaturesFactory = require('../identity/stateTransition/validatePublicKeySignaturesFactory'); const StateTransitionExecutionContext = require('./StateTransitionExecutionContext'); const calculateStateTransitionFeeFactory = require('./fee/calculateStateTransitionFeeFactory'); +const calculateOperationFees = require('./fee/calculateOperationFees'); class StateTransitionFacade { /** @@ -298,7 +299,10 @@ class StateTransitionFacade { [stateTransitionTypes.IDENTITY_UPDATE]: validateIdentityUpdateTransitionState, }); - const calculateStateTransitionFee = calculateStateTransitionFeeFactory(this.stateRepository); + const calculateStateTransitionFee = calculateStateTransitionFeeFactory( + this.stateRepository, + calculateOperationFees, + ); this.validateStateTransitionFee = validateStateTransitionFeeFactory( this.stateRepository, From 8b13fa5cf5b93a71dcba376b00b854eb9e9de099 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 22 Dec 2022 20:01:24 +0800 Subject: [PATCH 43/54] feat(drive): ABCI context logger --- .../abci/errors/createContextLoggerFactory.js | 13 ++++++ .../enrichErrorWithConsensusLoggerFactory.js | 42 ------------------- .../enrichErrorWithContextLoggerFactory.js | 41 ++++++++++++++++++ .../abci/errors/wrapInErrorHandlerFactory.js | 33 ++++----------- .../abci/handlers/checkTxHandlerFactory.js | 8 ++++ .../abci/handlers/extendVoteHandlerFactory.js | 13 +++--- .../lib/abci/handlers/infoHandlerFactory.js | 28 ++++++------- .../abci/handlers/initChainHandlerFactory.js | 21 +++++----- .../handlers/prepareProposalHandlerFactory.js | 22 +++++----- .../handlers/processProposalHandlerFactory.js | 13 +++--- .../handlers/proposal/beginBlockFactory.js | 16 +++---- .../handlers/proposal/deliverTxFactory.js | 5 +-- .../lib/abci/handlers/queryHandlerFactory.js | 6 ++- .../blockExecution/BlockExecutionContext.js | 26 ++++++------ .../BlockExecutionContextRepository.js | 2 +- packages/js-drive/lib/createDIContainer.js | 34 ++++++++------- packages/js-drive/lib/errorHandlerFactory.js | 2 +- ...ichErrorWithConsensusLoggerFactory.spec.js | 2 +- 18 files changed, 166 insertions(+), 161 deletions(-) create mode 100644 packages/js-drive/lib/abci/errors/createContextLoggerFactory.js delete mode 100644 packages/js-drive/lib/abci/errors/enrichErrorWithConsensusLoggerFactory.js create mode 100644 packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js diff --git a/packages/js-drive/lib/abci/errors/createContextLoggerFactory.js b/packages/js-drive/lib/abci/errors/createContextLoggerFactory.js new file mode 100644 index 00000000000..4e79ff71d92 --- /dev/null +++ b/packages/js-drive/lib/abci/errors/createContextLoggerFactory.js @@ -0,0 +1,13 @@ +function createContextLoggerFactory(abciAsyncLocalStorage) { + function createContextLogger(logger, context) { + const contextLogger = logger.child(context); + + abciAsyncLocalStorage.getStore().set('logger', contextLogger); + + return contextLogger; + } + + return createContextLogger; +} + +module.exports = createContextLoggerFactory; diff --git a/packages/js-drive/lib/abci/errors/enrichErrorWithConsensusLoggerFactory.js b/packages/js-drive/lib/abci/errors/enrichErrorWithConsensusLoggerFactory.js deleted file mode 100644 index 0128416d6de..00000000000 --- a/packages/js-drive/lib/abci/errors/enrichErrorWithConsensusLoggerFactory.js +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Add consensus logger to an error (factory) - * - * @param {BlockExecutionContext} latestBlockExecutionContext - * - * @return {enrichErrorWithConsensusLogger} - */ -function enrichErrorWithConsensusLoggerFactory(latestBlockExecutionContext) { - /** - * Add consensus logger to an error - * - * @typedef enrichErrorWithConsensusLogger - * - * @param {Function} method - * - * @return {Function} - */ - function enrichErrorWithConsensusLogger(method) { - /** - * @param {*[]} args - */ - async function methodHandler(...args) { - try { - return await method(...args); - } catch (e) { - const { consensusLogger } = latestBlockExecutionContext; - - if (consensusLogger) { - e.consensusLogger = consensusLogger; - } - - throw e; - } - } - - return methodHandler; - } - - return enrichErrorWithConsensusLogger; -} - -module.exports = enrichErrorWithConsensusLoggerFactory; diff --git a/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js b/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js new file mode 100644 index 00000000000..5f58cb79e54 --- /dev/null +++ b/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js @@ -0,0 +1,41 @@ +const { AsyncLocalStorage } = require('node:async_hooks'); + +/** + * Add consensus logger to an error (factory) + * + * @return {enrichErrorWithContextLogger} + */ +function enrichErrorWithContextLoggerFactory() { + /** + * Add consensus logger to an error + * + * @typedef enrichErrorWithContextLogger + * + * @param {Function} method + * + * @return {Function} + */ + function enrichErrorWithContextLogger(method) { + /** + * @param {*[]} args + */ + async function methodHandler(...args) { + const asyncLocalStorage = new AsyncLocalStorage(); + + return asyncLocalStorage.run(new Map(), () => { + return method(...args).catch((error) => { + // eslint-disable-next-line no-param-reassign + error.contextLogger = asyncLocalStorage.getStore().get('logger'); + + return Promise.reject(error); + }); + }); + } + + return methodHandler; + } + + return enrichErrorWithContextLogger; +} + +module.exports = enrichErrorWithContextLoggerFactory; diff --git a/packages/js-drive/lib/abci/errors/wrapInErrorHandlerFactory.js b/packages/js-drive/lib/abci/errors/wrapInErrorHandlerFactory.js index bbf95334443..909788b3a61 100644 --- a/packages/js-drive/lib/abci/errors/wrapInErrorHandlerFactory.js +++ b/packages/js-drive/lib/abci/errors/wrapInErrorHandlerFactory.js @@ -15,18 +15,10 @@ function wrapInErrorHandlerFactory(logger, isProductionEnvironment) { * @typedef wrapInErrorHandler * * @param {Function} method - * @param {Object} [options={}] - * @param {boolean} [options.respondWithInternalError=false] * * @return {Function} */ - function wrapInErrorHandler(method, options = {}) { - // eslint-disable-next-line no-param-reassign - options = { - respondWithInternalError: false, - ...options, - }; - + function wrapInErrorHandler(method) { /** * @param {*[]} args */ @@ -34,37 +26,28 @@ function wrapInErrorHandlerFactory(logger, isProductionEnvironment) { try { return await method(...args); } catch (e) { - let error = e; + let abciError = e; // Wrap all non ABCI errors to an internal ABCI error if (!(e instanceof AbstractAbciError)) { - error = new InternalAbciError(e); + abciError = new InternalAbciError(e); } // Log only internal ABCI errors - if (error instanceof InternalAbciError) { - // in consensus ABCI handlers (blockBegin, deliverTx, blockEnd, commit) - // we should propagate the error upwards - // to halt the Drive - // in order cases like query and checkTx - // we need to respond with internal errors - if (!options.respondWithInternalError) { - throw error.getError(); - } - - const originalError = error.getError(); + if (abciError instanceof InternalAbciError) { + const originalError = abciError.getError(); - (originalError.consensusLogger || logger).error( + (originalError.contextLogger || logger).error( { err: originalError }, originalError.message, ); if (!isProductionEnvironment) { - error = new VerboseInternalAbciError(error); + abciError = new VerboseInternalAbciError(abciError); } } - return error.getAbciResponse(); + return abciError.getAbciResponse(); } } diff --git a/packages/js-drive/lib/abci/handlers/checkTxHandlerFactory.js b/packages/js-drive/lib/abci/handlers/checkTxHandlerFactory.js index 1cf6707185c..d97e4b865a9 100644 --- a/packages/js-drive/lib/abci/handlers/checkTxHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/checkTxHandlerFactory.js @@ -8,11 +8,15 @@ const { /** * @param {unserializeStateTransition} unserializeStateTransition + * @param {AsyncLocalStorage} unserializeStateTransition + * @param {Logger} logger * * @returns {checkTxHandler} */ function checkTxHandlerFactory( unserializeStateTransition, + createContextLogger, + logger, ) { /** * CheckTx ABCI Handler @@ -24,6 +28,10 @@ function checkTxHandlerFactory( * @returns {Promise} */ async function checkTxHandler({ tx: stateTransitionByteArray }) { + createContextLogger(logger, { + abciMethod: 'checkTx', + }); + await unserializeStateTransition(stateTransitionByteArray); return new ResponseCheckTx(); diff --git a/packages/js-drive/lib/abci/handlers/extendVoteHandlerFactory.js b/packages/js-drive/lib/abci/handlers/extendVoteHandlerFactory.js index 76f178b3f3a..3c6ad431870 100644 --- a/packages/js-drive/lib/abci/handlers/extendVoteHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/extendVoteHandlerFactory.js @@ -14,18 +14,17 @@ const { * * @return {extendVoteHandler} */ -function extendVoteHandlerFactory(proposalBlockExecutionContext) { +function extendVoteHandlerFactory(proposalBlockExecutionContext, createContextLogger) { /** * @typedef extendVoteHandler * @return {Promise} */ async function extendVoteHandler() { - const consensusLogger = proposalBlockExecutionContext.getConsensusLogger() - .child({ - abciMethod: 'extendVote', - }); + const contextLogger = createContextLogger(proposalBlockExecutionContext.getContextLogger(), { + abciMethod: 'extendVote', + }); - consensusLogger.debug('ExtendVote ABCI method requested'); + contextLogger.debug('ExtendVote ABCI method requested'); const unsignedWithdrawalTransactionsMap = proposalBlockExecutionContext .getWithdrawalTransactionsMap(); @@ -50,7 +49,7 @@ function extendVoteHandlerFactory(proposalBlockExecutionContext) { Math.min(30, extensionString.length), ); - consensusLogger.debug({ + contextLogger.debug({ type, extension: extensionString, }, `Vote extended to obtain ${voteExtensionTypeName} signature for ${extensionTruncatedString}... payload`); diff --git a/packages/js-drive/lib/abci/handlers/infoHandlerFactory.js b/packages/js-drive/lib/abci/handlers/infoHandlerFactory.js index 427637a9d1a..68e3a9fd56f 100644 --- a/packages/js-drive/lib/abci/handlers/infoHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/infoHandlerFactory.js @@ -26,6 +26,7 @@ function infoHandlerFactory( updateSimplifiedMasternodeList, logger, groveDBStore, + createContextLogger, ) { /** * Info ABCI handler @@ -36,37 +37,32 @@ function infoHandlerFactory( * @return {Promise} */ async function infoHandler(request) { - let contextLogger = logger.child({ + let contextLogger = createContextLogger(logger, { abciMethod: 'info', }); contextLogger.debug('Info ABCI method requested'); contextLogger.trace({ abciRequest: request }); - // Initialize Block Execution Contexts - - const persistedBlockExecutionContext = await blockExecutionContextRepository.fetch(); - if (persistedBlockExecutionContext) { - latestBlockExecutionContext.populate(persistedBlockExecutionContext); - } - // Initialize current heights let lastHeight = Long.fromNumber(0); let lastCoreChainLockedHeight = 0; - if (!latestBlockExecutionContext.isEmpty()) { + // Initialize latest Block Execution Context + const persistedBlockExecutionContext = await blockExecutionContextRepository.fetch(); + if (!persistedBlockExecutionContext.isEmpty()) { + latestBlockExecutionContext.populate(persistedBlockExecutionContext); + lastHeight = latestBlockExecutionContext.getHeight(); lastCoreChainLockedHeight = latestBlockExecutionContext.getCoreChainLockedHeight(); - } - contextLogger = contextLogger.child({ - height: lastHeight.toString(), - }); + contextLogger = createContextLogger(contextLogger, { + height: lastHeight.toString(), + }); - // Update SML store to latest saved core chain lock to make sure - // that verify chain lock handler has updated SML Store to verify signatures - if (!latestBlockExecutionContext.isEmpty()) { + // Update SML store to latest saved core chain lock to make sure + // that verify chain lock handler has updated SML Store to verify signatures await updateSimplifiedMasternodeList(lastCoreChainLockedHeight, { logger: contextLogger, }); diff --git a/packages/js-drive/lib/abci/handlers/initChainHandlerFactory.js b/packages/js-drive/lib/abci/handlers/initChainHandlerFactory.js index 9db11dd6cec..9675a28ea4d 100644 --- a/packages/js-drive/lib/abci/handlers/initChainHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/initChainHandlerFactory.js @@ -35,6 +35,7 @@ function initChainHandlerFactory( groveDBStore, rsAbci, createCoreChainLockUpdate, + createContextLogger, ) { /** * @typedef initChainHandler @@ -45,17 +46,17 @@ function initChainHandlerFactory( async function initChainHandler(request) { const { time } = request; - const consensusLogger = logger.child({ + const contextLogger = createContextLogger(logger, { height: request.initialHeight.toString(), abciMethod: 'initChain', }); - consensusLogger.debug('InitChain ABCI method requested'); - consensusLogger.trace({ abciRequest: request }); + contextLogger.debug('InitChain ABCI method requested'); + contextLogger.trace({ abciRequest: request }); await updateSimplifiedMasternodeList( initialCoreChainLockedHeight, { - logger: consensusLogger, + logger: contextLogger, }, ); @@ -75,7 +76,7 @@ function initChainHandlerFactory( protoTimestampToMillis(time), ); - await registerSystemDataContracts(consensusLogger, blockInfo); + await registerSystemDataContracts(contextLogger, blockInfo); const synchronizeMasternodeIdentitiesResult = await synchronizeMasternodeIdentities( initialCoreChainLockedHeight, @@ -86,11 +87,11 @@ function initChainHandlerFactory( createdEntities, updatedEntities, removedEntities, fromHeight, toHeight, } = synchronizeMasternodeIdentitiesResult; - consensusLogger.info( + contextLogger.info( `Masternode identities are synced for heights from ${fromHeight} to ${toHeight}: ${createdEntities.length} created, ${updatedEntities.length} updated, ${removedEntities.length} removed`, ); - consensusLogger.trace( + contextLogger.trace( { createdEntities: createdEntities.map((item) => item.toJSON()), updatedEntities: updatedEntities.map((item) => item.toJSON()), @@ -110,16 +111,16 @@ function initChainHandlerFactory( const coreChainLockUpdate = await createCoreChainLockUpdate( initialCoreChainLockedHeight, 0, - consensusLogger, + contextLogger, ); const appHash = await groveDBStore.getRootHash({ useTransaction: true }); await groveDBStore.commitTransaction(); - consensusLogger.trace(validatorSetUpdate, `Validator set initialized with ${quorumHash} quorum`); + contextLogger.trace(validatorSetUpdate, `Validator set initialized with ${quorumHash} quorum`); - consensusLogger.info( + contextLogger.info( { chainId: request.chainId, appHash: appHash.toString('hex').toUpperCase(), diff --git a/packages/js-drive/lib/abci/handlers/prepareProposalHandlerFactory.js b/packages/js-drive/lib/abci/handlers/prepareProposalHandlerFactory.js index 924fe2e25c7..ae2aad4869d 100644 --- a/packages/js-drive/lib/abci/handlers/prepareProposalHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/prepareProposalHandlerFactory.js @@ -34,6 +34,7 @@ function prepareProposalHandlerFactory( endBlock, createCoreChainLockUpdate, executionTimer, + createContextLogger, ) { /** * @typedef prepareProposalHandler @@ -53,7 +54,7 @@ function prepareProposalHandlerFactory( round, } = request; - const consensusLogger = logger.child({ + const contextLogger = createContextLogger(logger, { height: height.toString(), round, abciMethod: 'prepareProposal', @@ -62,10 +63,10 @@ function prepareProposalHandlerFactory( const requestToLog = lodashCloneDeep(request); delete requestToLog.txs; - consensusLogger.debug('PrepareProposal ABCI method requested'); - consensusLogger.trace({ abciRequest: requestToLog }); + contextLogger.debug('PrepareProposal ABCI method requested'); + contextLogger.trace({ abciRequest: requestToLog }); - consensusLogger.info(`Preparing a block proposal for height #${height} round #${round}`); + contextLogger.info(`Preparing a block proposal for height #${height} round #${round}`); await beginBlock( { @@ -77,7 +78,7 @@ function prepareProposalHandlerFactory( proposerProTxHash: Buffer.from(proposerProTxHash), round, }, - consensusLogger, + contextLogger, ); let totalSizeBytes = 0; @@ -110,7 +111,7 @@ function prepareProposalHandlerFactory( code, info, fees, - } = await wrappedDeliverTx(tx, round, consensusLogger); + } = await wrappedDeliverTx(tx, round, contextLogger); if (code === 0) { validTxCount += 1; @@ -129,13 +130,10 @@ function prepareProposalHandlerFactory( txResults.push(txResult); } - // Revert consensus logger after deliverTx - proposalBlockExecutionContext.setConsensusLogger(consensusLogger); - const coreChainLockUpdate = await createCoreChainLockUpdate( coreChainLockedHeight, round, - consensusLogger, + contextLogger, ); const { @@ -147,13 +145,13 @@ function prepareProposalHandlerFactory( round, fees: feeResults, coreChainLockedHeight, - }, consensusLogger); + }, contextLogger); const roundExecutionTime = executionTimer.getTimer('roundExecution', true); const mempoolTxCount = txs.length - validTxCount - invalidTxCount; - consensusLogger.info( + contextLogger.info( { roundExecutionTime, validTxCount, diff --git a/packages/js-drive/lib/abci/handlers/processProposalHandlerFactory.js b/packages/js-drive/lib/abci/handlers/processProposalHandlerFactory.js index 33d21bbef7e..c009f28a37b 100644 --- a/packages/js-drive/lib/abci/handlers/processProposalHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/processProposalHandlerFactory.js @@ -21,6 +21,7 @@ function processProposalHandlerFactory( verifyChainLock, processProposal, proposalBlockExecutionContext, + createContextLogger, ) { /** * @typedef processProposalHandler @@ -34,7 +35,7 @@ function processProposalHandlerFactory( round, } = request; - const consensusLogger = logger.child({ + const contextLogger = createContextLogger(logger, { height: height.toString(), round, abciMethod: 'processProposal', @@ -43,8 +44,8 @@ function processProposalHandlerFactory( const requestToLog = lodashCloneDeep(request); delete requestToLog.txs; - consensusLogger.debug('ProcessProposal ABCI method requested'); - consensusLogger.trace({ abciRequest: requestToLog }); + contextLogger.debug('ProcessProposal ABCI method requested'); + contextLogger.trace({ abciRequest: requestToLog }); // Skip process proposal if it was already prepared for this height and round const prepareProposalResult = proposalBlockExecutionContext.getPrepareProposalResult(); @@ -52,7 +53,7 @@ function processProposalHandlerFactory( if (prepareProposalResult && proposalBlockExecutionContext.getHeight().toNumber() === height.toNumber() && proposalBlockExecutionContext.getRound() === round) { - consensusLogger.debug('Skip processing proposal and return prepared result'); + contextLogger.debug('Skip processing proposal and return prepared result'); const { appHash, @@ -74,7 +75,7 @@ function processProposalHandlerFactory( const chainLockIsValid = await verifyChainLock(coreChainLockUpdate); if (!chainLockIsValid) { - consensusLogger.warn({ + contextLogger.warn({ coreChainLockUpdate, }, `Block proposal #${height} round #${round} rejected due to invalid core chain locked height update`); @@ -88,7 +89,7 @@ function processProposalHandlerFactory( }, `ChainLock is valid for height ${coreChainLockUpdate.coreBlockHeight}`); } - return processProposal(request, consensusLogger); + return processProposal(request, contextLogger); } return processProposalHandler; diff --git a/packages/js-drive/lib/abci/handlers/proposal/beginBlockFactory.js b/packages/js-drive/lib/abci/handlers/proposal/beginBlockFactory.js index d5fc50c9a58..d202f09f90a 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/beginBlockFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/beginBlockFactory.js @@ -45,11 +45,11 @@ function beginBlockFactory( * @param {IConsensus} request.version * @param {ITimestamp} request.time * @param {Buffer} request.proposerProTxHash - * @param {BaseLogger} consensusLogger + * @param {BaseLogger} contextLogger * * @return {Promise} */ - async function beginBlock(request, consensusLogger) { + async function beginBlock(request, contextLogger) { const { lastCommitInfo, height, @@ -88,7 +88,7 @@ function beginBlockFactory( // Reset block execution context proposalBlockExecutionContext.reset(); - proposalBlockExecutionContext.setConsensusLogger(consensusLogger); + proposalBlockExecutionContext.setContextLogger(contextLogger); proposalBlockExecutionContext.setHeight(height); proposalBlockExecutionContext.setVersion(version); proposalBlockExecutionContext.setRound(round); @@ -124,7 +124,7 @@ function beginBlockFactory( rsRequest.previousBlockTimeMs = latestBlockExecutionContext.getTimeMs(); } - consensusLogger.debug(rsRequest, 'Request RS Drive\'s BlockBegin method'); + contextLogger.debug(rsRequest, 'Request RS Drive\'s BlockBegin method'); const rsResponse = await rsAbci.blockBegin(rsRequest, true); @@ -153,14 +153,14 @@ function beginBlockFactory( const blockTimeFormatted = new Date(proposalBlockExecutionContext.getTimeMs()).toUTCString(); - consensusLogger.info(debugData, `Epoch #${currentEpochIndex} started on block #${height} at ${blockTimeFormatted}`); + contextLogger.info(debugData, `Epoch #${currentEpochIndex} started on block #${height} at ${blockTimeFormatted}`); } // Update SML const isSimplifiedMasternodeListUpdated = await updateSimplifiedMasternodeList( coreChainLockedHeight, { - logger: consensusLogger, + logger: contextLogger, }, ); @@ -176,12 +176,12 @@ function beginBlockFactory( createdEntities, updatedEntities, removedEntities, fromHeight, toHeight, } = synchronizeMasternodeIdentitiesResult; - consensusLogger.info( + contextLogger.info( `Masternode identities are synced for heights from ${fromHeight} to ${toHeight}: ${createdEntities.length} created, ${updatedEntities.length} updated, ${removedEntities.length} removed`, ); if (createdEntities.length > 0 || updatedEntities.length > 0 || removedEntities.length > 0) { - consensusLogger.trace( + contextLogger.trace( { createdEntities: createdEntities.map((item) => item.toJSON()), updatedEntities: updatedEntities.map((item) => item.toJSON()), diff --git a/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js b/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js index 3d191028799..a0015219fdd 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js @@ -33,6 +33,7 @@ function deliverTxFactory( transactionalDpp, proposalBlockExecutionContext, executionTimer, + createContextLogger, ) { /** * @typedef deliverTx @@ -65,12 +66,10 @@ function deliverTxFactory( .toString('hex') .toUpperCase(); - const txConsensusLogger = consensusLogger.child({ + const txConsensusLogger = createContextLogger(consensusLogger, { txId: stHash, }); - proposalBlockExecutionContext.setConsensusLogger(txConsensusLogger); - txConsensusLogger.info(`Deliver state transition ${stHash} from block #${blockHeight}`); const stateTransition = await transactionalUnserializeStateTransition( diff --git a/packages/js-drive/lib/abci/handlers/queryHandlerFactory.js b/packages/js-drive/lib/abci/handlers/queryHandlerFactory.js index 77fd405435f..800639d7ca1 100644 --- a/packages/js-drive/lib/abci/handlers/queryHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/queryHandlerFactory.js @@ -7,7 +7,7 @@ const InvalidArgumentAbciError = require('../errors/InvalidArgumentAbciError'); * @param {Function} sanitizeUrl * @return {queryHandler} */ -function queryHandlerFactory(queryHandlerRouter, sanitizeUrl) { +function queryHandlerFactory(queryHandlerRouter, sanitizeUrl, logger, createContextLogger) { /** * Query ABCI Handler * @@ -19,6 +19,10 @@ function queryHandlerFactory(queryHandlerRouter, sanitizeUrl) { async function queryHandler(request) { const { path, data } = request; + createContextLogger(logger, { + abciMethod: 'query', + }); + const route = queryHandlerRouter.find('GET', sanitizeUrl(path)); if (!route) { diff --git a/packages/js-drive/lib/blockExecution/BlockExecutionContext.js b/packages/js-drive/lib/blockExecution/BlockExecutionContext.js index b1c21afa7cd..7f73864d6bf 100644 --- a/packages/js-drive/lib/blockExecution/BlockExecutionContext.js +++ b/packages/js-drive/lib/blockExecution/BlockExecutionContext.js @@ -140,25 +140,25 @@ class BlockExecutionContext { } /** - * Set consensus logger + * Set context logger * * @param {BaseLogger} logger */ - setConsensusLogger(logger) { - this.consensusLogger = logger; + setContextLogger(logger) { + this.contextLogger = logger; } /** - * Get consensus logger + * Get context logger * * @return {BaseLogger} */ - getConsensusLogger() { - if (!this.consensusLogger) { + getContextLogger() { + if (!this.contextLogger) { throw new Error('Consensus logger has not been set'); } - return this.consensusLogger; + return this.contextLogger; } /** @@ -251,7 +251,7 @@ class BlockExecutionContext { this.version = null; this.time = null; this.lastCommitInfo = null; - this.consensusLogger = null; + this.contextLogger = null; this.withdrawalTransactionsMap = {}; this.round = null; this.epochInfo = null; @@ -280,7 +280,7 @@ class BlockExecutionContext { this.height = blockExecutionContext.height; this.coreChainLockedHeight = blockExecutionContext.coreChainLockedHeight; this.version = blockExecutionContext.version; - this.consensusLogger = blockExecutionContext.consensusLogger || null; + this.contextLogger = blockExecutionContext.contextLogger || null; this.withdrawalTransactionsMap = blockExecutionContext.withdrawalTransactionsMap; this.round = blockExecutionContext.round; this.epochInfo = blockExecutionContext.epochInfo; @@ -297,7 +297,7 @@ class BlockExecutionContext { this.dataContracts = object.dataContracts .map((rawDataContract) => new DataContract(rawDataContract)); this.lastCommitInfo = CommitInfo.fromObject(object.lastCommitInfo); - this.consensusLogger = object.consensusLogger; + this.contextLogger = object.contextLogger; this.epochInfo = object.epochInfo; this.timeMs = object.timeMs; this.height = Long.fromNumber(object.height); @@ -310,7 +310,7 @@ class BlockExecutionContext { /** * @param {Object} options - * @param {boolean} [options.skipConsensusLogger=false] + * @param {boolean} [options.skipContextLogger=false] * @param {boolean} [options.skipPrepareProposalResult=false] * @return {{ * dataContracts: Object[], @@ -344,8 +344,8 @@ class BlockExecutionContext { epochInfo: this.epochInfo, }; - if (!options.skipConsensusLogger) { - object.consensusLogger = this.consensusLogger; + if (!options.skipContextLogger) { + object.contextLogger = this.contextLogger; } if (!options.skipPrepareProposalResult) { diff --git a/packages/js-drive/lib/blockExecution/BlockExecutionContextRepository.js b/packages/js-drive/lib/blockExecution/BlockExecutionContextRepository.js index bcd6d45c585..e7e2800ca81 100644 --- a/packages/js-drive/lib/blockExecution/BlockExecutionContextRepository.js +++ b/packages/js-drive/lib/blockExecution/BlockExecutionContextRepository.js @@ -23,7 +23,7 @@ class BlockExecutionContextRepository { await this.db.putAux( BlockExecutionContextRepository.EXTERNAL_STORE_KEY_NAME, await cbor.encodeAsync(blockExecutionContext.toObject({ - skipConsensusLogger: true, + skipContextLogger: true, skipPrepareProposalResult: true, })), options, diff --git a/packages/js-drive/lib/createDIContainer.js b/packages/js-drive/lib/createDIContainer.js index 5e30e59a235..792f3186861 100644 --- a/packages/js-drive/lib/createDIContainer.js +++ b/packages/js-drive/lib/createDIContainer.js @@ -8,6 +8,8 @@ const { const fs = require('fs'); +const { AsyncLocalStorage } = require('node:async_hooks'); + const Long = require('long'); const RSDrive = require('@dashevo/rs-drive'); @@ -106,7 +108,7 @@ const waitForChainLockedHeightFactory = require('./core/waitForChainLockedHeight const SimplifiedMasternodeList = require('./core/SimplifiedMasternodeList'); const SpentAssetLockTransactionsRepository = require('./identity/SpentAssetLockTransactionsRepository'); -const enrichErrorWithConsensusErrorFactory = require('./abci/errors/enrichErrorWithConsensusLoggerFactory'); +const enrichErrorWithConsensusErrorFactory = require('./abci/errors/enrichErrorWithContextLoggerFactory'); const closeAbciServerFactory = require('./abci/closeAbciServerFactory'); const getLatestFeatureFlagFactory = require('./featureFlag/getLatestFeatureFlagFactory'); const getFeatureFlagForHeightFactory = require('./featureFlag/getFeatureFlagForHeightFactory'); @@ -138,6 +140,7 @@ const fetchTransactionFactory = require('./core/fetchTransactionFactory'); const LastSyncedCoreHeightRepository = require('./identity/masternode/LastSyncedCoreHeightRepository'); const fetchSimplifiedMNListFactory = require('./core/fetchSimplifiedMNListFactory'); const processProposalFactory = require('./abci/handlers/proposal/processProposalFactory'); +const createContextLoggerFactory = require('./abci/errors/createContextLoggerFactory'); /** * @@ -693,6 +696,8 @@ function createDIContainer(options) { * Register ABCI handlers */ container.register({ + createContextLogger: asFunction(createContextLoggerFactory), + abciAsyncLocalStorage: asValue(new AsyncLocalStorage()), createQueryResponse: asFunction(createQueryResponseFactory).singleton(), createValidatorSetUpdate: asValue(createValidatorSetUpdate), identityQueryHandler: asFunction(identityQueryHandlerFactory).singleton(), @@ -730,11 +735,10 @@ function createDIContainer(options) { wrappedDeliverTx: asFunction(( wrapInErrorHandler, - enrichErrorWithConsensusError, + enrichErrorWithContextError, deliverTx, ) => wrapInErrorHandler( - enrichErrorWithConsensusError(deliverTx), - { respondWithInternalError: true }, + enrichErrorWithContextError(deliverTx), )).singleton(), endBlock: asFunction(endBlockFactory).singleton(), @@ -768,7 +772,7 @@ function createDIContainer(options) { verifyVoteExtensionHandler: asFunction(verifyVoteExtensionHandlerFactory).singleton(), wrapInErrorHandler: asFunction(wrapInErrorHandlerFactory).singleton(), - enrichErrorWithConsensusError: asFunction(enrichErrorWithConsensusErrorFactory).singleton(), + enrichErrorWithContextError: asFunction(enrichErrorWithConsensusErrorFactory).singleton(), errorHandler: asFunction(errorHandlerFactory).singleton(), abciHandlers: asFunction(( @@ -776,7 +780,7 @@ function createDIContainer(options) { checkTxHandler, initChainHandler, wrapInErrorHandler, - enrichErrorWithConsensusError, + enrichErrorWithContextError, queryHandler, extendVoteHandler, finalizeBlockHandler, @@ -784,15 +788,15 @@ function createDIContainer(options) { processProposalHandler, verifyVoteExtensionHandler, ) => ({ - info: infoHandler, - checkTx: wrapInErrorHandler(checkTxHandler, { respondWithInternalError: true }), - initChain: initChainHandler, - query: wrapInErrorHandler(queryHandler, { respondWithInternalError: true }), - extendVote: enrichErrorWithConsensusError(extendVoteHandler), - finalizeBlock: enrichErrorWithConsensusError(finalizeBlockHandler), - prepareProposal: enrichErrorWithConsensusError(prepareProposalHandler), - processProposal: enrichErrorWithConsensusError(processProposalHandler), - verifyVoteExtension: enrichErrorWithConsensusError(verifyVoteExtensionHandler), + info: enrichErrorWithContextError(infoHandler), + checkTx: wrapInErrorHandler(enrichErrorWithContextError(checkTxHandler)), + initChain: enrichErrorWithContextError(initChainHandler), + query: wrapInErrorHandler(enrichErrorWithContextError(queryHandler)), + extendVote: enrichErrorWithContextError(extendVoteHandler), + finalizeBlock: enrichErrorWithContextError(finalizeBlockHandler), + prepareProposal: enrichErrorWithContextError(prepareProposalHandler), + processProposal: enrichErrorWithContextError(processProposalHandler), + verifyVoteExtension: enrichErrorWithContextError(verifyVoteExtensionHandler), })).singleton(), closeAbciServer: asFunction(closeAbciServerFactory).singleton(), diff --git a/packages/js-drive/lib/errorHandlerFactory.js b/packages/js-drive/lib/errorHandlerFactory.js index cf06daf200b..7aafd8c15dd 100644 --- a/packages/js-drive/lib/errorHandlerFactory.js +++ b/packages/js-drive/lib/errorHandlerFactory.js @@ -37,7 +37,7 @@ function errorHandlerFactory(logger, container, closeAbciServer) { console.log(printErrorFace()); errors.forEach((e) => { - (error.consensusLogger || logger).fatal({ err: e }, e.message); + (error.contextLogger || logger).fatal({ err: e }, e.message); }); } finally { await container.dispose(); diff --git a/packages/js-drive/test/unit/abci/errors/enrichErrorWithConsensusLoggerFactory.spec.js b/packages/js-drive/test/unit/abci/errors/enrichErrorWithConsensusLoggerFactory.spec.js index 7ed0c86195a..f9f99cb23a3 100644 --- a/packages/js-drive/test/unit/abci/errors/enrichErrorWithConsensusLoggerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/errors/enrichErrorWithConsensusLoggerFactory.spec.js @@ -1,4 +1,4 @@ -const enrichErrorWithConsensusLoggerFactory = require('../../../../lib/abci/errors/enrichErrorWithConsensusLoggerFactory'); +const enrichErrorWithConsensusLoggerFactory = require('../../../../lib/abci/errors/enrichErrorWithContextLoggerFactory'); const BlockExecutionContextMock = require('../../../../lib/test/mock/BlockExecutionContextMock'); const LoggerMock = require('../../../../lib/test/mock/LoggerMock'); From 192b36f2adce6a8e1cff1d57016b6f370da85ce6 Mon Sep 17 00:00:00 2001 From: Konstantin Shuplenkov Date: Thu, 22 Dec 2022 17:50:00 +0300 Subject: [PATCH 44/54] WIP --- .../abci/errors/createContextLoggerFactory.js | 11 +++++++++++ .../lib/abci/handlers/checkTxHandlerFactory.js | 1 + .../handlers/finalizeBlockHandlerFactory.js | 18 +++++++++--------- .../lib/abci/handlers/infoHandlerFactory.js | 1 + .../abci/handlers/initChainHandlerFactory.js | 1 + .../handlers/prepareProposalHandlerFactory.js | 1 + .../handlers/processProposalHandlerFactory.js | 1 + .../abci/handlers/proposal/deliverTxFactory.js | 1 + .../lib/abci/handlers/queryHandlerFactory.js | 2 ++ 9 files changed, 28 insertions(+), 9 deletions(-) diff --git a/packages/js-drive/lib/abci/errors/createContextLoggerFactory.js b/packages/js-drive/lib/abci/errors/createContextLoggerFactory.js index 4e79ff71d92..4e9b52354f6 100644 --- a/packages/js-drive/lib/abci/errors/createContextLoggerFactory.js +++ b/packages/js-drive/lib/abci/errors/createContextLoggerFactory.js @@ -1,4 +1,15 @@ +/** + * + * @param {AsyncLocalStorage} abciAsyncLocalStorage + * @return {createContextLogger} + */ function createContextLoggerFactory(abciAsyncLocalStorage) { + /** + * @typedef {createContextLogger} + * @param {Logger} logger + * @param {Object} context + * @return {Logger} + */ function createContextLogger(logger, context) { const contextLogger = logger.child(context); diff --git a/packages/js-drive/lib/abci/handlers/checkTxHandlerFactory.js b/packages/js-drive/lib/abci/handlers/checkTxHandlerFactory.js index d97e4b865a9..05057f6a239 100644 --- a/packages/js-drive/lib/abci/handlers/checkTxHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/checkTxHandlerFactory.js @@ -9,6 +9,7 @@ const { /** * @param {unserializeStateTransition} unserializeStateTransition * @param {AsyncLocalStorage} unserializeStateTransition + * @param {createContextLogger} createContextLogger * @param {Logger} logger * * @returns {checkTxHandler} diff --git a/packages/js-drive/lib/abci/handlers/finalizeBlockHandlerFactory.js b/packages/js-drive/lib/abci/handlers/finalizeBlockHandlerFactory.js index b09282b9af7..6fcd3703a0b 100644 --- a/packages/js-drive/lib/abci/handlers/finalizeBlockHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/finalizeBlockHandlerFactory.js @@ -19,6 +19,8 @@ const lodashCloneDeep = require('lodash/cloneDeep'); * @param {BlockExecutionContext} latestBlockExecutionContext * @param {BlockExecutionContext} proposalBlockExecutionContext * @param {processProposal} processProposal + * @param {createContextLogger} createContextLogger + * */ function finalizeBlockHandlerFactory( groveDBStore, @@ -29,6 +31,7 @@ function finalizeBlockHandlerFactory( latestBlockExecutionContext, proposalBlockExecutionContext, processProposal, + createContextLogger, ) { /** * @typedef finalizeBlockHandler @@ -43,7 +46,7 @@ function finalizeBlockHandlerFactory( round, } = request; - const consensusLogger = logger.child({ + const contextLogger = createContextLogger(logger, { height: height.toString(), round, abciMethod: 'finalizeBlock', @@ -52,13 +55,13 @@ function finalizeBlockHandlerFactory( const requestToLog = lodashCloneDeep(request); delete requestToLog.block.data; - consensusLogger.debug('FinalizeBlock ABCI method requested'); - consensusLogger.trace({ abciRequest: requestToLog }); + contextLogger.debug('FinalizeBlock ABCI method requested'); + contextLogger.trace({ abciRequest: requestToLog }); const lastProcessedRound = proposalBlockExecutionContext.getRound(); if (lastProcessedRound !== round) { - consensusLogger.warn({ + contextLogger.warn({ lastProcessedRound, round, }, `Finalizing previously executed round ${round} instead of the last known ${lastProcessedRound}`); @@ -88,10 +91,7 @@ function finalizeBlockHandlerFactory( round, }); - await processProposal(processProposalRequest, consensusLogger); - - // Revert consensus logger - proposalBlockExecutionContext.setConsensusLogger(consensusLogger); + await processProposal(processProposalRequest, contextLogger); } proposalBlockExecutionContext.setLastCommitInfo(commitInfo); @@ -138,7 +138,7 @@ function finalizeBlockHandlerFactory( const blockExecutionTimings = executionTimer.stopTimer('blockExecution'); - consensusLogger.info( + contextLogger.info( { timings: blockExecutionTimings, }, diff --git a/packages/js-drive/lib/abci/handlers/infoHandlerFactory.js b/packages/js-drive/lib/abci/handlers/infoHandlerFactory.js index 68e3a9fd56f..0d056c005d0 100644 --- a/packages/js-drive/lib/abci/handlers/infoHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/infoHandlerFactory.js @@ -17,6 +17,7 @@ const { version: driveVersion } = require('../../../package.json'); * @param {updateSimplifiedMasternodeList} updateSimplifiedMasternodeList * @param {BaseLogger} logger * @param {GroveDBStore} groveDBStore + * @param {createContextLogger} createContextLogger * @return {infoHandler} */ function infoHandlerFactory( diff --git a/packages/js-drive/lib/abci/handlers/initChainHandlerFactory.js b/packages/js-drive/lib/abci/handlers/initChainHandlerFactory.js index 9675a28ea4d..55baffe64fd 100644 --- a/packages/js-drive/lib/abci/handlers/initChainHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/initChainHandlerFactory.js @@ -22,6 +22,7 @@ const protoTimestampToMillis = require('../../util/protoTimestampToMillis'); * @param {GroveDBStore} groveDBStore * @param {RSAbci} rsAbci * @param {createCoreChainLockUpdate} createCoreChainLockUpdate + * @param {createContextLogger} createContextLogger * @return {initChainHandler} */ function initChainHandlerFactory( diff --git a/packages/js-drive/lib/abci/handlers/prepareProposalHandlerFactory.js b/packages/js-drive/lib/abci/handlers/prepareProposalHandlerFactory.js index ae2aad4869d..0c0b84d915d 100644 --- a/packages/js-drive/lib/abci/handlers/prepareProposalHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/prepareProposalHandlerFactory.js @@ -24,6 +24,7 @@ const txAction = { * @param {endBlock} endBlock * @param {createCoreChainLockUpdate} createCoreChainLockUpdate * @param {ExecutionTimer} executionTimer + * @param {createContextLogger} createContextLogger * @return {prepareProposalHandler} */ function prepareProposalHandlerFactory( diff --git a/packages/js-drive/lib/abci/handlers/processProposalHandlerFactory.js b/packages/js-drive/lib/abci/handlers/processProposalHandlerFactory.js index c009f28a37b..ee942a773be 100644 --- a/packages/js-drive/lib/abci/handlers/processProposalHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/processProposalHandlerFactory.js @@ -14,6 +14,7 @@ const statuses = require('./proposal/statuses'); * @param {verifyChainLock} verifyChainLock * @param {processProposal} processProposal * @param {BlockExecutionContext} proposalBlockExecutionContext + * @param {createContextLogger} createContextLogger * @return {processProposalHandler} */ function processProposalHandlerFactory( diff --git a/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js b/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js index a0015219fdd..5846867e682 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js @@ -25,6 +25,7 @@ const TIMERS = require('../timers'); * @param {DashPlatformProtocol} transactionalDpp * @param {BlockExecutionContext} proposalBlockExecutionContext * @param {ExecutionTimer} executionTimer + * @param {createContextLogger} createContextLogger * * @return {deliverTx} */ diff --git a/packages/js-drive/lib/abci/handlers/queryHandlerFactory.js b/packages/js-drive/lib/abci/handlers/queryHandlerFactory.js index 800639d7ca1..338538af6a0 100644 --- a/packages/js-drive/lib/abci/handlers/queryHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/queryHandlerFactory.js @@ -5,6 +5,8 @@ const InvalidArgumentAbciError = require('../errors/InvalidArgumentAbciError'); /** * @param {Object} queryHandlerRouter * @param {Function} sanitizeUrl + * @param {BaseLogger} logger + * @param {createContextLogger} createContextLogger * @return {queryHandler} */ function queryHandlerFactory(queryHandlerRouter, sanitizeUrl, logger, createContextLogger) { From bce914a67310871336c1a34a6271ee4439c8b43c Mon Sep 17 00:00:00 2001 From: Konstantin Shuplenkov Date: Thu, 22 Dec 2022 18:12:04 +0300 Subject: [PATCH 45/54] WIP --- .../errors/enrichErrorWithContextLoggerFactory.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js b/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js index 5f58cb79e54..451b745c2d0 100644 --- a/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js +++ b/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js @@ -22,14 +22,12 @@ function enrichErrorWithContextLoggerFactory() { async function methodHandler(...args) { const asyncLocalStorage = new AsyncLocalStorage(); - return asyncLocalStorage.run(new Map(), () => { - return method(...args).catch((error) => { - // eslint-disable-next-line no-param-reassign - error.contextLogger = asyncLocalStorage.getStore().get('logger'); + return asyncLocalStorage.run(new Map(), () => method(...args).catch((error) => { + // eslint-disable-next-line no-param-reassign + error.contextLogger = asyncLocalStorage.getStore().get('logger'); - return Promise.reject(error); - }); - }); + return Promise.reject(error); + })); } return methodHandler; From 3eac947a515c97ee16a2ac0e172da71f3fb2d742 Mon Sep 17 00:00:00 2001 From: Konstantin Shuplenkov Date: Fri, 23 Dec 2022 11:38:02 +0300 Subject: [PATCH 46/54] WIP --- .../abci/handlers/extendVoteHandlerFactory.js | 1 + .../createConsensusParamUpdateFactory.js | 6 ++-- .../createCoreChainLockUpdateFactory.js | 6 ++-- .../handlers/proposal/deliverTxFactory.js | 28 +++++++++---------- .../abci/handlers/proposal/endBlockFactory.js | 16 +++++------ .../proposal/processProposalFactory.js | 16 +++++------ ...otateAndCreateValidatorSetUpdateFactory.js | 6 ++-- .../verifyVoteExtensionHandlerFactory.js | 4 +-- packages/js-drive/lib/createDIContainer.js | 7 ++++- .../lib/dpp/LoggedStateRepositoryDecorator.js | 2 +- .../getBlockExecutionContextObjectFixture.js | 4 +-- .../test/mock/BlockExecutionContextMock.js | 8 +++--- .../BlockExecutionContextRepository.spec.js | 12 ++++---- ...richErrorWithContextLoggerFactory.spec.js} | 18 ++++++------ .../handlers/extendVoteHandlerFactory.spec.js | 17 +++++++++-- .../proposal/beginBlockFactory.spec.js | 2 +- .../verifyVoteExtensionHandlerFactory.spec.js | 2 +- .../BlockExecutionContext.spec.js | 22 +++++++-------- .../LoggedStateRepositoryDecorator.spec.js | 2 +- .../unit/util/errorHandlerFactory.spec.js | 4 +-- 20 files changed, 99 insertions(+), 84 deletions(-) rename packages/js-drive/test/unit/abci/errors/{enrichErrorWithConsensusLoggerFactory.spec.js => enrichErrorWithContextLoggerFactory.spec.js} (50%) diff --git a/packages/js-drive/lib/abci/handlers/extendVoteHandlerFactory.js b/packages/js-drive/lib/abci/handlers/extendVoteHandlerFactory.js index 3c6ad431870..9d0e1ef786c 100644 --- a/packages/js-drive/lib/abci/handlers/extendVoteHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/extendVoteHandlerFactory.js @@ -11,6 +11,7 @@ const { /** * @param {BlockExecutionContext} proposalBlockExecutionContext + * @param {createContextLogger} createContextLogger * * @return {extendVoteHandler} */ diff --git a/packages/js-drive/lib/abci/handlers/proposal/createConsensusParamUpdateFactory.js b/packages/js-drive/lib/abci/handlers/proposal/createConsensusParamUpdateFactory.js index edb2aeb79f1..4ac31ad0377 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/createConsensusParamUpdateFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/createConsensusParamUpdateFactory.js @@ -21,10 +21,10 @@ function createConsensusParamUpdateFactory( * @typedef createConsensusParamUpdate * @param {number} height * @param {number} round - * @param {BaseLogger} consensusLogger + * @param {BaseLogger} contextLogger * @return {Promise} */ - async function createConsensusParamUpdate(height, round, consensusLogger) { + async function createConsensusParamUpdate(height, round, contextLogger) { const contextVersion = proposalBlockExecutionContext.getVersion(); // Update consensus params feature flag @@ -52,7 +52,7 @@ function createConsensusParamUpdateFactory( version, }); - consensusLogger.info( + contextLogger.info( { consensusParamUpdates, }, diff --git a/packages/js-drive/lib/abci/handlers/proposal/createCoreChainLockUpdateFactory.js b/packages/js-drive/lib/abci/handlers/proposal/createCoreChainLockUpdateFactory.js index d13008a3aea..8a712216696 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/createCoreChainLockUpdateFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/createCoreChainLockUpdateFactory.js @@ -17,10 +17,10 @@ function createCoreChainLockUpdateFactory( * @typedef createCoreChainLockUpdate * @param {number} contextCoreChainLockedHeight * @param {number} round - * @param {BaseLogger} consensusLogger + * @param {BaseLogger} contextLogger * @return {Promise} */ - async function createCoreChainLockUpdate(contextCoreChainLockedHeight, round, consensusLogger) { + async function createCoreChainLockUpdate(contextCoreChainLockedHeight, round, contextLogger) { // Update Core Chain Locks const coreChainLock = latestCoreChainLock.getChainLock(); @@ -32,7 +32,7 @@ function createCoreChainLockUpdateFactory( signature: coreChainLock.signature, }); - consensusLogger.debug( + contextLogger.debug( { nextCoreChainLockHeight: coreChainLock.height, }, diff --git a/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js b/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js index 5846867e682..8c21c54da05 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/deliverTxFactory.js @@ -41,12 +41,12 @@ function deliverTxFactory( * * @param {Buffer} stateTransitionByteArray * @param {number} round - * @param {BaseLogger} consensusLogger + * @param {BaseLogger} contextLogger * @return {Promise<{ * code: number, * fees: BlockFeeResult}>} */ - async function deliverTx(stateTransitionByteArray, round, consensusLogger) { + async function deliverTx(stateTransitionByteArray, round, contextLogger) { const blockHeight = proposalBlockExecutionContext.getHeight(); // Start execution timer @@ -67,16 +67,16 @@ function deliverTxFactory( .toString('hex') .toUpperCase(); - const txConsensusLogger = createContextLogger(consensusLogger, { + const txContextLogger = createContextLogger(contextLogger, { txId: stHash, }); - txConsensusLogger.info(`Deliver state transition ${stHash} from block #${blockHeight}`); + txContextLogger.info(`Deliver state transition ${stHash} from block #${blockHeight}`); const stateTransition = await transactionalUnserializeStateTransition( stateTransitionByteArray, { - logger: txConsensusLogger, + logger: txContextLogger, executionTimer, }, ); @@ -98,8 +98,8 @@ function deliverTxFactory( const consensusError = result.getFirstError(); const message = 'State transition is invalid against the state'; - txConsensusLogger.info(message); - txConsensusLogger.debug({ + txContextLogger.info(message); + txContextLogger.debug({ consensusError, }); @@ -122,7 +122,7 @@ function deliverTxFactory( const actualStateTransitionOperations = stateTransition.getExecutionContext().getOperations(); if (actualStateTransitionFees.total > predictedStateTransitionFees.total) { - txConsensusLogger.warn({ + txContextLogger.warn({ predictedFee: predictedStateTransitionFees.total, actualFee: actualStateTransitionFees.total, }, `Actual fees are greater than predicted for ${actualStateTransitionFees.total - predictedStateTransitionFees.total} credits`); @@ -155,7 +155,7 @@ function deliverTxFactory( const description = DATA_CONTRACT_ACTION_DESCRIPTIONS[stateTransition.getType()]; - txConsensusLogger.info( + txContextLogger.info( { dataContractId: dataContract.getId().toString(), }, @@ -167,7 +167,7 @@ function deliverTxFactory( case stateTransitionTypes.IDENTITY_CREATE: { const identityId = stateTransition.getIdentityId(); - txConsensusLogger.info( + txContextLogger.info( { identityId: identityId.toString(), }, @@ -179,7 +179,7 @@ function deliverTxFactory( case stateTransitionTypes.IDENTITY_TOP_UP: { const identityId = stateTransition.getIdentityId(); - txConsensusLogger.info( + txContextLogger.info( { identityId: identityId.toString(), }, @@ -191,7 +191,7 @@ function deliverTxFactory( case stateTransitionTypes.IDENTITY_UPDATE: { const identityId = stateTransition.getIdentityId(); - txConsensusLogger.info( + txContextLogger.info( { identityId: identityId.toString(), }, @@ -203,7 +203,7 @@ function deliverTxFactory( stateTransition.getTransitions().forEach((transition) => { const description = DOCUMENT_ACTION_DESCRIPTIONS[transition.getAction()]; - txConsensusLogger.info( + txContextLogger.info( { documentId: transition.getId().toString(), }, @@ -219,7 +219,7 @@ function deliverTxFactory( const deliverTxTiming = executionTimer.stopTimer(TIMERS.DELIVER_TX.OVERALL); - txConsensusLogger.trace( + txContextLogger.trace( { timings: { overall: deliverTxTiming, diff --git a/packages/js-drive/lib/abci/handlers/proposal/endBlockFactory.js b/packages/js-drive/lib/abci/handlers/proposal/endBlockFactory.js index a121a3d0536..1da21cc06d4 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/endBlockFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/endBlockFactory.js @@ -37,7 +37,7 @@ function endBlockFactory( * feeRefundsSum: number * } request.fees * @param {number} request.coreChainLockedHeight - * @param {BaseLogger} consensusLogger + * @param {BaseLogger} contextLogger * @return {Promise<{ * consensusParamUpdates: ConsensusParams, * validatorSetUpdate: ValidatorSetUpdate, @@ -46,7 +46,7 @@ function endBlockFactory( */ async function endBlock( request, - consensusLogger, + contextLogger, ) { const { height, @@ -61,11 +61,11 @@ function endBlockFactory( fees, }; - consensusLogger.debug(rsRequest, 'Request RS Drive\'s BlockEnd method'); + contextLogger.debug(rsRequest, 'Request RS Drive\'s BlockEnd method'); const rsResponse = await rsAbci.blockEnd(rsRequest, true); - consensusLogger.debug(rsResponse, 'RS Drive\'s BlockEnd method response'); + contextLogger.debug(rsResponse, 'RS Drive\'s BlockEnd method response'); const { currentEpochIndex } = proposalBlockExecutionContext.getEpochInfo(); @@ -76,7 +76,7 @@ function endBlockFactory( } = fees; if (processingFee > 0 || storageFee > 0) { - consensusLogger.debug({ + contextLogger.debug({ currentEpochIndex, processingFee, storageFee, @@ -85,20 +85,20 @@ function endBlockFactory( } if (rsResponse.proposersPaidCount) { - consensusLogger.debug({ + contextLogger.debug({ currentEpochIndex, proposersPaidCount: rsResponse.proposersPaidCount, paidEpochIndex: rsResponse.paidEpochIndex, }, `${rsResponse.proposersPaidCount} masternodes were paid for epoch #${rsResponse.paidEpochIndex}`); } - const consensusParamUpdates = await createConsensusParamUpdate(height, round, consensusLogger); + const consensusParamUpdates = await createConsensusParamUpdate(height, round, contextLogger); const validatorSetUpdate = await rotateAndCreateValidatorSetUpdate( height, coreChainLockedHeight, round, - consensusLogger, + contextLogger, ); const appHash = await groveDBStore.getRootHash({ useTransaction: true }); diff --git a/packages/js-drive/lib/abci/handlers/proposal/processProposalFactory.js b/packages/js-drive/lib/abci/handlers/proposal/processProposalFactory.js index d80fb6da913..506c1fc9b57 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/processProposalFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/processProposalFactory.js @@ -29,11 +29,11 @@ function processProposalFactory( ) { /** * @param {abci.RequestProcessProposal} request - * @param {BaseLogger} consensusLogger + * @param {BaseLogger} contextLogger * * @typedef processProposal */ - async function processProposal(request, consensusLogger) { + async function processProposal(request, contextLogger) { const { height, txs, @@ -45,7 +45,7 @@ function processProposalFactory( round, } = request; - consensusLogger.info(`Processing a block proposal for height #${height} round #${round}`); + contextLogger.info(`Processing a block proposal for height #${height} round #${round}`); await beginBlock( { @@ -57,7 +57,7 @@ function processProposalFactory( proposerProTxHash: Buffer.from(proposerProTxHash), round, }, - consensusLogger, + contextLogger, ); const txResults = []; @@ -76,7 +76,7 @@ function processProposalFactory( code, info, fees, - } = await wrappedDeliverTx(tx, round, consensusLogger); + } = await wrappedDeliverTx(tx, round, contextLogger); if (code === 0) { validTxCount += 1; @@ -96,7 +96,7 @@ function processProposalFactory( } // Revert consensus logger after deliverTx - proposalBlockExecutionContext.setConsensusLogger(consensusLogger); + proposalBlockExecutionContext.setContextLogger(contextLogger); const { consensusParamUpdates, @@ -107,11 +107,11 @@ function processProposalFactory( round, fees: feeResults, coreChainLockedHeight, - }, consensusLogger); + }, contextLogger); const roundExecutionTime = executionTimer.getTimer('roundExecution', true); - consensusLogger.info( + contextLogger.info( { roundExecutionTime, validTxCount, diff --git a/packages/js-drive/lib/abci/handlers/proposal/rotateAndCreateValidatorSetUpdateFactory.js b/packages/js-drive/lib/abci/handlers/proposal/rotateAndCreateValidatorSetUpdateFactory.js index 1b02f223a92..9307c5f05c2 100644 --- a/packages/js-drive/lib/abci/handlers/proposal/rotateAndCreateValidatorSetUpdateFactory.js +++ b/packages/js-drive/lib/abci/handlers/proposal/rotateAndCreateValidatorSetUpdateFactory.js @@ -14,14 +14,14 @@ function rotateAndCreateValidatorSetUpdateFactory( * @param {number} height * @param {number} coreChainLockedHeight * @param {number} round - * @param {BaseLogger} consensusLogger + * @param {BaseLogger} contextLogger * @return {Promise} */ async function rotateAndCreateValidatorSetUpdate( height, coreChainLockedHeight, round, - consensusLogger, + contextLogger, ) { const lastCommitInfo = proposalBlockExecutionContext.getLastCommitInfo(); @@ -34,7 +34,7 @@ function rotateAndCreateValidatorSetUpdateFactory( const { quorumHash } = validatorSet.getQuorum(); - consensusLogger.debug( + contextLogger.debug( { quorumHash, }, diff --git a/packages/js-drive/lib/abci/handlers/verifyVoteExtensionHandlerFactory.js b/packages/js-drive/lib/abci/handlers/verifyVoteExtensionHandlerFactory.js index 1c5a56479b9..948e539ce0b 100644 --- a/packages/js-drive/lib/abci/handlers/verifyVoteExtensionHandlerFactory.js +++ b/packages/js-drive/lib/abci/handlers/verifyVoteExtensionHandlerFactory.js @@ -22,12 +22,12 @@ function verifyVoteExtensionHandlerFactory(proposalBlockExecutionContext) { * @return {Promise} */ async function verifyVoteExtensionHandler() { - const consensusLogger = proposalBlockExecutionContext.getConsensusLogger() + const contextLogger = proposalBlockExecutionContext.getContextLogger() .child({ abciMethod: 'verifyVoteExtension', }); - consensusLogger.debug('VerifyVote ABCI method requested'); + contextLogger.debug('VerifyVote ABCI method requested'); // TODO Verify withdrawal vote extensions and add logs diff --git a/packages/js-drive/lib/createDIContainer.js b/packages/js-drive/lib/createDIContainer.js index 792f3186861..d9208acb9d6 100644 --- a/packages/js-drive/lib/createDIContainer.js +++ b/packages/js-drive/lib/createDIContainer.js @@ -697,7 +697,12 @@ function createDIContainer(options) { */ container.register({ createContextLogger: asFunction(createContextLoggerFactory), - abciAsyncLocalStorage: asValue(new AsyncLocalStorage()), + abciAsyncLocalStorage: asFunction(() => { + const asyncLocalStorage = new AsyncLocalStorage(); + asyncLocalStorage.enterWith(new Map()); + return asyncLocalStorage; + }).singleton(), + // abciAsyncLocalStorage: asValue(new AsyncLocalStorage()), createQueryResponse: asFunction(createQueryResponseFactory).singleton(), createValidatorSetUpdate: asValue(createValidatorSetUpdate), identityQueryHandler: asFunction(identityQueryHandlerFactory).singleton(), diff --git a/packages/js-drive/lib/dpp/LoggedStateRepositoryDecorator.js b/packages/js-drive/lib/dpp/LoggedStateRepositoryDecorator.js index 591e580409e..46fe71eca5b 100644 --- a/packages/js-drive/lib/dpp/LoggedStateRepositoryDecorator.js +++ b/packages/js-drive/lib/dpp/LoggedStateRepositoryDecorator.js @@ -21,7 +21,7 @@ class LoggedStateRepositoryDecorator { * @param {object} response - response of the state repository call */ log(method, parameters, response) { - const logger = this.blockExecutionContext.getConsensusLogger(); + const logger = this.blockExecutionContext.getContextLogger(); logger.trace({ stateRepository: { diff --git a/packages/js-drive/lib/test/fixtures/getBlockExecutionContextObjectFixture.js b/packages/js-drive/lib/test/fixtures/getBlockExecutionContextObjectFixture.js index c9abca30cd6..e0d0de4daa1 100644 --- a/packages/js-drive/lib/test/fixtures/getBlockExecutionContextObjectFixture.js +++ b/packages/js-drive/lib/test/fixtures/getBlockExecutionContextObjectFixture.js @@ -25,7 +25,7 @@ const { hash } = require('@dashevo/dpp/lib/util/hash'); * version: number, * timeMs: number, * validTxs: number, - * consensusLogger: Logger, + * contextLogger: Logger, * withdrawalTransactionsMap: Object, * round: number, * }} @@ -52,7 +52,7 @@ function getBlockExecutionContextObjectFixture(dataContract = getDataContractFix height: 10, coreChainLockedHeight: 10, version, - consensusLogger: pino(), + contextLogger: pino(), epochInfo: { height: 1, timeMs: 100, diff --git a/packages/js-drive/lib/test/mock/BlockExecutionContextMock.js b/packages/js-drive/lib/test/mock/BlockExecutionContextMock.js index dd2f95d5bee..82c68b9b5c2 100644 --- a/packages/js-drive/lib/test/mock/BlockExecutionContextMock.js +++ b/packages/js-drive/lib/test/mock/BlockExecutionContextMock.js @@ -11,8 +11,8 @@ * @method getLastCommitInfo * @method getValidTxCount * @method getInvalidTxCount - * @method setConsensusLogger - * @method getConsensusLogger + * @method setContextLogger + * @method getContextLogger * @method getRound * @method fromObject * @method toObject @@ -40,8 +40,8 @@ class BlockExecutionContextMock { this.getVersion = sinon.stub(); this.setLastCommitInfo = sinon.stub(); this.getLastCommitInfo = sinon.stub(); - this.setConsensusLogger = sinon.stub(); - this.getConsensusLogger = sinon.stub(); + this.setContextLogger = sinon.stub(); + this.getContextLogger = sinon.stub(); this.setWithdrawalTransactionsMap = sinon.stub(); this.getWithdrawalTransactionsMap = sinon.stub(); this.getRound = sinon.stub(); diff --git a/packages/js-drive/test/integration/blockExecution/BlockExecutionContextRepository.spec.js b/packages/js-drive/test/integration/blockExecution/BlockExecutionContextRepository.spec.js index 6895d6e4359..4d453ade22a 100644 --- a/packages/js-drive/test/integration/blockExecution/BlockExecutionContextRepository.spec.js +++ b/packages/js-drive/test/integration/blockExecution/BlockExecutionContextRepository.spec.js @@ -57,7 +57,7 @@ describe('BlockExecutionContextRepository', () => { const rawBlockExecutionContext = cbor.decode(blockExecutionContextEncoded); expect(rawBlockExecutionContext).to.deep.equal(blockExecutionContext.toObject({ - skipConsensusLogger: true, + skipContextLogger: true, skipPrepareProposalResult: true, })); }); @@ -66,7 +66,7 @@ describe('BlockExecutionContextRepository', () => { await store.putAux( BlockExecutionContextRepository.EXTERNAL_STORE_KEY_NAME, await cbor.encodeAsync(blockExecutionContext.toObject({ - skipConsensusLogger: true, + skipContextLogger: true, })), options, ); @@ -76,10 +76,10 @@ describe('BlockExecutionContextRepository', () => { expect(fetchedBlockExecutionContext).to.be.instanceOf(BlockExecutionContext); expect(fetchedBlockExecutionContext.toObject({ - skipConsensusLogger: true, + skipContextLogger: true, skipPrepareProposalResult: true, })).to.deep.equal(blockExecutionContext.toObject({ - skipConsensusLogger: true, + skipContextLogger: true, skipPrepareProposalResult: true, })); }); @@ -102,10 +102,10 @@ describe('BlockExecutionContextRepository', () => { expect(fetchedBlockExecutionContext).to.be.instanceOf(BlockExecutionContext); expect(fetchedBlockExecutionContext.toObject({ - skipConsensusLogger: true, + skipContextLogger: true, skipPrepareProposalResult: true, })).to.deep.equal(blockExecutionContext.toObject({ - skipConsensusLogger: true, + skipContextLogger: true, skipPrepareProposalResult: true, })); }); diff --git a/packages/js-drive/test/unit/abci/errors/enrichErrorWithConsensusLoggerFactory.spec.js b/packages/js-drive/test/unit/abci/errors/enrichErrorWithContextLoggerFactory.spec.js similarity index 50% rename from packages/js-drive/test/unit/abci/errors/enrichErrorWithConsensusLoggerFactory.spec.js rename to packages/js-drive/test/unit/abci/errors/enrichErrorWithContextLoggerFactory.spec.js index f9f99cb23a3..1083377fbc8 100644 --- a/packages/js-drive/test/unit/abci/errors/enrichErrorWithConsensusLoggerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/errors/enrichErrorWithContextLoggerFactory.spec.js @@ -1,31 +1,29 @@ -const enrichErrorWithConsensusLoggerFactory = require('../../../../lib/abci/errors/enrichErrorWithContextLoggerFactory'); +const enrichErrorWithContextLoggerFactory = require('../../../../lib/abci/errors/enrichErrorWithContextLoggerFactory'); const BlockExecutionContextMock = require('../../../../lib/test/mock/BlockExecutionContextMock'); const LoggerMock = require('../../../../lib/test/mock/LoggerMock'); -describe('enrichErrorWithConsensusLoggerFactory', () => { +describe('enrichErrorWithContextLoggerFactory', () => { let blockExecutionContextMock; - let enrichErrorWithConsensusLogger; + let enrichErrorWithContextLogger; let loggerMock; beforeEach(function beforeEach() { loggerMock = new LoggerMock(this.sinon); blockExecutionContextMock = new BlockExecutionContextMock(this.sinon); - blockExecutionContextMock.consensusLogger = loggerMock; + blockExecutionContextMock.contextLogger = loggerMock; - enrichErrorWithConsensusLogger = enrichErrorWithConsensusLoggerFactory( - blockExecutionContextMock, - ); + enrichErrorWithContextLogger = enrichErrorWithContextLoggerFactory(); }); - it('should add consensusLogger from BlockExecutionContext to thrown error', async () => { + it('should add contextLogger from BlockExecutionContext to thrown error', async () => { const error = new Error(); const method = () => { throw error; }; - const methodHandler = enrichErrorWithConsensusLogger(method); + const methodHandler = enrichErrorWithContextLogger(method); try { await methodHandler(); @@ -33,7 +31,7 @@ describe('enrichErrorWithConsensusLoggerFactory', () => { expect.fail('should throw an error'); } catch (e) { expect(e).to.equal(error); - expect(e.consensusLogger).to.equal(loggerMock); + expect(e.contextLogger).to.equal(loggerMock); } }); }); diff --git a/packages/js-drive/test/unit/abci/handlers/extendVoteHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/extendVoteHandlerFactory.spec.js index 5c313aefbe0..6651f65f0ea 100644 --- a/packages/js-drive/test/unit/abci/handlers/extendVoteHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/extendVoteHandlerFactory.spec.js @@ -16,17 +16,24 @@ const LoggerMock = require('../../../../lib/test/mock/LoggerMock'); describe('extendVoteHandlerFactory', () => { let extendVoteHandler; let blockExecutionContextMock; + let createContextLoggerMock; + let loggerMock; beforeEach(function beforeEach() { blockExecutionContextMock = new BlockExecutionContextMock(this.sinon); - const loggerMock = new LoggerMock(this.sinon); + loggerMock = new LoggerMock(this.sinon); - blockExecutionContextMock.getConsensusLogger.returns(loggerMock); + blockExecutionContextMock.getContextLogger.returns(loggerMock); blockExecutionContextMock.getWithdrawalTransactionsMap.returns({}); - extendVoteHandler = extendVoteHandlerFactory(blockExecutionContextMock); + createContextLoggerMock = this.sinon.stub().returns(loggerMock); + + extendVoteHandler = extendVoteHandlerFactory( + blockExecutionContextMock, + createContextLoggerMock, + ); }); it('should return ResponseExtendVote with vote extensions if withdrawal transactions are present', async () => { @@ -53,5 +60,9 @@ describe('extendVoteHandlerFactory', () => { extension: hash(txTwoBytes), }, ]); + + expect(createContextLoggerMock).to.be.calledOnceWith(loggerMock, { + abciMethod: 'extendVote', + }) }); }); diff --git a/packages/js-drive/test/unit/abci/handlers/proposal/beginBlockFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/proposal/beginBlockFactory.spec.js index 51d7d9dface..00f37b1c502 100644 --- a/packages/js-drive/test/unit/abci/handlers/proposal/beginBlockFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/proposal/beginBlockFactory.spec.js @@ -171,7 +171,7 @@ describe('beginBlockFactory', () => { expect(executionTimerMock.startTimer.getCall(1)).to.be.calledWithExactly('roundExecution'); expect(executionTimerMock.startTimer.getCall(0)).to.be.calledWithExactly('blockExecution'); - expect(proposalBlockExecutionContextMock.setConsensusLogger) + expect(proposalBlockExecutionContextMock.setContextLogger) .to.be.calledOnceWithExactly(loggerMock); expect(proposalBlockExecutionContextMock.setHeight) .to.be.calledOnceWithExactly(blockHeight); diff --git a/packages/js-drive/test/unit/abci/handlers/verifyVoteExtensionHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/verifyVoteExtensionHandlerFactory.spec.js index d1821dc96de..722f1443d1d 100644 --- a/packages/js-drive/test/unit/abci/handlers/verifyVoteExtensionHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/verifyVoteExtensionHandlerFactory.spec.js @@ -17,7 +17,7 @@ describe('verifyVoteExtensionHandlerFactory', () => { proposalBlockExecutionContextMock = new BlockExecutionContextMock(this.sinon); const loggerMock = new LoggerMock(this.sinon); - proposalBlockExecutionContextMock.getConsensusLogger.returns(loggerMock); + proposalBlockExecutionContextMock.getContextLogger.returns(loggerMock); verifyVoteExtensionHandler = verifyVoteExtensionHandlerFactory( proposalBlockExecutionContextMock, diff --git a/packages/js-drive/test/unit/blockExecution/BlockExecutionContext.spec.js b/packages/js-drive/test/unit/blockExecution/BlockExecutionContext.spec.js index 90b09bbb355..778713b9d4d 100644 --- a/packages/js-drive/test/unit/blockExecution/BlockExecutionContext.spec.js +++ b/packages/js-drive/test/unit/blockExecution/BlockExecutionContext.spec.js @@ -37,7 +37,7 @@ describe('BlockExecutionContext', () => { lastCommitInfo = CommitInfo.fromObject(plainObject.lastCommitInfo); - logger = plainObject.consensusLogger; + logger = plainObject.contextLogger; height = Long.fromNumber(plainObject.height); coreChainLockedHeight = plainObject.coreChainLockedHeight; version = Consensus.fromObject(plainObject.version); @@ -283,7 +283,7 @@ describe('BlockExecutionContext', () => { anotherBlockExecutionContext.height = height; anotherBlockExecutionContext.version = version; anotherBlockExecutionContext.coreChainLockedHeight = coreChainLockedHeight; - anotherBlockExecutionContext.consensusLogger = logger; + anotherBlockExecutionContext.contextLogger = logger; anotherBlockExecutionContext.withdrawalTransactionsMap = plainObject .withdrawalTransactionsMap; anotherBlockExecutionContext.epochInfo = epochInfo; @@ -306,8 +306,8 @@ describe('BlockExecutionContext', () => { expect(blockExecutionContext.coreChainLockedHeight).to.equal( anotherBlockExecutionContext.coreChainLockedHeight, ); - expect(blockExecutionContext.consensusLogger).to.equal( - anotherBlockExecutionContext.consensusLogger, + expect(blockExecutionContext.contextLogger).to.equal( + anotherBlockExecutionContext.contextLogger, ); expect(blockExecutionContext.withdrawalTransactionsMap).to.equal( anotherBlockExecutionContext.withdrawalTransactionsMap, @@ -328,7 +328,7 @@ describe('BlockExecutionContext', () => { blockExecutionContext.height = height; blockExecutionContext.coreChainLockedHeight = coreChainLockedHeight; blockExecutionContext.version = version; - blockExecutionContext.consensusLogger = logger; + blockExecutionContext.contextLogger = logger; blockExecutionContext.epochInfo = epochInfo; blockExecutionContext.timeMs = timeMs; blockExecutionContext.withdrawalTransactionsMap = plainObject.withdrawalTransactionsMap; @@ -338,22 +338,22 @@ describe('BlockExecutionContext', () => { expect(blockExecutionContext.toObject()).to.deep.equal(plainObject); }); - it('should skipConsensusLogger if the option passed', () => { + it('should skipContextLogger if the option passed', () => { blockExecutionContext.dataContracts = [dataContract]; blockExecutionContext.lastCommitInfo = lastCommitInfo; blockExecutionContext.height = height; blockExecutionContext.coreChainLockedHeight = coreChainLockedHeight; blockExecutionContext.version = version; - blockExecutionContext.consensusLogger = logger; + blockExecutionContext.contextLogger = logger; blockExecutionContext.withdrawalTransactionsMap = plainObject.withdrawalTransactionsMap; blockExecutionContext.round = plainObject.round; blockExecutionContext.epochInfo = epochInfo; blockExecutionContext.timeMs = timeMs; blockExecutionContext.prepareProposalResult = prepareProposalResult; - const result = blockExecutionContext.toObject({ skipConsensusLogger: true }); + const result = blockExecutionContext.toObject({ skipContextLogger: true }); - delete plainObject.consensusLogger; + delete plainObject.contextLogger; expect(result).to.deep.equal(plainObject); }); @@ -364,7 +364,7 @@ describe('BlockExecutionContext', () => { blockExecutionContext.height = height; blockExecutionContext.coreChainLockedHeight = coreChainLockedHeight; blockExecutionContext.version = version; - blockExecutionContext.consensusLogger = logger; + blockExecutionContext.contextLogger = logger; blockExecutionContext.withdrawalTransactionsMap = plainObject.withdrawalTransactionsMap; blockExecutionContext.round = plainObject.round; blockExecutionContext.epochInfo = epochInfo; @@ -393,7 +393,7 @@ describe('BlockExecutionContext', () => { expect(blockExecutionContext.height).to.deep.equal(height); expect(blockExecutionContext.version).to.deep.equal(version); expect(blockExecutionContext.coreChainLockedHeight).to.deep.equal(coreChainLockedHeight); - expect(blockExecutionContext.consensusLogger).to.equal(logger); + expect(blockExecutionContext.contextLogger).to.equal(logger); expect(blockExecutionContext.withdrawalTransactionsMap).to.deep.equal( plainObject.withdrawalTransactionsMap, ); diff --git a/packages/js-drive/test/unit/dpp/LoggedStateRepositoryDecorator.spec.js b/packages/js-drive/test/unit/dpp/LoggedStateRepositoryDecorator.spec.js index 81d9e3da579..6d277c074ea 100644 --- a/packages/js-drive/test/unit/dpp/LoggedStateRepositoryDecorator.spec.js +++ b/packages/js-drive/test/unit/dpp/LoggedStateRepositoryDecorator.spec.js @@ -19,7 +19,7 @@ describe('LoggedStateRepositoryDecorator', () => { loggerMock = new LoggerMock(this.sinon); blockExecutionContextMock = new BlockExecutionContextMock(this.sinon); - blockExecutionContextMock.getConsensusLogger.returns(loggerMock); + blockExecutionContextMock.getContextLogger.returns(loggerMock); loggedStateRepositoryDecorator = new LoggedStateRepositoryDecorator( stateRepositoryMock, diff --git a/packages/js-drive/test/unit/util/errorHandlerFactory.spec.js b/packages/js-drive/test/unit/util/errorHandlerFactory.spec.js index eed07ae33d8..c957ee76a55 100644 --- a/packages/js-drive/test/unit/util/errorHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/util/errorHandlerFactory.spec.js @@ -48,12 +48,12 @@ describe('errorHandlerFactory', () => { it('should use consensus logger if it\'s present', async function it() { const error = new Error('message'); - error.consensusLogger = new LoggerMock(this.sinon); + error.contextLogger = new LoggerMock(this.sinon); await errorHandler(error); expect(loggerMock.fatal).to.not.be.called(); - expect(error.consensusLogger.fatal).to.be.calledOnceWithExactly({ err: error }, error.message); + expect(error.contextLogger.fatal).to.be.calledOnceWithExactly({ err: error }, error.message); expect(containerMock.dispose).to.be.calledOnceWithExactly(); From 81c0c2c0e29e1ed7a2d84887cf710bb5d5565906 Mon Sep 17 00:00:00 2001 From: Konstantin Shuplenkov Date: Tue, 10 Jan 2023 12:53:04 +0300 Subject: [PATCH 47/54] WIP --- .../test/unit/abci/handlers/extendVoteHandlerFactory.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/js-drive/test/unit/abci/handlers/extendVoteHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/extendVoteHandlerFactory.spec.js index 6651f65f0ea..3d646e40a31 100644 --- a/packages/js-drive/test/unit/abci/handlers/extendVoteHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/extendVoteHandlerFactory.spec.js @@ -63,6 +63,6 @@ describe('extendVoteHandlerFactory', () => { expect(createContextLoggerMock).to.be.calledOnceWith(loggerMock, { abciMethod: 'extendVote', - }) + }); }); }); From 6fead49f06781ec0f1409b37bd039309fc5b4609 Mon Sep 17 00:00:00 2001 From: Konstantin Shuplenkov Date: Tue, 10 Jan 2023 14:02:51 +0300 Subject: [PATCH 48/54] WIP --- .../enrichErrorWithContextLoggerFactory.js | 19 ++++++++++--------- packages/js-drive/lib/createDIContainer.js | 7 +------ ...nrichErrorWithContextLoggerFactory.spec.js | 15 ++++++++------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js b/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js index 451b745c2d0..ad538b2cfee 100644 --- a/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js +++ b/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js @@ -1,11 +1,10 @@ -const { AsyncLocalStorage } = require('node:async_hooks'); - /** * Add consensus logger to an error (factory) * + * @param {AsyncLocalStorage} abciAsyncLocalStorage * @return {enrichErrorWithContextLogger} */ -function enrichErrorWithContextLoggerFactory() { +function enrichErrorWithContextLoggerFactory(abciAsyncLocalStorage) { /** * Add consensus logger to an error * @@ -20,14 +19,16 @@ function enrichErrorWithContextLoggerFactory() { * @param {*[]} args */ async function methodHandler(...args) { - const asyncLocalStorage = new AsyncLocalStorage(); - return asyncLocalStorage.run(new Map(), () => method(...args).catch((error) => { - // eslint-disable-next-line no-param-reassign - error.contextLogger = asyncLocalStorage.getStore().get('logger'); + return abciAsyncLocalStorage.run( + new Map(), + () => method(...args).catch((error) => { + // eslint-disable-next-line no-param-reassign + error.contextLogger = abciAsyncLocalStorage.getStore().get('logger'); - return Promise.reject(error); - })); + return Promise.reject(error); + }), + ); } return methodHandler; diff --git a/packages/js-drive/lib/createDIContainer.js b/packages/js-drive/lib/createDIContainer.js index 84e0bd95172..f69ad1204f6 100644 --- a/packages/js-drive/lib/createDIContainer.js +++ b/packages/js-drive/lib/createDIContainer.js @@ -697,12 +697,7 @@ function createDIContainer(options) { */ container.register({ createContextLogger: asFunction(createContextLoggerFactory), - abciAsyncLocalStorage: asFunction(() => { - const asyncLocalStorage = new AsyncLocalStorage(); - asyncLocalStorage.enterWith(new Map()); - return asyncLocalStorage; - }).singleton(), - // abciAsyncLocalStorage: asValue(new AsyncLocalStorage()), + abciAsyncLocalStorage: asValue(new AsyncLocalStorage()), createQueryResponse: asFunction(createQueryResponseFactory).singleton(), createValidatorSetUpdate: asValue(createValidatorSetUpdate), identityQueryHandler: asFunction(identityQueryHandlerFactory).singleton(), diff --git a/packages/js-drive/test/unit/abci/errors/enrichErrorWithContextLoggerFactory.spec.js b/packages/js-drive/test/unit/abci/errors/enrichErrorWithContextLoggerFactory.spec.js index 1083377fbc8..d182a4caf3b 100644 --- a/packages/js-drive/test/unit/abci/errors/enrichErrorWithContextLoggerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/errors/enrichErrorWithContextLoggerFactory.spec.js @@ -1,25 +1,26 @@ +const { AsyncLocalStorage } = require('node:async_hooks'); const enrichErrorWithContextLoggerFactory = require('../../../../lib/abci/errors/enrichErrorWithContextLoggerFactory'); -const BlockExecutionContextMock = require('../../../../lib/test/mock/BlockExecutionContextMock'); const LoggerMock = require('../../../../lib/test/mock/LoggerMock'); describe('enrichErrorWithContextLoggerFactory', () => { - let blockExecutionContextMock; let enrichErrorWithContextLogger; let loggerMock; + let asyncLocalStorage; beforeEach(function beforeEach() { loggerMock = new LoggerMock(this.sinon); - blockExecutionContextMock = new BlockExecutionContextMock(this.sinon); - blockExecutionContextMock.contextLogger = loggerMock; + asyncLocalStorage = new AsyncLocalStorage(); - enrichErrorWithContextLogger = enrichErrorWithContextLoggerFactory(); + enrichErrorWithContextLogger = enrichErrorWithContextLoggerFactory(asyncLocalStorage); }); it('should add contextLogger from BlockExecutionContext to thrown error', async () => { - const error = new Error(); + const error = new Error('my error'); + + const method = async () => { + asyncLocalStorage.getStore().set('logger', loggerMock); - const method = () => { throw error; }; From 24d01f713ae55d686d506c6361773f389c90cf56 Mon Sep 17 00:00:00 2001 From: Konstantin Shuplenkov Date: Tue, 10 Jan 2023 14:36:21 +0300 Subject: [PATCH 49/54] WIP --- .../lib/abci/errors/enrichErrorWithContextLoggerFactory.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js b/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js index ad538b2cfee..2493c6d7c92 100644 --- a/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js +++ b/packages/js-drive/lib/abci/errors/enrichErrorWithContextLoggerFactory.js @@ -19,7 +19,6 @@ function enrichErrorWithContextLoggerFactory(abciAsyncLocalStorage) { * @param {*[]} args */ async function methodHandler(...args) { - return abciAsyncLocalStorage.run( new Map(), () => method(...args).catch((error) => { From b71df6fd75ecaa5ace8affe8bb62823fd4f813e0 Mon Sep 17 00:00:00 2001 From: Konstantin Shuplenkov Date: Tue, 10 Jan 2023 16:01:43 +0300 Subject: [PATCH 50/54] WIP --- .../errors/wrapInErrorHandlerFactory.spec.js | 30 ------------------- .../handlers/checkTxHandlerFactory.spec.js | 12 ++++++++ .../finalizeBlockHandlerFactory.spec.js | 24 +++++++++++++++ .../abci/handlers/infoHandlerFactory.spec.js | 22 +++++++++++++- .../handlers/initChainHandlerFactory.spec.js | 7 +++++ .../prepareProposalHandlerFactory.spec.js | 8 +++++ .../processProposalHandlerFactory.spec.js | 8 +++++ .../proposal/deliverTxFactory.spec.js | 22 ++++++++++++-- .../abci/handlers/queryHandlerFactory.spec.js | 7 +++++ 9 files changed, 106 insertions(+), 34 deletions(-) diff --git a/packages/js-drive/test/unit/abci/errors/wrapInErrorHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/errors/wrapInErrorHandlerFactory.spec.js index fb51a98d179..cfda8a6f1d8 100644 --- a/packages/js-drive/test/unit/abci/errors/wrapInErrorHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/errors/wrapInErrorHandlerFactory.spec.js @@ -28,36 +28,6 @@ describe('wrapInErrorHandlerFactory', () => { ); }); - it('should throw an internal error if any Error is thrown in handler', async () => { - const error = new Error('Custom error'); - - methodMock.throws(error); - - try { - await handler(request); - - expect.fail('Internal error must be thrown'); - } catch (e) { - expect(e).to.equal(error); - } - }); - - it('should throw en internal error if an InternalAbciError is thrown in handler', async () => { - const originError = new Error(); - const metadata = { sample: 'data' }; - const error = new InternalAbciError(originError, metadata); - - methodMock.throws(error); - - try { - await handler(request); - - expect.fail('Internal error must be thrown'); - } catch (e) { - expect(e).to.equal(originError); - } - }); - it('should respond with internal error code if any Error is thrown in handler and respondWithInternalError enabled', async () => { handler = wrapInErrorHandler( methodMock, { respondWithInternalError: true }, diff --git a/packages/js-drive/test/unit/abci/handlers/checkTxHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/checkTxHandlerFactory.spec.js index 00f48adde14..fd4a5621278 100644 --- a/packages/js-drive/test/unit/abci/handlers/checkTxHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/checkTxHandlerFactory.spec.js @@ -9,12 +9,15 @@ const { const getIdentityCreateTransitionFixture = require('@dashevo/dpp/lib/test/fixtures/getIdentityCreateTransitionFixture'); const checkTxHandlerFactory = require('../../../../lib/abci/handlers/checkTxHandlerFactory'); +const LoggerMock = require('../../../../lib/test/mock/LoggerMock'); describe('checkTxHandlerFactory', () => { let checkTxHandler; let request; let stateTransitionFixture; let unserializeStateTransitionMock; + let loggerMock; + let createContextLoggerMock; beforeEach(function beforeEach() { stateTransitionFixture = getIdentityCreateTransitionFixture(); @@ -26,8 +29,13 @@ describe('checkTxHandlerFactory', () => { unserializeStateTransitionMock = this.sinon.stub() .resolves(stateTransitionFixture); + loggerMock = new LoggerMock(this.sinon); + createContextLoggerMock = this.sinon.stub(); + checkTxHandler = checkTxHandlerFactory( unserializeStateTransitionMock, + createContextLoggerMock, + loggerMock, ); }); @@ -38,5 +46,9 @@ describe('checkTxHandlerFactory', () => { expect(response.code).to.equal(0); expect(unserializeStateTransitionMock).to.be.calledOnceWith(request.tx); + + expect(createContextLoggerMock).to.be.calledOnceWithExactly(loggerMock, { + abciMethod: 'checkTx', + }); }); }); diff --git a/packages/js-drive/test/unit/abci/handlers/finalizeBlockHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/finalizeBlockHandlerFactory.spec.js index 919fe252f06..062329f12bc 100644 --- a/packages/js-drive/test/unit/abci/handlers/finalizeBlockHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/finalizeBlockHandlerFactory.spec.js @@ -32,6 +32,7 @@ describe('finalizeBlockHandlerFactory', () => { let round; let block; let processProposalMock; + let createContextLoggerMock; beforeEach(function beforeEach() { round = 0; @@ -97,6 +98,7 @@ describe('finalizeBlockHandlerFactory', () => { ); processProposalMock = this.sinon.stub(); + createContextLoggerMock = this.sinon.stub().returns(loggerMock); finalizeBlockHandler = finalizeBlockHandlerFactory( groveDBStoreMock, @@ -107,6 +109,7 @@ describe('finalizeBlockHandlerFactory', () => { latestBlockExecutionContextMock, proposalBlockExecutionContextMock, processProposalMock, + createContextLoggerMock, ); }); @@ -130,6 +133,13 @@ describe('finalizeBlockHandlerFactory', () => { expect(latestBlockExecutionContextMock.populate).to.be.calledOnce(); expect(processProposalMock).to.be.not.called(); + expect(createContextLoggerMock).to.be.calledOnceWithExactly( + loggerMock, { + height: '42', + round, + abciMethod: 'finalizeBlock', + }, + ); }); it('should send withdrawal transaction if vote extensions are present', async () => { @@ -160,6 +170,13 @@ describe('finalizeBlockHandlerFactory', () => { expect(coreRpcClientMock.sendRawTransaction).to.have.been.calledTwice(); expect(processProposalMock).to.be.not.called(); + expect(createContextLoggerMock).to.be.calledOnceWithExactly( + loggerMock, { + height: '42', + round, + abciMethod: 'finalizeBlock', + }, + ); }); it('should call processProposal if round is not equal to execution context', async () => { @@ -181,5 +198,12 @@ describe('finalizeBlockHandlerFactory', () => { }); expect(processProposalMock).to.be.calledOnceWithExactly(processProposalRequest, loggerMock); + expect(createContextLoggerMock).to.be.calledOnceWithExactly( + loggerMock, { + height: '42', + round, + abciMethod: 'finalizeBlock', + }, + ); }); }); diff --git a/packages/js-drive/test/unit/abci/handlers/infoHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/infoHandlerFactory.spec.js index 0701535ea36..c97411142b7 100644 --- a/packages/js-drive/test/unit/abci/handlers/infoHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/infoHandlerFactory.spec.js @@ -28,6 +28,7 @@ describe('infoHandlerFactory', () => { let blockExecutionContextMock; let blockExecutionContextRepositoryMock; let groveDBStoreMock; + let createContextLoggerMock; beforeEach(function beforeEach() { lastBlockHeight = Long.fromInt(0); @@ -51,6 +52,8 @@ describe('infoHandlerFactory', () => { groveDBStoreMock.getRootHash.resolves(lastBlockAppHash); + createContextLoggerMock = this.sinon.stub().returns(loggerMock); + infoHandler = infoHandlerFactory( blockExecutionContextMock, blockExecutionContextRepositoryMock, @@ -58,11 +61,12 @@ describe('infoHandlerFactory', () => { updateSimplifiedMasternodeListMock, loggerMock, groveDBStoreMock, + createContextLoggerMock, ); }); it('should return respond with genesis heights and app hash on the first run', async () => { - blockExecutionContextRepositoryMock.fetch.resolves(null); + blockExecutionContextRepositoryMock.fetch.resolves(blockExecutionContextMock); blockExecutionContextMock.getHeight.returns(null); blockExecutionContextMock.isEmpty.returns(true); @@ -83,6 +87,11 @@ describe('infoHandlerFactory', () => { expect(blockExecutionContextMock.getCoreChainLockedHeight).to.not.be.called(); expect(updateSimplifiedMasternodeListMock).to.not.be.called(); expect(groveDBStoreMock.getRootHash).to.be.calledOnce(); + expect(createContextLoggerMock).to.be.calledOnceWithExactly( + loggerMock, { + abciMethod: 'info', + }, + ); }); it('should populate context and update SML on subsequent runs', async () => { @@ -105,5 +114,16 @@ describe('infoHandlerFactory', () => { logger: loggerMock, }, ); + expect(createContextLoggerMock).to.be.calledTwice(); + expect(createContextLoggerMock.getCall(0)).to.be.calledWithExactly( + loggerMock, { + abciMethod: 'info', + }, + ); + expect(createContextLoggerMock.getCall(1)).to.be.calledWithExactly( + loggerMock, { + height: lastBlockHeight.toString(), + }, + ); }); }); diff --git a/packages/js-drive/test/unit/abci/handlers/initChainHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/initChainHandlerFactory.spec.js index f637a2cca3f..17616434537 100644 --- a/packages/js-drive/test/unit/abci/handlers/initChainHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/initChainHandlerFactory.spec.js @@ -34,6 +34,7 @@ describe('initChainHandlerFactory', () => { let rsAbciMock; let createCoreChainLockUpdateMock; let coreChainLockUpdate; + let createContextLoggerMock; beforeEach(function beforeEach() { initialCoreChainLockedHeight = 1; @@ -78,6 +79,7 @@ describe('initChainHandlerFactory', () => { }); createCoreChainLockUpdateMock = this.sinon.stub().resolves(coreChainLockUpdate); + createContextLoggerMock = this.sinon.stub().returns(loggerMock); initChainHandler = initChainHandlerFactory( updateSimplifiedMasternodeListMock, @@ -90,6 +92,7 @@ describe('initChainHandlerFactory', () => { groveDBStoreMock, rsAbciMock, createCoreChainLockUpdateMock, + createContextLoggerMock, ); }); @@ -147,5 +150,9 @@ describe('initChainHandlerFactory', () => { expect(createCoreChainLockUpdateMock) .to.be.calledOnceWithExactly(initialCoreChainLockedHeight, 0, loggerMock); + expect(createContextLoggerMock).to.be.calledOnceWithExactly(loggerMock, { + height: request.initialHeight.toString(), + abciMethod: 'initChain', + }); }); }); diff --git a/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js index c52e03cb2b0..2bb3e3f5640 100644 --- a/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/prepareProposalHandlerFactory.spec.js @@ -33,6 +33,7 @@ describe('prepareProposalHandlerFactory', () => { let proposalBlockExecutionContextMock; let round; let executionTimerMock; + let createContextLoggerMock; beforeEach(function beforeEach() { round = 1; @@ -93,6 +94,7 @@ describe('prepareProposalHandlerFactory', () => { executionTimerMock = { getTimer: this.sinon.stub().returns(0.1), }; + createContextLoggerMock = this.sinon.stub().returns(loggerMock); prepareProposalHandler = prepareProposalHandlerFactory( deliverTxMock, @@ -102,6 +104,7 @@ describe('prepareProposalHandlerFactory', () => { endBlockMock, updateCoreChainLockMock, executionTimerMock, + createContextLoggerMock, ); const maxTxBytes = 42; @@ -198,6 +201,11 @@ describe('prepareProposalHandlerFactory', () => { consensusParamUpdates, validatorSetUpdate, }); + expect(createContextLoggerMock).to.be.calledOnceWithExactly(loggerMock, { + height: '42', + round, + abciMethod: 'prepareProposal', + }); }); it('should cut txs that are not fit into the size limit', async () => { diff --git a/packages/js-drive/test/unit/abci/handlers/processProposalHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/processProposalHandlerFactory.spec.js index 647901fc7e9..ccb4b0deb5f 100644 --- a/packages/js-drive/test/unit/abci/handlers/processProposalHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/processProposalHandlerFactory.spec.js @@ -28,6 +28,7 @@ describe('processProposalHandlerFactory', () => { let proposalBlockExecutionContextMock; let consensusParamUpdates; let validatorSetUpdate; + let createContextLoggerMock; beforeEach(function beforeEach() { round = 0; @@ -57,12 +58,14 @@ describe('processProposalHandlerFactory', () => { verifyChainLockMock = this.sinon.stub().resolves(true); processProposalMock = this.sinon.stub().resolves(new ResponseProcessProposal({ status: 1 })); + createContextLoggerMock = this.sinon.stub().returns(loggerMock); processProposalHandler = processProposalHandlerFactory( loggerMock, verifyChainLockMock, processProposalMock, proposalBlockExecutionContextMock, + createContextLoggerMock, ); const txs = new Array(3).fill(Buffer.alloc(5, 0)); @@ -148,5 +151,10 @@ describe('processProposalHandlerFactory', () => { expect(processProposalMock).to.not.be.called(); expect(verifyChainLockMock).to.not.be.called(); + expect(createContextLoggerMock).to.be.calledOnceWithExactly(loggerMock, { + height: '42', + round, + abciMethod: 'processProposal', + }); }); }); diff --git a/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js index 91e4e6793d9..27313697037 100644 --- a/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/proposal/deliverTxFactory.spec.js @@ -1,21 +1,22 @@ +const crypto = require('crypto'); + const DashPlatformProtocol = require('@dashevo/dpp'); const ValidationResult = require('@dashevo/dpp/lib/validation/ValidationResult'); const getIdentityFixture = require('@dashevo/dpp/lib/test/fixtures/getIdentityFixture'); - const createDPPMock = require('@dashevo/dpp/lib/test/mocks/createDPPMock'); const createStateRepositoryMock = require('@dashevo/dpp/lib/test/mocks/createStateRepositoryMock'); const getDataContractFixture = require('@dashevo/dpp/lib/test/fixtures/getDataContractFixture'); const getDocumentFixture = require('@dashevo/dpp/lib/test/fixtures/getDocumentsFixture'); const GrpcErrorCodes = require('@dashevo/grpc-common/lib/server/error/GrpcErrorCodes'); + const StateTransitionExecutionContext = require('@dashevo/dpp/lib/stateTransition/StateTransitionExecutionContext'); const SomeConsensusError = require('@dashevo/dpp/lib/test/mocks/SomeConsensusError'); - const BlockExecutionContextMock = require('../../../../../lib/test/mock/BlockExecutionContextMock'); -const deliverTxFactory = require('../../../../../lib/abci/handlers/proposal/deliverTxFactory'); +const deliverTxFactory = require('../../../../../lib/abci/handlers/proposal/deliverTxFactory'); const LoggerMock = require('../../../../../lib/test/mock/LoggerMock'); const DPPValidationAbciError = require('../../../../../lib/abci/errors/DPPValidationAbciError'); const InvalidArgumentAbciError = require('../../../../../lib/abci/errors/InvalidArgumentAbciError'); @@ -39,6 +40,7 @@ describe('deliverTxFactory', () => { let round; let proposalBlockExecutionContextMock; let stateTransitionExecutionContextMock; + let createContextLoggerMock; beforeEach(async function beforeEach() { round = 42; @@ -105,11 +107,14 @@ describe('deliverTxFactory', () => { isStarted: this.sinon.stub(), }; + createContextLoggerMock = this.sinon.stub().returns(loggerMock); + deliverTx = deliverTxFactory( unserializeStateTransitionMock, dppMock, proposalBlockExecutionContextMock, executionTimerMock, + createContextLoggerMock, ); }); @@ -148,6 +153,17 @@ describe('deliverTxFactory', () => { identity.reduceBalance(stateTransitionExecutionContextMock.getLastCalculatedFeeDetails().total); expect(stateRepositoryMock.updateIdentity).to.be.calledOnceWith(identity); + + const stHash = crypto + .createHash('sha256') + .update(documentTx) + .digest() + .toString('hex') + .toUpperCase(); + + expect(createContextLoggerMock).to.be.calledOnceWithExactly(loggerMock, { + txId: stHash, + }); }); it('should apply a DataContractCreateTransition, add it to block execution state and return ResponseDeliverTx', async () => { diff --git a/packages/js-drive/test/unit/abci/handlers/queryHandlerFactory.spec.js b/packages/js-drive/test/unit/abci/handlers/queryHandlerFactory.spec.js index d7027723173..bac844658ee 100644 --- a/packages/js-drive/test/unit/abci/handlers/queryHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/abci/handlers/queryHandlerFactory.spec.js @@ -12,6 +12,7 @@ describe('queryHandlerFactory', () => { let request; let routeMock; let loggerMock; + let createContextLoggerMock; beforeEach(function beforeEach() { request = { @@ -32,10 +33,13 @@ describe('queryHandlerFactory', () => { find: this.sinon.stub().returns(routeMock), }; + createContextLoggerMock = this.sinon.stub(); + queryHandler = queryHandlerFactory( queryHandlerRouterMock, sanitizeUrlMock, loggerMock, + createContextLoggerMock, ); }); @@ -118,5 +122,8 @@ describe('queryHandlerFactory', () => { expect(queryHandlerRouterMock.find).to.be.calledOnceWith('GET', sanitizedUrl); expect(routeMock.handler).to.be.calledOnceWith(routeMock.params, encodedData, request); expect(result).to.equal(data); + expect(createContextLoggerMock).to.be.calledOnceWithExactly(loggerMock, { + abciMethod: 'query', + }); }); }); From 5f9161125056444c4f1fcfb363d1e7d563e508e5 Mon Sep 17 00:00:00 2001 From: Konstantin Shuplenkov Date: Tue, 10 Jan 2023 16:31:48 +0300 Subject: [PATCH 51/54] WIP --- .../test/integration/abci/handlers/queryHandlerFactory.spec.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/js-drive/test/integration/abci/handlers/queryHandlerFactory.spec.js b/packages/js-drive/test/integration/abci/handlers/queryHandlerFactory.spec.js index a5f55f9405d..a06c3c3e9a4 100644 --- a/packages/js-drive/test/integration/abci/handlers/queryHandlerFactory.spec.js +++ b/packages/js-drive/test/integration/abci/handlers/queryHandlerFactory.spec.js @@ -49,7 +49,8 @@ describe('queryHandlerFactory', function main() { container.register('dataContractQueryHandler', asValue(dataContractQueryHandlerMock)); container.register('documentQueryHandler', asValue(documentQueryHandlerMock)); - queryHandler = container.resolve('queryHandler'); + const enrichErrorWithContextError = container.resolve('enrichErrorWithContextError'); + queryHandler = enrichErrorWithContextError(container.resolve('queryHandler')); }); afterEach(async () => { From 2ca74c47cfc6f8cc2c2f26dfc8f083478647650c Mon Sep 17 00:00:00 2001 From: Konstantin Shuplenkov Date: Wed, 11 Jan 2023 17:22:36 +0300 Subject: [PATCH 52/54] WIP --- .../js-drive/lib/abci/errors/wrapInErrorHandlerFactory.js | 5 ++++- packages/js-drive/lib/errorHandlerFactory.js | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/js-drive/lib/abci/errors/wrapInErrorHandlerFactory.js b/packages/js-drive/lib/abci/errors/wrapInErrorHandlerFactory.js index 909788b3a61..8445605f767 100644 --- a/packages/js-drive/lib/abci/errors/wrapInErrorHandlerFactory.js +++ b/packages/js-drive/lib/abci/errors/wrapInErrorHandlerFactory.js @@ -37,7 +37,10 @@ function wrapInErrorHandlerFactory(logger, isProductionEnvironment) { if (abciError instanceof InternalAbciError) { const originalError = abciError.getError(); - (originalError.contextLogger || logger).error( + const preferredLogger = originalError.contextLogger || logger; + delete originalError.contextLogger; + + preferredLogger.error( { err: originalError }, originalError.message, ); diff --git a/packages/js-drive/lib/errorHandlerFactory.js b/packages/js-drive/lib/errorHandlerFactory.js index 7aafd8c15dd..6025c252f01 100644 --- a/packages/js-drive/lib/errorHandlerFactory.js +++ b/packages/js-drive/lib/errorHandlerFactory.js @@ -36,8 +36,11 @@ function errorHandlerFactory(logger, container, closeAbciServer) { // eslint-disable-next-line no-console console.log(printErrorFace()); + const preferredLogger = error.contextLogger || logger; + errors.forEach((e) => { - (error.contextLogger || logger).fatal({ err: e }, e.message); + delete e.contextLogger; + preferredLogger.fatal({ err: e }, e.message); }); } finally { await container.dispose(); From df8eef67d95773f6c09f110edd9afe1a0b7af7bf Mon Sep 17 00:00:00 2001 From: Konstantin Shuplenkov Date: Wed, 11 Jan 2023 17:48:20 +0300 Subject: [PATCH 53/54] WIP --- .../js-drive/test/unit/util/errorHandlerFactory.spec.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/js-drive/test/unit/util/errorHandlerFactory.spec.js b/packages/js-drive/test/unit/util/errorHandlerFactory.spec.js index c957ee76a55..466f87948a4 100644 --- a/packages/js-drive/test/unit/util/errorHandlerFactory.spec.js +++ b/packages/js-drive/test/unit/util/errorHandlerFactory.spec.js @@ -48,12 +48,14 @@ describe('errorHandlerFactory', () => { it('should use consensus logger if it\'s present', async function it() { const error = new Error('message'); - error.contextLogger = new LoggerMock(this.sinon); + const contextLogger = new LoggerMock(this.sinon); + + error.contextLogger = contextLogger; await errorHandler(error); expect(loggerMock.fatal).to.not.be.called(); - expect(error.contextLogger.fatal).to.be.calledOnceWithExactly({ err: error }, error.message); + expect(contextLogger.fatal).to.be.calledOnceWithExactly({ err: error }, error.message); expect(containerMock.dispose).to.be.calledOnceWithExactly(); From 9b75046daf78578313151b80ba938e453f260bdf Mon Sep 17 00:00:00 2001 From: Konstantin Shuplenkov Date: Wed, 11 Jan 2023 18:08:53 +0300 Subject: [PATCH 54/54] WIP --- packages/js-drive/lib/errorHandlerFactory.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/js-drive/lib/errorHandlerFactory.js b/packages/js-drive/lib/errorHandlerFactory.js index 6025c252f01..7172646507a 100644 --- a/packages/js-drive/lib/errorHandlerFactory.js +++ b/packages/js-drive/lib/errorHandlerFactory.js @@ -36,10 +36,10 @@ function errorHandlerFactory(logger, container, closeAbciServer) { // eslint-disable-next-line no-console console.log(printErrorFace()); - const preferredLogger = error.contextLogger || logger; - errors.forEach((e) => { + const preferredLogger = e.contextLogger || logger; delete e.contextLogger; + preferredLogger.fatal({ err: e }, e.message); }); } finally {