Skip to content

perf: Optimize synchronization logic for multiple wallets in the light client. #3160

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yanguoyu merged 2 commits into from
May 28, 2024
Merged

perf: Optimize synchronization logic for multiple wallets in the light client. #3160

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
921c838
perf: Optimize synchronization logic for multiple wallets in a light …
yanguoyu May 20, 2024
a9cb8e2
Merge branch 'develop' into opt-light-sync
Keith-CY May 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 33 additions & 20 deletions packages/neuron-wallet/src/block-sync-renderer/sync/light-synchronizer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import NetworksService from '../../services/networks'
import { getConnection } from '../../database/chain/connection'

const unpackGroup = molecule.vector(blockchain.OutPoint)
const THRESHOLD_BLOCK_NUMBER_IN_DIFF_WALLET = 100_000

export default class LightSynchronizer extends Synchronizer {
private lightRpc: LightRPC
Expand Down Expand Up @@ -171,20 +172,32 @@ export default class LightSynchronizer extends Synchronizer {
await getConnection('light').manager.save(updatedSyncProgress, { chunk: 100 })
}

private static async getWalletsSyncedMinBlockNumber() {
const walletMinBlockNumber = await SyncProgressService.getWalletMinLocalSavedBlockNumber()
const wallets = await WalletService.getInstance().getAll()
return wallets.reduce<Record<string, number>>(
(pre, cur) => ({
...pre,
[cur.id]: Math.max(parseInt(cur.startBlockNumber ?? '0x0'), walletMinBlockNumber?.[cur.id] ?? 0),
}),
{}
)
}

private async initSyncProgress(appendScripts: AppendScript[] = []) {
if (!this.addressMetas.length && !appendScripts.length) {
return
}
const existSyncArgses = await SyncProgressService.getExistingSyncArgses()
const syncScripts = await this.lightRpc.getScripts()
const retainedSyncScripts = syncScripts.filter(v => existSyncArgses.has(v.script.args))
const existSyncscripts: Record<string, LightScriptFilter> = {}
const existSyncScripts: Record<string, LightScriptFilter> = {}
retainedSyncScripts.forEach(v => {
existSyncscripts[scriptToHash(v.script)] = v
existSyncScripts[scriptToHash(v.script)] = v
})
const walletMinBlockNumber = await LightSynchronizer.getWalletsSyncedMinBlockNumber()
const currentWalletId = WalletService.getInstance().getCurrent()?.id
const allScripts = this.addressMetas
.filter(v => (currentWalletId ? v.walletId === currentWalletId : true))
.map(addressMeta => {
const lockScripts = [
addressMeta.generateDefaultLockScript(),
Expand All @@ -198,32 +211,32 @@ export default class LightSynchronizer extends Synchronizer {
}))
})
.flat()
const walletMinBlockNumber = await SyncProgressService.getWalletMinLocalSavedBlockNumber()
const wallets = await WalletService.getInstance().getAll()
const walletStartBlockMap = wallets.reduce<Record<string, string | undefined>>(
(pre, cur) => ({ ...pre, [cur.id]: cur.startBlockNumber }),
{}
)
.filter(v => {
return (
!currentWalletId ||
v.walletId === currentWalletId ||
walletMinBlockNumber[v.walletId] >
walletMinBlockNumber[currentWalletId] - THRESHOLD_BLOCK_NUMBER_IN_DIFF_WALLET
)
})
const otherTypeSyncBlockNumber = await SyncProgressService.getOtherTypeSyncBlockNumber()
const addScripts = [
...allScripts
.filter(
v =>
!existSyncscripts[scriptToHash(v.script)] ||
+existSyncscripts[scriptToHash(v.script)].blockNumber < +(walletStartBlockMap[v.walletId] ?? 0)
)
.map(v => {
const blockNumber = Math.max(
parseInt(walletStartBlockMap[v.walletId] ?? '0x0'),
walletMinBlockNumber?.[v.walletId] ?? 0
.filter(v => {
const scriptHash = scriptToHash(v.script)
return (
!existSyncScripts[scriptHash] ||
+existSyncScripts[scriptHash].blockNumber < walletMinBlockNumber[v.walletId]
)
})
.map(v => {
return {
...v,
blockNumber: `0x${blockNumber.toString(16)}`,
blockNumber: `0x${walletMinBlockNumber[v.walletId].toString(16)}`,
}
}),
...appendScripts
.filter(v => !existSyncscripts[scriptToHash(v.script)])
.filter(v => !existSyncScripts[scriptToHash(v.script)])
.map(v => ({
...v,
blockNumber: `0x${(otherTypeSyncBlockNumber[scriptToHash(v.script)] ?? 0).toString(16)}`,
Expand Down
165 changes: 157 additions & 8 deletions packages/neuron-wallet/tests/block-sync-renderer/light-synchronizer.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ const schedulerWaitMock = jest.fn()
const getMultisigConfigForLightMock = jest.fn()
const walletGetCurrentMock = jest.fn()
const walletGetAllMock = jest.fn()
const updateBlockStartNumberMock = jest.fn()

function mockReset() {
getSyncStatusMock.mockReset()
Expand All @@ -46,6 +47,8 @@ function mockReset() {
removeByHashesAndAddressType.mockReset()
walletGetCurrentMock.mockReset()
walletGetAllMock.mockReset()

updateBlockStartNumberMock.mockReset()
}

jest.mock('../../src/services/sync-progress', () => {
Expand Down Expand Up @@ -138,6 +141,7 @@ describe('test light synchronizer', () => {
})
it('when syncing script in the local DB', async () => {
getScriptsMock.mockResolvedValue([{ script, blockNumber: '0xaa' }])
walletGetAllMock.mockReturnValue([{ id: 'walletId' }])
getExistingSyncArgsesMock.mockResolvedValue(new Set([script.args]))
const addressMeta = AddressMeta.fromObject({
walletId: 'walletId',
Expand Down Expand Up @@ -187,6 +191,7 @@ describe('test light synchronizer', () => {
})
it('when syncing script not in the local DB', async () => {
getScriptsMock.mockResolvedValue([{ script, blockNumber: '0xaa' }])
walletGetAllMock.mockReturnValue([{ id: 'walletId' }])
getExistingSyncArgsesMock.mockResolvedValue(new Set())
const addressMeta = AddressMeta.fromObject({
walletId: 'walletId',
Expand Down Expand Up @@ -256,6 +261,7 @@ describe('test light synchronizer', () => {
addressType: 0,
blake160: script.args,
})
walletGetAllMock.mockReturnValue([{ id: 'walletId' }])
getWalletMinLocalSavedBlockNumberMock.mockResolvedValue({ walletId: 170 })
const connect = new LightSynchronizer([addressMeta], '')
//@ts-ignore
Expand Down Expand Up @@ -367,6 +373,152 @@ describe('test light synchronizer', () => {
)
expect(setScriptsMock).toHaveBeenLastCalledWith([], 'delete')
})
it('when other wallet min synced block number near the current wallet', async () => {
walletGetCurrentMock.mockReturnValue({ id: 'walletId1' })
walletGetAllMock.mockReturnValue([
{ id: 'walletId1', startBlockNumber: '0xaa' },
{ id: 'walletId2', startBlockNumber: '0xab' },
])
const addressMeta1 = AddressMeta.fromObject({
walletId: 'walletId1',
address,
path: '',
addressIndex: 10,
addressType: 0,
blake160: script.args,
})
const script2: Script = {
args: '0x403f0d4e833b2a8d372772a63facaa310dfeef93',
codeHash: '0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8',
hashType: 'type',
}
const getScriptsResult = [{ script: script2, blockNumber: '0xaa', scriptType: 'lock' }]
getScriptsMock.mockResolvedValue(getScriptsResult)
getExistingSyncArgsesMock.mockResolvedValue(new Set([script2.args]))
const addressMeta2 = AddressMeta.fromObject({
walletId: 'walletId2',
address,
path: '',
addressIndex: 10,
addressType: 0,
blake160: script2.args,
})
const connect = new LightSynchronizer([addressMeta1, addressMeta2], '')
//@ts-ignore
await connect.initSyncProgress()
expect(setScriptsMock).toHaveBeenNthCalledWith(
1,
[
...[
addressMeta1.generateDefaultLockScript().toSDK(),
addressMeta1.generateACPLockScript().toSDK(),
addressMeta1.generateLegacyACPLockScript().toSDK(),
].map(script => ({
script,
scriptType: 'lock',
walletId: addressMeta1.walletId,
blockNumber: '0xaa',
})),
...[
addressMeta2.generateDefaultLockScript().toSDK(),
addressMeta2.generateACPLockScript().toSDK(),
addressMeta2.generateLegacyACPLockScript().toSDK(),
].map(script => ({
script,
scriptType: 'lock',
walletId: addressMeta2.walletId,
blockNumber: '0xab',
})),
],
'partial'
)
expect(setScriptsMock).toHaveBeenLastCalledWith([], 'delete')
expect(initSyncProgressMock).toBeCalledWith([
...[
addressMeta1.generateDefaultLockScript().toSDK(),
addressMeta1.generateACPLockScript().toSDK(),
addressMeta1.generateLegacyACPLockScript().toSDK(),
].map(script => ({
script,
scriptType: 'lock',
walletId: addressMeta1.walletId,
blockNumber: '0xaa',
})),
...[
addressMeta2.generateDefaultLockScript().toSDK(),
addressMeta2.generateACPLockScript().toSDK(),
addressMeta2.generateLegacyACPLockScript().toSDK(),
].map(script => ({
script,
scriptType: 'lock',
walletId: addressMeta2.walletId,
blockNumber: '0xab',
})),
])
expect(updateSyncProgressFlagMock).toBeCalledWith(['walletId1', 'walletId2'])
})
it('when other wallet min synced block number bigger than the current wallet', async () => {
walletGetCurrentMock.mockReturnValue({ id: 'walletId1' })
walletGetAllMock.mockReturnValue([
{ id: 'walletId1', startBlockNumber: '0xaaaa0' },
{ id: 'walletId2', startBlockNumber: '0xab' },
])
const addressMeta1 = AddressMeta.fromObject({
walletId: 'walletId1',
address,
path: '',
addressIndex: 10,
addressType: 0,
blake160: script.args,
})
const script2: Script = {
args: '0x403f0d4e833b2a8d372772a63facaa310dfeef93',
codeHash: '0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8',
hashType: 'type',
}
const getScriptsResult = [{ script: script2, blockNumber: '0xab', scriptType: 'lock' }]
getScriptsMock.mockResolvedValue(getScriptsResult)
getExistingSyncArgsesMock.mockResolvedValue(new Set([script2.args]))
const addressMeta2 = AddressMeta.fromObject({
walletId: 'walletId2',
address,
path: '',
addressIndex: 10,
addressType: 0,
blake160: script2.args,
})
const connect = new LightSynchronizer([addressMeta1, addressMeta2], '')
//@ts-ignore
await connect.initSyncProgress()
expect(setScriptsMock).toHaveBeenNthCalledWith(
1,
[
addressMeta1.generateDefaultLockScript().toSDK(),
addressMeta1.generateACPLockScript().toSDK(),
addressMeta1.generateLegacyACPLockScript().toSDK(),
].map(script => ({
script,
scriptType: 'lock',
walletId: addressMeta1.walletId,
blockNumber: '0xaaaa0',
})),
'partial'
)
expect(setScriptsMock).toHaveBeenLastCalledWith(getScriptsResult, 'delete')
expect(initSyncProgressMock).toBeCalledWith(
[
addressMeta1.generateDefaultLockScript().toSDK(),
addressMeta1.generateACPLockScript().toSDK(),
addressMeta1.generateLegacyACPLockScript().toSDK(),
].map(script => ({
script,
scriptType: 'lock',
walletId: addressMeta1.walletId,
blockNumber: '0xaaaa0',
}))
)
expect(updateSyncProgressFlagMock).toBeCalledWith(['walletId1', 'walletId2'])
})
})

describe('test initSync', () => {
Expand Down Expand Up @@ -428,16 +580,10 @@ describe('test light synchronizer', () => {
})

describe('#notifyCurrentBlockNumberProcessed', () => {
const synchronizer = new LightSynchronizer([], '')
const updateBlockStartNumberMock = jest.fn()
beforeAll(() => {
it('last process block number finish', async () => {
const synchronizer = new LightSynchronizer([], '')
// @ts-ignore private property
synchronizer.updateBlockStartNumber = updateBlockStartNumberMock
})
beforeEach(() => {
updateBlockStartNumberMock.mockReset()
})
it('last process block number finish', async () => {
// @ts-ignore private property
synchronizer.processingBlockNumber = '0xaa'
getCurrentWalletMinSyncedBlockNumberMock.mockResolvedValueOnce(100)
Expand All @@ -447,6 +593,9 @@ describe('test light synchronizer', () => {
expect(updateBlockStartNumberMock).toBeCalledWith(100)
})
it('not last process block number finish', async () => {
const synchronizer = new LightSynchronizer([], '')
// @ts-ignore private property
synchronizer.updateBlockStartNumber = updateBlockStartNumberMock
// @ts-ignore private property
synchronizer.processingBlockNumber = undefined
await synchronizer.notifyCurrentBlockNumberProcessed('0xaa')
Expand Down