From 014dc7dbcb9fc2776491b810971a2e77313468b5 Mon Sep 17 00:00:00 2001 From: Lantt Date: Thu, 16 Dec 2021 14:50:17 +0800 Subject: [PATCH 01/59] feat: add producer --- packages/web3-providers/src/debank/format.ts | 40 +++++ packages/web3-providers/src/debank/index.ts | 44 +++++ packages/web3-providers/src/debank/type.ts | 164 ++++++++++++++++++ .../src/proxy/producers/fungibleTokenAsset.ts | 34 ++++ .../src/proxy/producers/index.ts | 6 + .../src/proxy/producers/producer.d.ts | 11 ++ 6 files changed, 299 insertions(+) create mode 100644 packages/web3-providers/src/debank/format.ts create mode 100644 packages/web3-providers/src/debank/index.ts create mode 100644 packages/web3-providers/src/debank/type.ts create mode 100644 packages/web3-providers/src/proxy/producers/fungibleTokenAsset.ts create mode 100644 packages/web3-providers/src/proxy/producers/index.ts create mode 100644 packages/web3-providers/src/proxy/producers/producer.d.ts diff --git a/packages/web3-providers/src/debank/format.ts b/packages/web3-providers/src/debank/format.ts new file mode 100644 index 000000000000..c7abbed22552 --- /dev/null +++ b/packages/web3-providers/src/debank/format.ts @@ -0,0 +1,40 @@ +import BigNumber from 'bignumber.js' +import type { WalletTokenRecord } from './type' +import { ChainId, formatEthereumAddress, getChainIdFromName } from '@masknet/web3-shared-evm' +import { CurrencyType, TokenType, Web3Plugin } from '@masknet/plugin-infra' +import { multipliedBy, rightShift } from '@masknet/web3-shared-base' + +type Asset = Web3Plugin.Asset + +export function formatAssets(data: WalletTokenRecord[]): Asset[] { + return data + .filter((x) => x.is_verified) + .map((y): Asset => { + const chainIdFromChain = getChainIdFromName(y.chain) ?? ChainId.Mainnet + const chainIdFromId = getChainIdFromName(y.id) + const address = y.id === 'eth' ? '' : formatEthereumAddress(y.id) + + return { + id: address, + chainId: chainIdFromChain, + token: { + id: address, + address: address, + chainId: chainIdFromChain, + type: TokenType.Fungible, + decimals: y.decimals, + name: y.name, + symbol: y.symbol, + logoURI: y.logo_url, + }, + balance: rightShift(y.amount, y.decimals).toFixed(), + price: { + [CurrencyType.USD]: new BigNumber(y.price ?? 0).toFixed(), + }, + value: { + [CurrencyType.USD]: multipliedBy(y.price ?? 0, y.amount).toFixed(), + }, + logoURI: y.logo_url, + } + }) +} diff --git a/packages/web3-providers/src/debank/index.ts b/packages/web3-providers/src/debank/index.ts new file mode 100644 index 000000000000..c9b9a4ab141d --- /dev/null +++ b/packages/web3-providers/src/debank/index.ts @@ -0,0 +1,44 @@ +import type { HistoryResponse, GasPriceDictResponse, WalletTokenRecord } from './type' +import urlcat from 'urlcat' +import { ChainId } from '@masknet/web3-shared-evm' + +const DEBANK_API = 'https://api.debank.com' +const DEBANK_OPEN_API = 'https://openapi.debank.com' + +export async function getTransactionList(address: string, chain: string) { + const response = await fetch(`${DEBANK_API}/history/list?user_addr=${address.toLowerCase()}&chain=${chain}`) + return (await response.json()) as HistoryResponse +} + +export async function getAssetsList(address: string) { + const response = await fetch( + `${DEBANK_OPEN_API}/v1/user/token_list?is_all=true&has_balance=true&id=${address.toLowerCase()}`, + ) + try { + return ((await response.json()) ?? []) as WalletTokenRecord[] + } catch { + return [] + } +} + +const chainIdMap: Record = { + [ChainId.Mainnet]: 'eth', + [ChainId.BSC]: 'bsc', + [ChainId.xDai]: 'xdai', + [ChainId.Matic]: 'matic', + [ChainId.Arbitrum]: 'arb', +} + +const getDebankChain = (chainId: number) => { + return chainIdMap[chainId] ?? '' +} + +export async function getGasPriceDict(chainId: ChainId) { + const chain = getDebankChain(chainId) + const response = await fetch(urlcat(DEBANK_API, '/chain/gas_price_dict_v2', { chain })) + const result = await response.json() + if (result.error_code === 0) { + return result as GasPriceDictResponse + } + return null +} diff --git a/packages/web3-providers/src/debank/type.ts b/packages/web3-providers/src/debank/type.ts new file mode 100644 index 000000000000..9bede1241adf --- /dev/null +++ b/packages/web3-providers/src/debank/type.ts @@ -0,0 +1,164 @@ +export enum DebankTransactionDirection { + SEND = 'send', + RECEIVE = 'receive', +} + +export interface DictItem { + id: string + cn: string + en: string +} + +export interface ProjectItem { + id: string + log_url: string + name: Omit +} + +export interface TokenItem { + decimals: number + display_symbol?: string + id: string + is_core: boolean + is_swap_common?: boolean + is_swap_hot?: boolean + is_verified: boolean + logo_url: string + name: string + optimized_symbol: string + price: number + symbol: string + time_at: number +} + +export interface HistoryItem { + cate_id: keyof HistoryResponse['data']['cate_dict'] + // TODO: + // implement debt_liquidated + debt_liquidated: null + id: string + other_addr: string + project_id?: string + receives: AmountTokenPair[] + sends: AmountTokenPair[] + spot_trade?: SpotTrade + time_at: number + token_approve?: TokenApprove + tx?: Record +} + +export interface AmountTokenPair { + amount: number + to_addr: string + token_id: string +} + +export interface SpotTrade { + dex_id: string + pay_token_amount: number + pay_token_id: string + receive_token_amount: number + receive_token_id: string +} + +export interface TokenApprove { + spender: string + token_id: string + value: number +} + +export interface Record { + eth_gas_fee: number + from_addr: string + name: string + // Note: this is JSON string + params: string + // Note: 0 - failed, 1 - succeed + status: 0 | 1 + to_addr: string + usd_gas_fee: number + value: number +} + +export interface HistoryRecord { + cate_dict: { + approve: DictItem + receive: DictItem + send: DictItem + spot_trade: DictItem + } + history_list: HistoryItem[] + project_dict: { + [key in string]: ProjectItem + } + token_dict: { + [key in string | 'eth']: TokenItem + } +} + +export interface BalanceRecord { + balance: number + chain: 'eth' | 'bsc' | string + decimals: number + display_symbol: null + id: 'eth' | string + is_core: boolean + is_swap_common: boolean + is_swap_hot: null + is_verified: boolean + logo_url: string + name: string + optimized_symbol: string + price: number + symbol: string + time_at: null +} + +export interface WalletTokenRecord { + id: 'eth' | string + amount: string + is_wallet: boolean + protocol_id: string + chain: 'eth' | 'bsc' | string + decimals: number + display_symbol: null + is_core: boolean + is_verified: boolean + logo_url: string + name: string + optimized_symbol: string + price: number + symbol: string + time_at: null +} + +export interface HistoryResponse { + data: HistoryRecord + error_code: number + _cache_seconds: number + _seconds: number +} + +export interface BalanceListResponse { + data?: BalanceRecord[] + error_code: number + _cache_seconds: number + _seconds: number +} + +export interface GasPriceRecord { + estimated_seconds: number + front_tx_count: number + price: number +} + +export interface GasPriceDictResponse { + data: { + fast: GasPriceRecord + normal: GasPriceRecord + slow: GasPriceRecord + update_at: number + } + error_code: number + _seconds: number +} diff --git a/packages/web3-providers/src/proxy/producers/fungibleTokenAsset.ts b/packages/web3-providers/src/proxy/producers/fungibleTokenAsset.ts new file mode 100644 index 000000000000..385a001da07c --- /dev/null +++ b/packages/web3-providers/src/proxy/producers/fungibleTokenAsset.ts @@ -0,0 +1,34 @@ +import { isValidAddress } from '@masknet/web3-shared-evm' +import { getAssetsList } from '../..' +import type { Web3Plugin } from '@masknet/plugin-infra' +import { formatAssets } from '../../debank/format' + +export interface FungibleTokenAssetArgs extends ProducerArgBase { + address: string +} + +const fungibleTokenAsset = async ( + push: ProducerPushFunction>, + getKeys: GetKeyFunction, + args: FungibleTokenAssetArgs, +): Promise => { + const { address } = args + if (!isValidAddress(address)) { + // TODO: i18n?? + throw new Error('Invalid Address') + } + try { + const data = await getAssetsList(address) + await push(formatAssets(data)) + } catch { + // await zerion + throw new Error('Fetch failed from debank') + } +} + +const producer: Producer, FungibleTokenAssetArgs> = { + producerHandler: fungibleTokenAsset, + unionBy: (item) => `${item.id}_${item.chainId}`, +} + +export default producer diff --git a/packages/web3-providers/src/proxy/producers/index.ts b/packages/web3-providers/src/proxy/producers/index.ts new file mode 100644 index 000000000000..c8fd099f4ea3 --- /dev/null +++ b/packages/web3-providers/src/proxy/producers/index.ts @@ -0,0 +1,6 @@ +/// +import fungibleTokenProducer from './fungibleTokenAsset' + +const producers = [fungibleTokenProducer] + +export default producers diff --git a/packages/web3-providers/src/proxy/producers/producer.d.ts b/packages/web3-providers/src/proxy/producers/producer.d.ts new file mode 100644 index 000000000000..8fc127b1a8bb --- /dev/null +++ b/packages/web3-providers/src/proxy/producers/producer.d.ts @@ -0,0 +1,11 @@ +type ProducerPushFunction = (item: T[]) => Promise +type GetKeyFunction = (provider: string) => Promise + +interface ProducerArgBase { + size: number +} + +interface Producer { + producerHandler(push: ProducerPushFunction, getKey: GetKeyFunction, args: R): Promise + unionBy(item: T): unknown +} From 7996dca64ea225fa71d864ccccda9d246a90e155 Mon Sep 17 00:00:00 2001 From: Lantt Date: Thu, 16 Dec 2021 18:30:57 +0800 Subject: [PATCH 02/59] feat: add rollup --- .../components/Restore/RestoreFromLocal.tsx | 1 + .../background-script/WelcomeService.ts | 1 + packages/provider-proxy/package.json | 22 +++++++++++ packages/provider-proxy/rollup.config.js | 38 +++++++++++++++++++ packages/provider-proxy/src/index.ts | 10 +++++ packages/provider-proxy/src/producer.d.ts | 11 ++++++ .../src/producers/fungibleTokenAsset.ts | 28 ++++++++++++++ packages/provider-proxy/tsconfig.json | 10 +++++ packages/web3-providers/package.json | 4 +- packages/web3-providers/src/debank/index.ts | 6 ++- .../src/proxy/producers/fungibleTokenAsset.ts | 34 ----------------- .../src/proxy/producers/index.ts | 6 --- .../src/proxy/producers/producer.d.ts | 11 ------ packages/web3-providers/tsconfig.json | 2 +- pnpm-workspace.yaml | 1 + 15 files changed, 129 insertions(+), 56 deletions(-) create mode 100644 packages/provider-proxy/package.json create mode 100644 packages/provider-proxy/rollup.config.js create mode 100644 packages/provider-proxy/src/index.ts create mode 100644 packages/provider-proxy/src/producer.d.ts create mode 100644 packages/provider-proxy/src/producers/fungibleTokenAsset.ts create mode 100644 packages/provider-proxy/tsconfig.json delete mode 100644 packages/web3-providers/src/proxy/producers/fungibleTokenAsset.ts delete mode 100644 packages/web3-providers/src/proxy/producers/index.ts delete mode 100644 packages/web3-providers/src/proxy/producers/producer.d.ts diff --git a/packages/dashboard/src/components/Restore/RestoreFromLocal.tsx b/packages/dashboard/src/components/Restore/RestoreFromLocal.tsx index 8ef85e403af1..2a6ea47fc8c5 100644 --- a/packages/dashboard/src/components/Restore/RestoreFromLocal.tsx +++ b/packages/dashboard/src/components/Restore/RestoreFromLocal.tsx @@ -102,6 +102,7 @@ export const RestoreFromLocal = memo(() => { }, [currentPersona, changeCurrentPersona]) const restoreDB = useCallback(async () => { + console.log(json) try { // If json has wallets, restore in popup. if (json?.wallets) { diff --git a/packages/mask/src/extension/background-script/WelcomeService.ts b/packages/mask/src/extension/background-script/WelcomeService.ts index 4ef43878ca32..b63da6ebd17f 100644 --- a/packages/mask/src/extension/background-script/WelcomeService.ts +++ b/packages/mask/src/extension/background-script/WelcomeService.ts @@ -132,6 +132,7 @@ export async function checkPermissionsAndRestore(id: string) { export async function checkPermissionAndOpenWalletRecovery(id: string) { const json = await getUnconfirmedBackup(id) + console.log(json) if (json) { const permissions = await extraPermissions(json.grantedHostPermissions) if (permissions.length) { diff --git a/packages/provider-proxy/package.json b/packages/provider-proxy/package.json new file mode 100644 index 000000000000..799b203caec9 --- /dev/null +++ b/packages/provider-proxy/package.json @@ -0,0 +1,22 @@ +{ + "name": "@masknet/provider-proxy", + "private": true, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "scripts": { + "build": "rollup --config" + }, + "dependencies": { + "@masknet/plugin-infra": "workspace:*", + "@masknet/web3-providers": "workspace:*", + "urlcat": "^2.0.4" + }, + "devDependencies": { + "@rollup/plugin-commonjs": "^21.0.1", + "@rollup/plugin-json": "^4.1.0", + "@rollup/plugin-node-resolve": "^13.0.6", + "@rollup/plugin-typescript": "^8.3.0", + "rollup": "^2.60.1", + "rollup-plugin-typescript2": "^0.31.1" + } +} diff --git a/packages/provider-proxy/rollup.config.js b/packages/provider-proxy/rollup.config.js new file mode 100644 index 000000000000..0bad8a69fe84 --- /dev/null +++ b/packages/provider-proxy/rollup.config.js @@ -0,0 +1,38 @@ +import typescript from 'rollup-plugin-typescript2' +import nodeResolve from '@rollup/plugin-node-resolve' +import commonjs from '@rollup/plugin-commonjs' +import json from '@rollup/plugin-json' + +function parseMaybe(s) { + return typeof s === 'string' ? JSON.parse(s) : {} +} + +const config = { + input: './src/index.ts', + output: { + dir: 'dist', + format: 'cjs', + }, + plugins: [ + nodeResolve({ + preferBuiltins: false, + mainFields: ['module', 'main'], + customResolveOptions: { + moduleDirectories: [process.env.MODULE_DIR || 'node_modules'], + }, + }), + typescript({ + compilerOptions: { + declaration: false, + declarationMap: false, + ...parseMaybe(process.env.TS_OPTS), + }, + }), + commonjs({ + extensions: ['.js', '.ts', '.tsx'], + }), + json(), + ], +} + +export default config diff --git a/packages/provider-proxy/src/index.ts b/packages/provider-proxy/src/index.ts new file mode 100644 index 000000000000..b595ff2e32e1 --- /dev/null +++ b/packages/provider-proxy/src/index.ts @@ -0,0 +1,10 @@ +/// +import fungibleTokenProducer from './producers/fungibleTokenAsset' + +// TODO: unit test + +const producers = { + fungibleTokenAsset: fungibleTokenProducer, +} + +export default producers diff --git a/packages/provider-proxy/src/producer.d.ts b/packages/provider-proxy/src/producer.d.ts new file mode 100644 index 000000000000..b0515a350bda --- /dev/null +++ b/packages/provider-proxy/src/producer.d.ts @@ -0,0 +1,11 @@ +type ProducerPushFunction = (item: T[]) => Promise +type ProducerKeyFunction = (provider: string) => Promise + +interface ProducerArgBase { + size: number +} + +interface RpcMethodRegistrationValue { + producer(push: ProducerPushFunction, getKey: ProducerKeyFunction, args: TArgs): Promise + keyHasher(item: T): string +} diff --git a/packages/provider-proxy/src/producers/fungibleTokenAsset.ts b/packages/provider-proxy/src/producers/fungibleTokenAsset.ts new file mode 100644 index 000000000000..41787e99d067 --- /dev/null +++ b/packages/provider-proxy/src/producers/fungibleTokenAsset.ts @@ -0,0 +1,28 @@ +import type { Web3Plugin } from '@masknet/plugin-infra' +import { getDebankAssetsList } from '@masknet/web3-providers' + +export interface FungibleTokenAssetArgs extends ProducerArgBase { + address: string +} + +const fungibleTokenAsset = async ( + push: ProducerPushFunction>, + getKeys: ProducerKeyFunction, + args: FungibleTokenAssetArgs, +): Promise => { + const { address } = args + try { + const data = await getDebankAssetsList(address) + await push(data) + } catch { + // await zerion + throw new Error('Fetch failed from debank') + } +} + +const producer: RpcMethodRegistrationValue, FungibleTokenAssetArgs> = { + producer: fungibleTokenAsset, + keyHasher: (item) => `${item.id}_${item.chainId}`, +} + +export default producer diff --git a/packages/provider-proxy/tsconfig.json b/packages/provider-proxy/tsconfig.json new file mode 100644 index 000000000000..ad74f238f62d --- /dev/null +++ b/packages/provider-proxy/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "./src/", + "outDir": "./dist/", + "tsBuildInfoFile": "./dist/.tsbuildinfo" + }, + "include": ["./src", "./src/**/*.json"], + "references": [{ "path": "../plugin-infra" }, { "path": "../web3-providers" }] +} diff --git a/packages/web3-providers/package.json b/packages/web3-providers/package.json index fab7ba579e2e..8459216f0aa6 100644 --- a/packages/web3-providers/package.json +++ b/packages/web3-providers/package.json @@ -5,9 +5,9 @@ "types": "./dist/index.d.ts", "dependencies": { "@masknet/plugin-infra": "workspace:*", - "@masknet/web3-shared-evm": "workspace:*", "@masknet/web3-shared-base": "workspace:*", - "bignumber.js": "^9.0.1", + "@masknet/web3-shared-evm": "workspace:*", + "bignumber.js": "9.0.1", "date-fns": "^2.27.0", "json-stable-stringify": "^1.0.1", "lodash-unified": "1.0.1", diff --git a/packages/web3-providers/src/debank/index.ts b/packages/web3-providers/src/debank/index.ts index c9b9a4ab141d..3844ac23a647 100644 --- a/packages/web3-providers/src/debank/index.ts +++ b/packages/web3-providers/src/debank/index.ts @@ -1,5 +1,6 @@ import type { HistoryResponse, GasPriceDictResponse, WalletTokenRecord } from './type' import urlcat from 'urlcat' +import { formatAssets } from './format' import { ChainId } from '@masknet/web3-shared-evm' const DEBANK_API = 'https://api.debank.com' @@ -10,12 +11,13 @@ export async function getTransactionList(address: string, chain: string) { return (await response.json()) as HistoryResponse } -export async function getAssetsList(address: string) { +export async function getDebankAssetsList(address: string) { const response = await fetch( `${DEBANK_OPEN_API}/v1/user/token_list?is_all=true&has_balance=true&id=${address.toLowerCase()}`, ) try { - return ((await response.json()) ?? []) as WalletTokenRecord[] + const result = ((await response.json()) ?? []) as WalletTokenRecord[] + return formatAssets(result) } catch { return [] } diff --git a/packages/web3-providers/src/proxy/producers/fungibleTokenAsset.ts b/packages/web3-providers/src/proxy/producers/fungibleTokenAsset.ts deleted file mode 100644 index 385a001da07c..000000000000 --- a/packages/web3-providers/src/proxy/producers/fungibleTokenAsset.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { isValidAddress } from '@masknet/web3-shared-evm' -import { getAssetsList } from '../..' -import type { Web3Plugin } from '@masknet/plugin-infra' -import { formatAssets } from '../../debank/format' - -export interface FungibleTokenAssetArgs extends ProducerArgBase { - address: string -} - -const fungibleTokenAsset = async ( - push: ProducerPushFunction>, - getKeys: GetKeyFunction, - args: FungibleTokenAssetArgs, -): Promise => { - const { address } = args - if (!isValidAddress(address)) { - // TODO: i18n?? - throw new Error('Invalid Address') - } - try { - const data = await getAssetsList(address) - await push(formatAssets(data)) - } catch { - // await zerion - throw new Error('Fetch failed from debank') - } -} - -const producer: Producer, FungibleTokenAssetArgs> = { - producerHandler: fungibleTokenAsset, - unionBy: (item) => `${item.id}_${item.chainId}`, -} - -export default producer diff --git a/packages/web3-providers/src/proxy/producers/index.ts b/packages/web3-providers/src/proxy/producers/index.ts deleted file mode 100644 index c8fd099f4ea3..000000000000 --- a/packages/web3-providers/src/proxy/producers/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -/// -import fungibleTokenProducer from './fungibleTokenAsset' - -const producers = [fungibleTokenProducer] - -export default producers diff --git a/packages/web3-providers/src/proxy/producers/producer.d.ts b/packages/web3-providers/src/proxy/producers/producer.d.ts deleted file mode 100644 index 8fc127b1a8bb..000000000000 --- a/packages/web3-providers/src/proxy/producers/producer.d.ts +++ /dev/null @@ -1,11 +0,0 @@ -type ProducerPushFunction = (item: T[]) => Promise -type GetKeyFunction = (provider: string) => Promise - -interface ProducerArgBase { - size: number -} - -interface Producer { - producerHandler(push: ProducerPushFunction, getKey: GetKeyFunction, args: R): Promise - unionBy(item: T): unknown -} diff --git a/packages/web3-providers/tsconfig.json b/packages/web3-providers/tsconfig.json index e2eb901529ec..5d389dbc4c67 100644 --- a/packages/web3-providers/tsconfig.json +++ b/packages/web3-providers/tsconfig.json @@ -6,5 +6,5 @@ "tsBuildInfoFile": "./dist/.tsbuildinfo" }, "include": ["./src", "./src/**/*.json"], - "references": [{ "path": "../plugin-infra" }] + "references": [{ "path": "../plugin-infra" }, { "path": "../web3-shared/evm" }, { "path": "../web3-shared/base" }] } diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c43b8674a4ef..dab9fff242bf 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -35,3 +35,4 @@ packages: - 'packages/external-plugin-previewer' - 'packages/backup-format' - 'packages/encryption' + - 'packages/provider-proxy' From 5b9088a5c9724df94fb1a0d19d3b58850c079a02 Mon Sep 17 00:00:00 2001 From: Lantt Date: Fri, 17 Dec 2021 15:15:53 +0800 Subject: [PATCH 03/59] feat: ci --- .github/workflows/deploy-proxy.yml | 53 ++++++++++++++++++++++++ cspell.json | 1 + packages/provider-proxy/package.json | 5 ++- packages/provider-proxy/rollup.config.js | 26 +++++------- packages/provider-proxy/src/index.ts | 4 +- packages/provider-proxy/tsconfig.json | 10 +++-- 6 files changed, 77 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/deploy-proxy.yml diff --git a/.github/workflows/deploy-proxy.yml b/.github/workflows/deploy-proxy.yml new file mode 100644 index 000000000000..f52fbb4a5754 --- /dev/null +++ b/.github/workflows/deploy-proxy.yml @@ -0,0 +1,53 @@ +name: Publish Porxy Package + +on: + push: + branches: [feat/api-provider-proxy*] + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - name: Get cache date + id: get-date + run: echo "::set-output name=date::$(/bin/date -u "+%Y%m%d")" + shell: bash + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - uses: pnpm/action-setup@v2 + with: + version: '6.23.1' + - uses: actions/setup-node@v2 + with: + node-version: '16' + cache: 'pnpm' + - name: Restore cache + uses: actions/cache@v2 + with: + path: packages/mask/node_modules/.cache/ + key: ${{ runner.os }}-extension-${{ hashFiles('pnpm-lock.yaml') }}-${{ steps.get-date.outputs.date }} + restore-keys: ${{ runner.os }}-extension-${{ hashFiles('pnpm-lock.yaml') }} + - run: pnpm install + - run: pnpm build + - name: Run Install + working-directory: ./packages/provider-proxy + run: pnpm install && pnpm build + publish-gpr: + needs: build + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: 16 + registry-url: https://npm.pkg.github.com/ + - name: Run Install + working-directory: ./packages/provider-proxy + run: npm publish --scope=@nikomaniac --registry=https://npm.pkg.github.com + env: + NODE_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/cspell.json b/cspell.json index 069b320a239f..770e6ca88358 100644 --- a/cspell.json +++ b/cspell.json @@ -245,6 +245,7 @@ "Görli", "resizer", "Pageable", + "Hasher", "metaverse", "hexlify", "contribs", diff --git a/packages/provider-proxy/package.json b/packages/provider-proxy/package.json index 799b203caec9..1db642a44121 100644 --- a/packages/provider-proxy/package.json +++ b/packages/provider-proxy/package.json @@ -4,11 +4,13 @@ "main": "./dist/index.js", "types": "./dist/index.d.ts", "scripts": { - "build": "rollup --config" + "build": "rollup -c -w" }, "dependencies": { "@masknet/plugin-infra": "workspace:*", "@masknet/web3-providers": "workspace:*", + "@masknet/web3-shared-evm": "workspace:*", + "@masknet/web3-shared-base": "workspace:*", "urlcat": "^2.0.4" }, "devDependencies": { @@ -17,6 +19,7 @@ "@rollup/plugin-node-resolve": "^13.0.6", "@rollup/plugin-typescript": "^8.3.0", "rollup": "^2.60.1", + "rollup-plugin-terser": "^7.0.2", "rollup-plugin-typescript2": "^0.31.1" } } diff --git a/packages/provider-proxy/rollup.config.js b/packages/provider-proxy/rollup.config.js index 0bad8a69fe84..83fd0a9b21ba 100644 --- a/packages/provider-proxy/rollup.config.js +++ b/packages/provider-proxy/rollup.config.js @@ -1,11 +1,8 @@ -import typescript from 'rollup-plugin-typescript2' import nodeResolve from '@rollup/plugin-node-resolve' import commonjs from '@rollup/plugin-commonjs' import json from '@rollup/plugin-json' - -function parseMaybe(s) { - return typeof s === 'string' ? JSON.parse(s) : {} -} +import typescript from 'rollup-plugin-typescript2' +import { terser } from 'rollup-plugin-terser' const config = { input: './src/index.ts', @@ -14,24 +11,23 @@ const config = { format: 'cjs', }, plugins: [ - nodeResolve({ - preferBuiltins: false, - mainFields: ['module', 'main'], - customResolveOptions: { - moduleDirectories: [process.env.MODULE_DIR || 'node_modules'], - }, - }), typescript({ compilerOptions: { declaration: false, declarationMap: false, - ...parseMaybe(process.env.TS_OPTS), + tsconfig: 'tsconfig.json', }, }), - commonjs({ - extensions: ['.js', '.ts', '.tsx'], + nodeResolve({ + exportConditions: ['node'], + preferBuiltins: false, + mainFields: ['module', 'main'], + rootDir: '../..', + moduleDirectories: ['../../node_modules', 'node_modules'], }), + commonjs(), json(), + terser(), ], } diff --git a/packages/provider-proxy/src/index.ts b/packages/provider-proxy/src/index.ts index b595ff2e32e1..5371f1280cc9 100644 --- a/packages/provider-proxy/src/index.ts +++ b/packages/provider-proxy/src/index.ts @@ -3,8 +3,6 @@ import fungibleTokenProducer from './producers/fungibleTokenAsset' // TODO: unit test -const producers = { +export const producers = { fungibleTokenAsset: fungibleTokenProducer, } - -export default producers diff --git a/packages/provider-proxy/tsconfig.json b/packages/provider-proxy/tsconfig.json index ad74f238f62d..8d5e01442c2f 100644 --- a/packages/provider-proxy/tsconfig.json +++ b/packages/provider-proxy/tsconfig.json @@ -2,9 +2,13 @@ "extends": "../../tsconfig.json", "compilerOptions": { "rootDir": "./src/", - "outDir": "./dist/", - "tsBuildInfoFile": "./dist/.tsbuildinfo" + "outDir": "./dist/" }, "include": ["./src", "./src/**/*.json"], - "references": [{ "path": "../plugin-infra" }, { "path": "../web3-providers" }] + "references": [ + { "path": "../plugin-infra" }, + { "path": "../web3-providers" }, + { "path": "../web3-shared/base" }, + { "path": "../web3-shared/evm" } + ] } From bd1886eaa23b03197f26be29e28816b73d7409c7 Mon Sep 17 00:00:00 2001 From: Lantt Date: Tue, 21 Dec 2021 14:29:21 +0800 Subject: [PATCH 04/59] feat: add proxy class --- package.json | 2 + packages/web3-providers/package.json | 1 + packages/web3-providers/src/proxy/index.ts | 72 +++++++++++++++++ .../web3-providers/src/proxy/proxy.test.ts | 78 +++++++++++++++++++ packages/web3-providers/tsconfig.tests.json | 10 +++ 5 files changed, 163 insertions(+) create mode 100644 packages/web3-providers/src/proxy/index.ts create mode 100644 packages/web3-providers/src/proxy/proxy.test.ts create mode 100644 packages/web3-providers/tsconfig.tests.json diff --git a/package.json b/package.json index 800ff2a47c28..2091263ca26c 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,9 @@ "gulp": "^4.0.2", "husky": "^7.0.4", "jest": "^27.4.5", + "jest-websocket-mock": "^2.2.1", "lint-staged": "^12.1.4", + "mock-socket": "^9.0.8", "only-allow": "^1.0.0", "prettier": "^2.5.1", "ts-jest": "^27.1.2", diff --git a/packages/web3-providers/package.json b/packages/web3-providers/package.json index 8459216f0aa6..95db3e0d67ff 100644 --- a/packages/web3-providers/package.json +++ b/packages/web3-providers/package.json @@ -11,6 +11,7 @@ "date-fns": "^2.27.0", "json-stable-stringify": "^1.0.1", "lodash-unified": "1.0.1", + "uuid": "^8.3.2", "urlcat": "^2.0.4" } } diff --git a/packages/web3-providers/src/proxy/index.ts b/packages/web3-providers/src/proxy/index.ts new file mode 100644 index 000000000000..fe0f7fc5cf2d --- /dev/null +++ b/packages/web3-providers/src/proxy/index.ts @@ -0,0 +1,72 @@ +const PROXY_URL = 'wss://api-v4.zerion.io' + +export interface MessageBase { + id: string +} + +export interface RequestMessage extends MessageBase { + method: string + params: unknown +} + +export interface PayloadMessage extends MessageBase { + error?: unknown + result?: unknown[] +} + +interface PollItem { + pickedAt?: Date + createdAt: Date + updatedAt?: Date + data: T[] +} + +// todo: +// 1. avoid duplicate instance +// 2. auto reconnection +// 3. handle data is end +export class ProviderProxy { + private readonly _socket: WebSocket + private readonly _poll: Map + + constructor(point: string, notifyFn: (id: string) => void) { + this._socket = new WebSocket(point) + this._poll = new Map() + this._socket.addEventListener('message', (event: MessageEvent) => { + console.debug('Message from server ', event.data) + const { id, result, error } = JSON.parse(event.data) as PayloadMessage + if (error || !id) { + // TODO: handle errors + } + if (result) { + const updatedAt = new Date() + const itemInPoll = this._poll.get(id) ?? { data: [], createdAt: updatedAt } + const dataInPool = itemInPoll?.data ?? [] + const patchData = [...dataInPool, ...(result ?? [])] + this._poll.set(id, { ...itemInPoll, updatedAt: updatedAt, data: patchData }) + notifyFn(id) + } + }) + } + + /** + * Send request to proxy websocket + * @param message + */ + send(message: RequestMessage) { + if (this._poll.has(message.id)) { + //TODO: notify to pick + return + } + this._socket.send(JSON.stringify(message)) + this._poll.set(message.id, { data: [], createdAt: new Date() }) + } + + get socket(): WebSocket { + return this._socket + } + + getResult(id: string): T[] { + return (this._poll.get(id)?.data ?? []) as T[] + } +} diff --git a/packages/web3-providers/src/proxy/proxy.test.ts b/packages/web3-providers/src/proxy/proxy.test.ts new file mode 100644 index 000000000000..50840b77733d --- /dev/null +++ b/packages/web3-providers/src/proxy/proxy.test.ts @@ -0,0 +1,78 @@ +import { describe, test, expect, beforeEach, afterEach, xtest, jest } from '@jest/globals' +import { ProviderProxy } from './index' +import * as mockWS from 'jest-websocket-mock' + +const POINT = 'ws://localhost:1235' + +describe('Proxy websocket', () => { + let server: mockWS.WS + let client: ProviderProxy + let mockNotifyCallback: any + + // TODO: remove any + const pushToClientMockData = (data: any) => server.send(data) + + beforeEach(async () => { + mockNotifyCallback = jest.fn() + server = new mockWS.WS(POINT, { jsonProtocol: true }) + client = new ProviderProxy(POINT, mockNotifyCallback) + + await server.connected + }) + + test('should cache data by request id', async () => { + const id = 'fetchAsset' + const mockData = { id, result: ['eth', 'bsc'] } + const testMethod = { method: 'fetchAsset', params: [], id } + client.send(testMethod) + pushToClientMockData(mockData) + + // @ts-ignore + await expect(server).toReceiveMessage(testMethod) + + const data = client.getResult(id) + expect(data).toEqual(['eth', 'bsc']) + expect(mockNotifyCallback?.mock.calls.length).toBe(1) + expect(mockNotifyCallback?.mock.calls[0]).toEqual([id]) + }) + + test('should merge cache data when server push two times data', async () => { + const id = 'fetchAsset2' + const testMethod = { method: 'fetchAsset2', params: [], id } + client.send(testMethod) + const mockData1 = { id, result: ['eth', 'bsc'] } + const mockData2 = { id, result: ['matic'] } + pushToClientMockData(mockData1) + pushToClientMockData(mockData2) + + // @ts-ignore + await expect(server).toReceiveMessage(testMethod) + + const data = client.getResult(id) + expect(data).toEqual(['eth', 'bsc', 'matic']) + expect(mockNotifyCallback?.mock.calls.length).toBe(2) + expect(mockNotifyCallback?.mock.calls[0]).toEqual([id]) + expect(mockNotifyCallback?.mock.calls[1]).toEqual([id]) + }) + + xtest('should merge cache data when server push two times data and uniq', async () => { + const id = 'fetchAsset3' + const testMethod = { method: 'fetchAsset2', params: [], id } + client.send(testMethod) + const mockData1 = { id, result: ['eth', 'bsc'] } + const mockData2 = { id, result: ['eth', 'matic'] } + pushToClientMockData(mockData1) + pushToClientMockData(mockData2) + + // @ts-ignore + await expect(server).toReceiveMessage(testMethod) + + const data = client.getResult(id) + expect(data).toEqual(['eth', 'bsc', 'matic']) + }) + + afterEach(() => { + server?.close() + client?.socket?.close() + }) +}) diff --git a/packages/web3-providers/tsconfig.tests.json b/packages/web3-providers/tsconfig.tests.json new file mode 100644 index 000000000000..15a9322f9963 --- /dev/null +++ b/packages/web3-providers/tsconfig.tests.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "noEmit": true, + "tsBuildInfoFile": "./dist/tests.tsbuildinfo", + "types": ["node"] + }, + "include": ["./**/*.test.ts"], + "references": [{ "path": "./tsconfig.json" }] +} From ed057f5e6a835ba7bc3a95e1c1e579c6a7d05dfa Mon Sep 17 00:00:00 2001 From: Lantt Date: Tue, 21 Dec 2021 14:44:46 +0800 Subject: [PATCH 05/59] test: should cache divide data when server push different response --- .../web3-providers/src/proxy/proxy.test.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/web3-providers/src/proxy/proxy.test.ts b/packages/web3-providers/src/proxy/proxy.test.ts index 50840b77733d..aba3b5e70ca8 100644 --- a/packages/web3-providers/src/proxy/proxy.test.ts +++ b/packages/web3-providers/src/proxy/proxy.test.ts @@ -55,6 +55,35 @@ describe('Proxy websocket', () => { expect(mockNotifyCallback?.mock.calls[1]).toEqual([id]) }) + test('should cache divide data when server push different response', async () => { + const requestID1 = 'fetchAsset1' + const requestID2 = 'fetchAsset2' + const testMethod1 = { method: requestID1, params: [], id: requestID1 } + const testMethod2 = { method: requestID1, params: [], id: requestID2 } + client.send(testMethod1) + client.send(testMethod2) + const mockData1 = { id: requestID1, result: ['eth', 'bsc'] } + const mockData2 = { id: requestID2, result: [1] } + const mockData3 = { id: requestID1, result: ['matic'] } + pushToClientMockData(mockData1) + pushToClientMockData(mockData2) + pushToClientMockData(mockData3) + + // @ts-ignore + await expect(server).toReceiveMessage(testMethod1) + // @ts-ignore + await expect(server).toReceiveMessage(testMethod2) + + const request1Data = client.getResult(requestID1) + expect(request1Data).toEqual(['eth', 'bsc', 'matic']) + const request2Data = client.getResult(requestID2) + expect(request2Data).toEqual([1]) + expect(mockNotifyCallback?.mock.calls.length).toBe(3) + expect(mockNotifyCallback?.mock.calls[0]).toEqual([requestID1]) + expect(mockNotifyCallback?.mock.calls[1]).toEqual([requestID2]) + expect(mockNotifyCallback?.mock.calls[2]).toEqual([requestID1]) + }) + xtest('should merge cache data when server push two times data and uniq', async () => { const id = 'fetchAsset3' const testMethod = { method: 'fetchAsset2', params: [], id } From 38858bb20fce7efea0c11fa4b556f3886e3c284b Mon Sep 17 00:00:00 2001 From: Lantt Date: Tue, 21 Dec 2021 23:58:44 +0800 Subject: [PATCH 06/59] feat: happy pass for get debank asset --- .../components/FungibleTokenTable/index.tsx | 7 +- packages/dashboard/src/web3/context.ts | 5 + .../plugins/EVM/UI/Web3State/getAssetsFn.ts | 14 +- packages/mask/src/web3/context.ts | 2 + packages/plugins/Wallet/src/messages.ts | 2 + packages/web3-providers/src/proxy/index.ts | 72 ---------- packages/web3-shared/base/index.ts | 1 + packages/web3-shared/base/package.json | 3 +- packages/web3-shared/base/proxy/index.ts | 131 ++++++++++++++++++ .../base}/proxy/proxy.test.ts | 67 ++++++++- packages/web3-shared/evm/context/type.ts | 2 + 11 files changed, 222 insertions(+), 84 deletions(-) delete mode 100644 packages/web3-providers/src/proxy/index.ts create mode 100644 packages/web3-shared/base/proxy/index.ts rename packages/{web3-providers/src => web3-shared/base}/proxy/proxy.test.ts (62%) diff --git a/packages/dashboard/src/pages/Wallets/components/FungibleTokenTable/index.tsx b/packages/dashboard/src/pages/Wallets/components/FungibleTokenTable/index.tsx index c88ff649b6b5..148f1f64fda8 100644 --- a/packages/dashboard/src/pages/Wallets/components/FungibleTokenTable/index.tsx +++ b/packages/dashboard/src/pages/Wallets/components/FungibleTokenTable/index.tsx @@ -87,8 +87,13 @@ export const FungibleTokenTable = memo(({ selectedChainId }) => useEffect(() => { PluginMessages.Wallet.events.erc20TokensUpdated.on(() => - setTimeout(() => setTokenUpdateCount(tokenUpdateCount + 1), 100), + setTimeout(() => setTokenUpdateCount((prev) => prev + 1), 100), ) + PluginMessages.Wallet.events.websocketDataUpdate.on((id) => { + if ('fetchFungibleTokenAsset' + account + network?.chainId) { + setTimeout(() => setTokenUpdateCount((prev) => prev + 1), 100) + } + }) }, []) const onSwap = useCallback((token: Web3Plugin.FungibleToken) => { diff --git a/packages/dashboard/src/web3/context.ts b/packages/dashboard/src/web3/context.ts index e1d4d70b5824..3bd12093209e 100644 --- a/packages/dashboard/src/web3/context.ts +++ b/packages/dashboard/src/web3/context.ts @@ -16,6 +16,7 @@ import { isInjectedProvider, } from '@masknet/web3-shared-evm' import { Services, Messages, PluginServices, PluginMessages } from '../API' +import { getProxyWebsocketInstance } from '@masknet/web3-shared-base' const Web3Provider = createExternalProvider() @@ -113,6 +114,10 @@ export const Web3Context: Web3ProviderType = { getAddressNamesList: PluginServices.Wallet.getAddressNames, getTransactionList: PluginServices.Wallet.getTransactionList, fetchERC20TokensFromTokenLists: Services.Ethereum.fetchERC20TokensFromTokenLists, + // TODO: send to local + providerSocketInstance: getProxyWebsocketInstance((id) => + PluginMessages.Wallet.events.websocketDataUpdate.sendToAll(id), + ), } export function createExternalProvider() { diff --git a/packages/mask/src/plugins/EVM/UI/Web3State/getAssetsFn.ts b/packages/mask/src/plugins/EVM/UI/Web3State/getAssetsFn.ts index 6e882c61ea2d..bbf66233fc1d 100644 --- a/packages/mask/src/plugins/EVM/UI/Web3State/getAssetsFn.ts +++ b/packages/mask/src/plugins/EVM/UI/Web3State/getAssetsFn.ts @@ -5,7 +5,6 @@ import { createNativeToken, ERC721TokenDetailed, formatEthereumAddress, - FungibleAssetProvider, getERC721TokenDetailedFromChain, getERC721TokenAssetFromChain, getEthereumConstants, @@ -27,6 +26,7 @@ import type { ERC721 } from '@masknet/web3-contracts/types/ERC721' export const getFungibleAssetsFn = (context: Web3ProviderType) => async (address: string, providerType: string, network: Web3Plugin.NetworkDescriptor, pagination?: Pagination) => { + const ws = await context.providerSocketInstance const chainId = context.chainId.getCurrentValue() const provider = context.provider.getCurrentValue() const wallet = context.wallets.getCurrentValue().find((x) => isSameAddress(x.address, address)) @@ -38,8 +38,16 @@ export const getFungibleAssetsFn = const web3 = new Web3(provider) const { BALANCE_CHECKER_ADDRESS } = getEthereumConstants(chainId) const { NATIVE_TOKEN_ADDRESS } = getTokenConstants() - const dataFromProvider = await context.getAssetsList(address, FungibleAssetProvider.DEBANK) - const assetsFromProvider: Web3Plugin.Asset[] = dataFromProvider.map((x) => ({ + const socketId = 'fetchFungibleTokenAsset' + address + network.chainId + await ws.send({ + id: socketId, + method: 'fetchFungibleTokenAsset', + params: { + pageSize: 10, + }, + }) + const dataFromProvider = ws.getResult>(socketId) + const assetsFromProvider: Web3Plugin.Asset[] = dataFromProvider.map((x: any) => ({ id: x.token.address, chainId: x.token.chainId, balance: x.balance, diff --git a/packages/mask/src/web3/context.ts b/packages/mask/src/web3/context.ts index ae80a7c7e220..f7893706f11c 100644 --- a/packages/mask/src/web3/context.ts +++ b/packages/mask/src/web3/context.ts @@ -32,6 +32,7 @@ import type { InternalSettings } from '../settings/createSettings' import { Flags } from '../../shared' import { createExternalProvider } from './helpers' import Services from '../extension/service' +import { getProxyWebsocketInstance } from '@masknet/web3-shared-base' function createWeb3Context(disablePopup = false, isMask = false): Web3ProviderType { const Web3Provider = createExternalProvider( @@ -134,6 +135,7 @@ function createWeb3Context(disablePopup = false, isMask = false): Web3ProviderTy getAddressNamesList: WalletRPC.getAddressNames, getTransactionList: WalletRPC.getTransactionList, fetchERC20TokensFromTokenLists: Services.Ethereum.fetchERC20TokensFromTokenLists, + providerSocketInstance: getProxyWebsocketInstance(WalletMessages.events.websocketDataUpdate.sendToLocal), } } diff --git a/packages/plugins/Wallet/src/messages.ts b/packages/plugins/Wallet/src/messages.ts index 4146a2c7d109..0c4125bde866 100644 --- a/packages/plugins/Wallet/src/messages.ts +++ b/packages/plugins/Wallet/src/messages.ts @@ -218,6 +218,8 @@ export interface WalletMessage { erc1155TokensUpdated: void /** true: Now locked; false: Now unlocked */ walletLockStatusUpdated: boolean + /** request id */ + websocketDataUpdate: string rpc: unknown } diff --git a/packages/web3-providers/src/proxy/index.ts b/packages/web3-providers/src/proxy/index.ts deleted file mode 100644 index fe0f7fc5cf2d..000000000000 --- a/packages/web3-providers/src/proxy/index.ts +++ /dev/null @@ -1,72 +0,0 @@ -const PROXY_URL = 'wss://api-v4.zerion.io' - -export interface MessageBase { - id: string -} - -export interface RequestMessage extends MessageBase { - method: string - params: unknown -} - -export interface PayloadMessage extends MessageBase { - error?: unknown - result?: unknown[] -} - -interface PollItem { - pickedAt?: Date - createdAt: Date - updatedAt?: Date - data: T[] -} - -// todo: -// 1. avoid duplicate instance -// 2. auto reconnection -// 3. handle data is end -export class ProviderProxy { - private readonly _socket: WebSocket - private readonly _poll: Map - - constructor(point: string, notifyFn: (id: string) => void) { - this._socket = new WebSocket(point) - this._poll = new Map() - this._socket.addEventListener('message', (event: MessageEvent) => { - console.debug('Message from server ', event.data) - const { id, result, error } = JSON.parse(event.data) as PayloadMessage - if (error || !id) { - // TODO: handle errors - } - if (result) { - const updatedAt = new Date() - const itemInPoll = this._poll.get(id) ?? { data: [], createdAt: updatedAt } - const dataInPool = itemInPoll?.data ?? [] - const patchData = [...dataInPool, ...(result ?? [])] - this._poll.set(id, { ...itemInPoll, updatedAt: updatedAt, data: patchData }) - notifyFn(id) - } - }) - } - - /** - * Send request to proxy websocket - * @param message - */ - send(message: RequestMessage) { - if (this._poll.has(message.id)) { - //TODO: notify to pick - return - } - this._socket.send(JSON.stringify(message)) - this._poll.set(message.id, { data: [], createdAt: new Date() }) - } - - get socket(): WebSocket { - return this._socket - } - - getResult(id: string): T[] { - return (this._poll.get(id)?.data ?? []) as T[] - } -} diff --git a/packages/web3-shared/base/index.ts b/packages/web3-shared/base/index.ts index 9c56149efa59..d2cace8c6e1e 100644 --- a/packages/web3-shared/base/index.ts +++ b/packages/web3-shared/base/index.ts @@ -1 +1,2 @@ export * from './utils' +export * from './proxy' diff --git a/packages/web3-shared/base/package.json b/packages/web3-shared/base/package.json index c09ae1ec6285..ca723cd70368 100644 --- a/packages/web3-shared/base/package.json +++ b/packages/web3-shared/base/package.json @@ -4,6 +4,7 @@ "main": "./dist/index.js", "types": "./dist/index.d.ts", "dependencies": { - "bignumber.js": "9.0.2" + "bignumber.js": "9.0.2", + "date-fns": "^2.27.0" } } diff --git a/packages/web3-shared/base/proxy/index.ts b/packages/web3-shared/base/proxy/index.ts new file mode 100644 index 000000000000..08745113ba41 --- /dev/null +++ b/packages/web3-shared/base/proxy/index.ts @@ -0,0 +1,131 @@ +import differenceInSeconds from 'date-fns/differenceInSeconds' +export interface MessageBase { + id: string +} + +export interface RequestMessage extends MessageBase { + method: string + params: unknown +} + +export interface PayloadMessage extends MessageBase { + error?: unknown + results?: unknown[] +} + +export interface PollItem { + pickedAt?: Date + createdAt: Date + updatedAt?: Date + data: T[] + done?: boolean +} + +// todo: +// 1. avoid duplicate instance +// 2. auto reconnection +// 3. handle data is end +// 4. handle open status +export class ProviderProxy { + private readonly _socket: WebSocket + private readonly _poll: Map + private readonly _notify: (id: string) => void + + constructor(point: string, notifyFn: (id: string) => void) { + this._socket = new WebSocket(point) + this._poll = new Map() + this._notify = notifyFn + } + + public waitingOpen = () => { + return new Promise((resolve, reject) => { + this._socket.addEventListener('open', () => resolve()) + this._socket.addEventListener('error', () => reject()) + }) + } + + public registerMessage = () => { + this._socket.addEventListener('message', (event: MessageEvent) => { + console.debug('Message from server ', event.data) + const { id, results, error } = JSON.parse(event.data) as PayloadMessage + if (error || !id) { + // TODO: handle errors + } + if (results) { + const updatedAt = new Date() + const itemInPoll = this._poll.get(id) ?? { data: [], createdAt: updatedAt } + const dataInPool = itemInPoll?.data ?? [] + const patchData = [...dataInPool, ...(results ?? [])] + this._poll.set(id, { ...itemInPoll, updatedAt, data: patchData }) + this._notify(id) + } + }) + } + + /** + * Send request to proxy websocket + * @param message + */ + send(message: RequestMessage) { + const cache = this._poll.get(message.id) + if (cache && !this.isExpired(cache!)) return + + this._socket.send(JSON.stringify(message)) + this._poll.set(message.id, { data: [], createdAt: new Date() }) + } + + get socket(): WebSocket { + return this._socket + } + + getResult(id: string): T[] { + // todo: update pick time + return (this._poll.get(id)?.data ?? []) as T[] + } + + /** + * Cache is expired + * @param item cache item + * @returns boolean + */ + isExpired(item: PollItem) { + const now = new Date() + // lasted update time > 30s + if (!!item.updatedAt && differenceInSeconds(now, item.updatedAt) > 30) return true + + // lasted pick time > 30s + return !!item.pickedAt && differenceInSeconds(now, item.pickedAt) > 30 + } +} + +const DEV = 'wss://hyper-proxy-development.mask-reverse-proxy.workers.dev' + +enum SocketState { + CONNECTING = 0, + OPEN = 1, + CLOSING = 2, + CLOSED = 3, +} + +function getProxyWebsocketInstanceWrapper(): (notify: (id: string) => void) => Promise { + let cached: ProviderProxy + return async (notify: (id: string) => void) => { + const state = cached?.socket.readyState + if (!cached || state === SocketState.CLOSING || state === SocketState.CLOSED) { + cached = new ProviderProxy(DEV, notify) + await cached.waitingOpen() + cached.registerMessage() + return cached + } + + if (cached?.socket.readyState === SocketState.CONNECTING) { + await cached.waitingOpen() + cached.registerMessage() + return cached + } + + return cached + } +} + +export const getProxyWebsocketInstance = getProxyWebsocketInstanceWrapper() diff --git a/packages/web3-providers/src/proxy/proxy.test.ts b/packages/web3-shared/base/proxy/proxy.test.ts similarity index 62% rename from packages/web3-providers/src/proxy/proxy.test.ts rename to packages/web3-shared/base/proxy/proxy.test.ts index aba3b5e70ca8..5f8a3bed8723 100644 --- a/packages/web3-providers/src/proxy/proxy.test.ts +++ b/packages/web3-shared/base/proxy/proxy.test.ts @@ -1,6 +1,7 @@ import { describe, test, expect, beforeEach, afterEach, xtest, jest } from '@jest/globals' -import { ProviderProxy } from './index' +import { PollItem, ProviderProxy } from './index' import * as mockWS from 'jest-websocket-mock' +import addSeconds from 'date-fns/addSeconds' const POINT = 'ws://localhost:1235' @@ -16,13 +17,15 @@ describe('Proxy websocket', () => { mockNotifyCallback = jest.fn() server = new mockWS.WS(POINT, { jsonProtocol: true }) client = new ProviderProxy(POINT, mockNotifyCallback) + await client.waitingOpen() + client.registerMessage() await server.connected }) test('should cache data by request id', async () => { const id = 'fetchAsset' - const mockData = { id, result: ['eth', 'bsc'] } + const mockData = { id, results: ['eth', 'bsc'] } const testMethod = { method: 'fetchAsset', params: [], id } client.send(testMethod) pushToClientMockData(mockData) @@ -31,6 +34,7 @@ describe('Proxy websocket', () => { await expect(server).toReceiveMessage(testMethod) const data = client.getResult(id) + console.log(data) expect(data).toEqual(['eth', 'bsc']) expect(mockNotifyCallback?.mock.calls.length).toBe(1) expect(mockNotifyCallback?.mock.calls[0]).toEqual([id]) @@ -40,8 +44,8 @@ describe('Proxy websocket', () => { const id = 'fetchAsset2' const testMethod = { method: 'fetchAsset2', params: [], id } client.send(testMethod) - const mockData1 = { id, result: ['eth', 'bsc'] } - const mockData2 = { id, result: ['matic'] } + const mockData1 = { id, results: ['eth', 'bsc'] } + const mockData2 = { id, results: ['matic'] } pushToClientMockData(mockData1) pushToClientMockData(mockData2) @@ -62,9 +66,9 @@ describe('Proxy websocket', () => { const testMethod2 = { method: requestID1, params: [], id: requestID2 } client.send(testMethod1) client.send(testMethod2) - const mockData1 = { id: requestID1, result: ['eth', 'bsc'] } - const mockData2 = { id: requestID2, result: [1] } - const mockData3 = { id: requestID1, result: ['matic'] } + const mockData1 = { id: requestID1, results: ['eth', 'bsc'] } + const mockData2 = { id: requestID2, results: [1] } + const mockData3 = { id: requestID1, results: ['matic'] } pushToClientMockData(mockData1) pushToClientMockData(mockData2) pushToClientMockData(mockData3) @@ -84,6 +88,17 @@ describe('Proxy websocket', () => { expect(mockNotifyCallback?.mock.calls[2]).toEqual([requestID1]) }) + test('should use cache when last pick within 30 second', async () => { + const id = 'fetchAsset' + const testMethod = { method: 'fetchAsset', params: [], id } + client.send(testMethod) + const mockData = { id, results: ['matic'] } + pushToClientMockData(mockData) + + // @ts-ignore + await expect(server).toReceiveMessage(testMethod) + }) + xtest('should merge cache data when server push two times data and uniq', async () => { const id = 'fetchAsset3' const testMethod = { method: 'fetchAsset2', params: [], id } @@ -105,3 +120,41 @@ describe('Proxy websocket', () => { client?.socket?.close() }) }) + +describe('Proxy websocket unit', () => { + test('should expired when pick before 30s', () => { + const client = new ProviderProxy(POINT, () => {}) + const item = { + pickedAt: addSeconds(new Date(), -31), + } + const isExpired = client.isExpired(item as PollItem) + expect(isExpired).toBe(true) + }) + + test('should not expired when pick within 30s', () => { + const client = new ProviderProxy(POINT, () => {}) + const item = { + pickedAt: addSeconds(new Date(), -20), + } + const isExpired = client.isExpired(item as PollItem) + expect(isExpired).toBe(false) + }) + + test('should not expired when update before 30s', () => { + const client = new ProviderProxy(POINT, () => {}) + const item = { + updatedAt: addSeconds(new Date(), -20), + } + const isExpired = client.isExpired(item as PollItem) + expect(isExpired).toBe(false) + }) + + test('should not expired when update within 30s', () => { + const client = new ProviderProxy(POINT, () => {}) + const item = { + updatedAt: addSeconds(new Date(), -20), + } + const isExpired = client.isExpired(item as PollItem) + expect(isExpired).toBe(false) + }) +}) diff --git a/packages/web3-shared/evm/context/type.ts b/packages/web3-shared/evm/context/type.ts index e84400683b85..4482e645a0df 100644 --- a/packages/web3-shared/evm/context/type.ts +++ b/packages/web3-shared/evm/context/type.ts @@ -18,6 +18,7 @@ import type { BalanceOfChains, ERC721TokenCollectionInfo, } from '../types' +import type { ProviderProxy } from '@masknet/web3-shared-base' export interface Web3ProviderType { allowTestnet: Subscription @@ -79,4 +80,5 @@ export interface Web3ProviderType { hasNextPage: boolean }> fetchERC20TokensFromTokenLists: (urls: string[], chainId: ChainId) => Promise + providerSocketInstance: Promise } From 1ad26f8df640700bcaeae63e3c56383c95c8ce24 Mon Sep 17 00:00:00 2001 From: Lantt Date: Wed, 22 Dec 2021 00:07:47 +0800 Subject: [PATCH 07/59] fix: remove console --- packages/dashboard/src/components/Restore/RestoreFromLocal.tsx | 1 - packages/mask/src/extension/background-script/WelcomeService.ts | 1 - packages/web3-shared/base/proxy/proxy.test.ts | 1 - 3 files changed, 3 deletions(-) diff --git a/packages/dashboard/src/components/Restore/RestoreFromLocal.tsx b/packages/dashboard/src/components/Restore/RestoreFromLocal.tsx index 2a6ea47fc8c5..8ef85e403af1 100644 --- a/packages/dashboard/src/components/Restore/RestoreFromLocal.tsx +++ b/packages/dashboard/src/components/Restore/RestoreFromLocal.tsx @@ -102,7 +102,6 @@ export const RestoreFromLocal = memo(() => { }, [currentPersona, changeCurrentPersona]) const restoreDB = useCallback(async () => { - console.log(json) try { // If json has wallets, restore in popup. if (json?.wallets) { diff --git a/packages/mask/src/extension/background-script/WelcomeService.ts b/packages/mask/src/extension/background-script/WelcomeService.ts index b63da6ebd17f..4ef43878ca32 100644 --- a/packages/mask/src/extension/background-script/WelcomeService.ts +++ b/packages/mask/src/extension/background-script/WelcomeService.ts @@ -132,7 +132,6 @@ export async function checkPermissionsAndRestore(id: string) { export async function checkPermissionAndOpenWalletRecovery(id: string) { const json = await getUnconfirmedBackup(id) - console.log(json) if (json) { const permissions = await extraPermissions(json.grantedHostPermissions) if (permissions.length) { diff --git a/packages/web3-shared/base/proxy/proxy.test.ts b/packages/web3-shared/base/proxy/proxy.test.ts index 5f8a3bed8723..4b75f8751808 100644 --- a/packages/web3-shared/base/proxy/proxy.test.ts +++ b/packages/web3-shared/base/proxy/proxy.test.ts @@ -34,7 +34,6 @@ describe('Proxy websocket', () => { await expect(server).toReceiveMessage(testMethod) const data = client.getResult(id) - console.log(data) expect(data).toEqual(['eth', 'bsc']) expect(mockNotifyCallback?.mock.calls.length).toBe(1) expect(mockNotifyCallback?.mock.calls[0]).toEqual([id]) From 4f1ce8d93406de09e60e91390d7e68012580e3c1 Mon Sep 17 00:00:00 2001 From: Jack Works Date: Mon, 20 Dec 2021 10:04:51 +0800 Subject: [PATCH 08/59] chore: use esbuild --- packages/provider-proxy/build.mjs | 64 ++++++++++++++++++++++++ packages/provider-proxy/package.json | 10 +--- packages/provider-proxy/rollup.config.js | 34 ------------- packages/web3-shared/evm/package.json | 1 + 4 files changed, 67 insertions(+), 42 deletions(-) create mode 100644 packages/provider-proxy/build.mjs delete mode 100644 packages/provider-proxy/rollup.config.js diff --git a/packages/provider-proxy/build.mjs b/packages/provider-proxy/build.mjs new file mode 100644 index 000000000000..a7b7ce0052cf --- /dev/null +++ b/packages/provider-proxy/build.mjs @@ -0,0 +1,64 @@ +// @ts-check +import { build } from 'esbuild' +import { join, dirname } from 'path' +import { fileURLToPath } from 'url' +import { readdir } from 'fs/promises' + +// @ts-ignore +const __dirname = dirname(fileURLToPath(import.meta.url)) +const externals = [ + // @ts-ignore + ...new Set((await readdir(join(__dirname, '../../node_modules/.pnpm'))).map(resolvePackageName)), +].filter(isExternal) + +// @ts-ignore +const result = await build({ + absWorkingDir: __dirname, + bundle: true, + entryPoints: [join(__dirname, './src/index.ts')], + treeShaking: true, + outfile: join(__dirname, './dist/out.js'), + platform: 'node', + external: externals.concat('electron'), + format: 'esm', + // minify: true, + plugins: [ + { + name: 'everything-side-effect-free', + setup: (build) => { + build.onResolve({ filter: /.*/ }, (args) => { + // if ( + // !args.path.startsWith('.') && + // (args.kind === 'import-statement' || args.kind === 'require-call') + // ) { + // console.log(`[${args.importer}] import '${args.path}'`) + // if (args.path.startsWith('@masknet/')) { + // return { external: false, sideEffects: false } + // } + // return { + // external: true, + // sideEffects: false, + // } + // } + return { sideEffects: false } + }) + }, + }, + ], +}) + +/** @param pkg {string} */ +function resolvePackageName(pkg) { + if (pkg.startsWith('@')) { + return `@` + pkg.slice(1).split('@')[0].replace('+', '/') + } + return pkg.split('@')[0] +} +/** @param pkg {string} */ +function isExternal(pkg) { + // Monorepo packages + if (pkg.startsWith('@masknet')) return false + // Some of @dimensiondev packages are published on the Github registry which is not convenient to use. + if (pkg.startsWith('@dimensiondev')) return false + return true +} diff --git a/packages/provider-proxy/package.json b/packages/provider-proxy/package.json index 1db642a44121..d486fd9363c1 100644 --- a/packages/provider-proxy/package.json +++ b/packages/provider-proxy/package.json @@ -9,17 +9,11 @@ "dependencies": { "@masknet/plugin-infra": "workspace:*", "@masknet/web3-providers": "workspace:*", - "@masknet/web3-shared-evm": "workspace:*", "@masknet/web3-shared-base": "workspace:*", + "@masknet/web3-shared-evm": "workspace:*", "urlcat": "^2.0.4" }, "devDependencies": { - "@rollup/plugin-commonjs": "^21.0.1", - "@rollup/plugin-json": "^4.1.0", - "@rollup/plugin-node-resolve": "^13.0.6", - "@rollup/plugin-typescript": "^8.3.0", - "rollup": "^2.60.1", - "rollup-plugin-terser": "^7.0.2", - "rollup-plugin-typescript2": "^0.31.1" + "esbuild": "^0.14.5" } } diff --git a/packages/provider-proxy/rollup.config.js b/packages/provider-proxy/rollup.config.js deleted file mode 100644 index 83fd0a9b21ba..000000000000 --- a/packages/provider-proxy/rollup.config.js +++ /dev/null @@ -1,34 +0,0 @@ -import nodeResolve from '@rollup/plugin-node-resolve' -import commonjs from '@rollup/plugin-commonjs' -import json from '@rollup/plugin-json' -import typescript from 'rollup-plugin-typescript2' -import { terser } from 'rollup-plugin-terser' - -const config = { - input: './src/index.ts', - output: { - dir: 'dist', - format: 'cjs', - }, - plugins: [ - typescript({ - compilerOptions: { - declaration: false, - declarationMap: false, - tsconfig: 'tsconfig.json', - }, - }), - nodeResolve({ - exportConditions: ['node'], - preferBuiltins: false, - mainFields: ['module', 'main'], - rootDir: '../..', - moduleDirectories: ['../../node_modules', 'node_modules'], - }), - commonjs(), - json(), - terser(), - ], -} - -export default config diff --git a/packages/web3-shared/evm/package.json b/packages/web3-shared/evm/package.json index 5e3ee7ba8e4c..f4814d096e1f 100644 --- a/packages/web3-shared/evm/package.json +++ b/packages/web3-shared/evm/package.json @@ -3,6 +3,7 @@ "private": true, "main": "./dist/index.js", "types": "./dist/index.d.ts", + "sideEffects": false, "dependencies": { "@ensdomains/address-encoder": "^0.2.6", "@masknet/shared-base": "workspace:*", From 067bec0692fc4cf3e0a76a748897e6f9348304e8 Mon Sep 17 00:00:00 2001 From: Jack Works Date: Mon, 20 Dec 2021 10:06:29 +0800 Subject: [PATCH 09/59] chore: extra --- packages/provider-proxy/build.mjs | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/packages/provider-proxy/build.mjs b/packages/provider-proxy/build.mjs index a7b7ce0052cf..156f864543ff 100644 --- a/packages/provider-proxy/build.mjs +++ b/packages/provider-proxy/build.mjs @@ -22,29 +22,6 @@ const result = await build({ external: externals.concat('electron'), format: 'esm', // minify: true, - plugins: [ - { - name: 'everything-side-effect-free', - setup: (build) => { - build.onResolve({ filter: /.*/ }, (args) => { - // if ( - // !args.path.startsWith('.') && - // (args.kind === 'import-statement' || args.kind === 'require-call') - // ) { - // console.log(`[${args.importer}] import '${args.path}'`) - // if (args.path.startsWith('@masknet/')) { - // return { external: false, sideEffects: false } - // } - // return { - // external: true, - // sideEffects: false, - // } - // } - return { sideEffects: false } - }) - }, - }, - ], }) /** @param pkg {string} */ From ce32ce264898e96d3d27ad016c6336b2c08ef312 Mon Sep 17 00:00:00 2001 From: Jack Works Date: Tue, 21 Dec 2021 17:03:57 +0800 Subject: [PATCH 10/59] fix: build --- packages/provider-proxy/package.json | 7 +- packages/provider-proxy/rollup.config.js | 32 ++++++++ .../src/Identifier/IdentifierMap.ts | 2 - packages/shared-base/src/Identifier/type.ts | 6 -- packages/shared-base/src/serializer/index.ts | 79 +++++++++++++------ 5 files changed, 91 insertions(+), 35 deletions(-) create mode 100644 packages/provider-proxy/rollup.config.js diff --git a/packages/provider-proxy/package.json b/packages/provider-proxy/package.json index d486fd9363c1..7e40937a0943 100644 --- a/packages/provider-proxy/package.json +++ b/packages/provider-proxy/package.json @@ -14,6 +14,11 @@ "urlcat": "^2.0.4" }, "devDependencies": { - "esbuild": "^0.14.5" + "@rollup/plugin-alias": "^3.1.8", + "@rollup/plugin-commonjs": "^21.0.1", + "@rollup/plugin-json": "^4.1.0", + "@rollup/plugin-node-resolve": "^13.0.6", + "@rollup/plugin-sucrase": "^4.0.1", + "rollup": "^2.60.1" } } diff --git a/packages/provider-proxy/rollup.config.js b/packages/provider-proxy/rollup.config.js new file mode 100644 index 000000000000..0aff6ee8d4d2 --- /dev/null +++ b/packages/provider-proxy/rollup.config.js @@ -0,0 +1,32 @@ +import commonjs from '@rollup/plugin-commonjs' +import node from '@rollup/plugin-node-resolve' +import sucrase from '@rollup/plugin-sucrase' +import json from '@rollup/plugin-json' +import alias from '@rollup/plugin-alias' + +export default { + input: './src/index.ts', + output: { + file: 'dist/output.js', + format: 'cjs', + }, + plugins: [ + commonjs(), + node(), + json(), + alias({ + entries: [ + { find: '@masknet/web3-shared-evm', replacement: '../web3-shared/evm/index.ts' }, + { find: '@masknet/web3-shared-base', replacement: '../web3-shared/base/index.ts' }, + ], + }), + sucrase({ transforms: ['typescript', 'jsx'] }), + ], + external: (id) => { + if (id.startsWith('.')) return false + if (id.includes('@masknet')) return false + if (id.includes('node_modules')) return true + return false + }, + treeshake: { moduleSideEffects: false, propertyReadSideEffects: false }, +} diff --git a/packages/shared-base/src/Identifier/IdentifierMap.ts b/packages/shared-base/src/Identifier/IdentifierMap.ts index 0643263e4347..095e57d1b2e8 100644 --- a/packages/shared-base/src/Identifier/IdentifierMap.ts +++ b/packages/shared-base/src/Identifier/IdentifierMap.ts @@ -1,5 +1,4 @@ import { Identifier } from './type' -import { serialize } from '../serializer' import { immerable } from 'immer' /** @@ -100,7 +99,6 @@ export class IdentifierMap implements Map< return this.entries() } } -serialize('IdentifierMap')(IdentifierMap) IdentifierMap.prototype[Symbol.toStringTag] = 'IdentifierMap' export type ReadonlyIdentifierMap = ReadonlyMap & { diff --git a/packages/shared-base/src/Identifier/type.ts b/packages/shared-base/src/Identifier/type.ts index e5d259c9896f..90bb5255141b 100644 --- a/packages/shared-base/src/Identifier/type.ts +++ b/packages/shared-base/src/Identifier/type.ts @@ -1,4 +1,3 @@ -import { serialize } from '../serializer' import { Result, Ok, Err } from 'ts-results' /** @@ -113,7 +112,6 @@ export class ProfileIdentifier extends Identifier { return new ProfileIdentifier(network, userId) } } -serialize('ProfileIdentifier')(ProfileIdentifier) export class GroupIdentifier extends Identifier { static getFriendsGroupIdentifier(who: ProfileIdentifier, groupId: string) { return new GroupIdentifier(who.network, who.userId, groupId) @@ -147,7 +145,6 @@ export class GroupIdentifier extends Identifier { return new GroupIdentifier(network, belongs, groupID) } } -serialize('GroupIdentifier')(GroupIdentifier) export class PostIdentifier extends Identifier { static readonly unknown = new PostIdentifier(ProfileIdentifier.unknown, '$unknown') get isUnknown() { @@ -171,7 +168,6 @@ export class PostIdentifier extends Identifie return new PostIdentifier(id.val, postId) } } -serialize('PostIdentifier')(PostIdentifier) export class PostIVIdentifier extends Identifier { constructor(public readonly network: string, public readonly postIV: string) { super() @@ -185,7 +181,6 @@ export class PostIVIdentifier extends Identifier { return new PostIVIdentifier(network, iv) } } -serialize('PostIVIdentifier')(PostIVIdentifier) /** * This class identify the point on an EC curve. @@ -208,7 +203,6 @@ export class ECKeyIdentifier extends Identifier { return new ECKeyIdentifier(curve, point) } } -serialize('ECKeyIdentifier')(ECKeyIdentifier) export type PersonaIdentifier = ECKeyIdentifier // eslint-disable-next-line no-redeclare export const PersonaIdentifier = [ECKeyIdentifier] diff --git a/packages/shared-base/src/serializer/index.ts b/packages/shared-base/src/serializer/index.ts index ec0fa5b1831b..577aa9a14663 100644 --- a/packages/shared-base/src/serializer/index.ts +++ b/packages/shared-base/src/serializer/index.ts @@ -3,11 +3,52 @@ import Typeson from 'typeson' import type { Serialization } from 'async-call-rpc' import { Ok, Err, Some, None } from 'ts-results' import * as BN from 'bignumber.js' -const { default: BigNumber } = BN -/** @internal */ -export function serialize(name: string, ser?: (x: T) => Q, des?: (x: Q) => T) { - return (constructor: T) => { +// @ts-ignore +import builtins from 'typeson-registry/dist/presets/builtin' // @ts-ignore +import blob from 'typeson-registry/dist/types/blob' // @ts-ignore +import file from 'typeson-registry/dist/types/file' // @ts-ignore +import fileList from 'typeson-registry/dist/types/filelist' // @ts-ignore +import imageBitMap from 'typeson-registry/dist/types/imagebitmap' // @ts-ignore +import num from 'typeson-registry/dist/presets/special-numbers' +import { IdentifierMap } from '../Identifier/IdentifierMap' +import { + ECKeyIdentifier, + GroupIdentifier, + PostIdentifier, + PostIVIdentifier, + ProfileIdentifier, +} from '../Identifier/type' + +let typeson: Typeson | undefined +function setup() { + const { default: BigNumber } = BN + type f = (...args: any[]) => any + type T = [name: string, ser: f | undefined, de_ser: f | undefined, ctor: f] + const all: T[] = [] + function addClass(name: string, constructor: any) { + all.push([name, undefined, undefined, constructor]) + } + typeson = new Typeson({}) + typeson.register(builtins) + typeson.register(num) + typeson.register([blob, file, fileList, imageBitMap, num]) + typeson.register({ None: [(x) => x === None, () => 'None', () => None] }) + + addClass('Ok', Ok) + addClass('Err', Err) + addClass('Some', Some) + + addClass('BigNumber', BigNumber) + + addClass('ProfileIdentifier', ProfileIdentifier) + addClass('ECKeyIdentifier', ECKeyIdentifier) + addClass('GroupIdentifier', GroupIdentifier) + addClass('PostIdentifier', PostIdentifier) + addClass('PostIVIdentifier', PostIVIdentifier) + addClass('IdentifierMap', IdentifierMap) + + for (const [name, ser, des, constructor] of all) { Object.defineProperty(constructor, 'name', { configurable: true, enumerable: false, @@ -20,45 +61,31 @@ export function serialize(name: string, ser?: (x: T) => Q, des?: (x: Q) => ? [(x) => x instanceof constructor, ser, des] : [ (x) => x instanceof constructor, + // eslint-disable-next-line @typescript-eslint/no-loop-func (x: unknown) => { const y = Object.assign({}, x) Object.getOwnPropertySymbols(y).forEach((x) => Reflect.deleteProperty(y, x)) - return typeson.encapsulate(y) + return typeson!.encapsulate(y) }, + // eslint-disable-next-line @typescript-eslint/no-loop-func (x: unknown) => { - const y = typeson.revive(x) + const y = typeson!.revive(x) Object.setPrototypeOf(y, constructor.prototype) return y }, ], }) - return constructor } } - -// @ts-ignore -import builtins from 'typeson-registry/dist/presets/builtin' // @ts-ignore -import blob from 'typeson-registry/dist/types/blob' // @ts-ignore -import file from 'typeson-registry/dist/types/file' // @ts-ignore -import fileList from 'typeson-registry/dist/types/filelist' // @ts-ignore -import imageBitMap from 'typeson-registry/dist/types/imagebitmap' // @ts-ignore -import num from 'typeson-registry/dist/presets/special-numbers' -const typeson = new Typeson({}) -typeson.register(builtins) -typeson.register(num) -typeson.register([blob, file, fileList, imageBitMap, num]) -serialize('Ok')(Ok) -serialize('Err')(Err) -serialize('Some')(Some) -typeson.register({ None: [(x) => x === None, () => 'None', () => None] }) -serialize('BigNumber')(BigNumber) export const serializer: Serialization = { serialization(from: unknown) { - return typeson.encapsulate(from) + if (!typeson) setup() + return typeson!.encapsulate(from) }, deserialization(to: string) { + if (!typeson) setup() try { - return typeson.revive(to) + return typeson!.revive(to) } catch (error) { console.error(error) } From ed0de87e5a18b6bae37064104f82fe437b4894e9 Mon Sep 17 00:00:00 2001 From: Jack Works Date: Tue, 21 Dec 2021 17:25:47 +0800 Subject: [PATCH 11/59] feat: pack the package --- packages/provider-proxy/build.mjs | 84 ++++++++++++++++------------ packages/provider-proxy/package.json | 4 +- 2 files changed, 52 insertions(+), 36 deletions(-) diff --git a/packages/provider-proxy/build.mjs b/packages/provider-proxy/build.mjs index 156f864543ff..19af74531ed0 100644 --- a/packages/provider-proxy/build.mjs +++ b/packages/provider-proxy/build.mjs @@ -1,41 +1,55 @@ // @ts-check -import { build } from 'esbuild' -import { join, dirname } from 'path' +import { spawn } from 'child_process' +import { readFile, writeFile } from 'fs/promises' +import { join, resolve } from 'path' import { fileURLToPath } from 'url' -import { readdir } from 'fs/promises' -// @ts-ignore -const __dirname = dirname(fileURLToPath(import.meta.url)) -const externals = [ - // @ts-ignore - ...new Set((await readdir(join(__dirname, '../../node_modules/.pnpm'))).map(resolvePackageName)), -].filter(isExternal) +const __file = fileURLToPath(import.meta.url) +const __dirname = resolve(__file, '../') -// @ts-ignore -const result = await build({ - absWorkingDir: __dirname, - bundle: true, - entryPoints: [join(__dirname, './src/index.ts')], - treeShaking: true, - outfile: join(__dirname, './dist/out.js'), - platform: 'node', - external: externals.concat('electron'), - format: 'esm', - // minify: true, -}) - -/** @param pkg {string} */ -function resolvePackageName(pkg) { - if (pkg.startsWith('@')) { - return `@` + pkg.slice(1).split('@')[0].replace('+', '/') - } - return pkg.split('@')[0] +function awaitChildProcess(child) { + return new Promise((resolve, reject) => { + child.on('error', () => reject(child.exitCode || 0)) + child.on('exit', (code) => resolve(code || 0)) + }) } -/** @param pkg {string} */ -function isExternal(pkg) { - // Monorepo packages - if (pkg.startsWith('@masknet')) return false - // Some of @dimensiondev packages are published on the Github registry which is not convenient to use. - if (pkg.startsWith('@dimensiondev')) return false - return true + +const rollup = awaitChildProcess( + spawn('pnpm run build:rollup', { + cwd: __dirname, + shell: true, + stdio: 'inherit', + }), +) +await rollup +const version = JSON.parse(await readFile(join(__dirname, './package.json'), 'utf-8')).version +const now = new Date() +const packageJSON = { + name: '@dimensiondev/provider-proxy', + version: `${version}-${now.getFullYear()}${String(now.getMonth() + 1).padStart(2)}${String(now.getDate()).padStart( + 2, + )}-${String(now.getHours()).padStart(2)}${String(now.getMinutes()).padStart(2)}`, + dependencies: { + 'wallet.ts': '1.0.1', + 'bignumber.js': '9.0.1', + }, + main: './output.js', } +await writeFile(join(__dirname, './dist/package.json'), JSON.stringify(packageJSON, undefined, 4)) + +const out = join(__dirname, './dist') +// validate if there is dependency missing +await awaitChildProcess( + spawn('npm install', { + cwd: out, + shell: true, + stdio: 'inherit', + }), +) +await awaitChildProcess( + spawn('node output.js', { + cwd: out, + shell: true, + stdio: 'inherit', + }), +) diff --git a/packages/provider-proxy/package.json b/packages/provider-proxy/package.json index 7e40937a0943..a99b917838df 100644 --- a/packages/provider-proxy/package.json +++ b/packages/provider-proxy/package.json @@ -1,10 +1,12 @@ { "name": "@masknet/provider-proxy", + "version": "0.0.1", "private": true, "main": "./dist/index.js", "types": "./dist/index.d.ts", "scripts": { - "build": "rollup -c -w" + "build:rollup": "rollup -c", + "build": "node ./build.mjs" }, "dependencies": { "@masknet/plugin-infra": "workspace:*", From ff55e32a520c9e59e9f1b80a7cd04ab684e70e3c Mon Sep 17 00:00:00 2001 From: Lantt Date: Wed, 22 Dec 2021 01:05:11 +0800 Subject: [PATCH 12/59] feat: update deploy ci config --- .github/workflows/deploy-proxy.yml | 6 +++--- packages/provider-proxy/build.mjs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-proxy.yml b/.github/workflows/deploy-proxy.yml index f52fbb4a5754..00aa3bb2167e 100644 --- a/.github/workflows/deploy-proxy.yml +++ b/.github/workflows/deploy-proxy.yml @@ -46,8 +46,8 @@ jobs: with: node-version: 16 registry-url: https://npm.pkg.github.com/ - - name: Run Install - working-directory: ./packages/provider-proxy - run: npm publish --scope=@nikomaniac --registry=https://npm.pkg.github.com + - name: Publish to github npm pkg + working-directory: ./packages/provider-proxy/dist + run: npm publish --scope=@dimensiondev --registry=https://npm.pkg.github.com env: NODE_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/packages/provider-proxy/build.mjs b/packages/provider-proxy/build.mjs index 19af74531ed0..1bc411f39e0c 100644 --- a/packages/provider-proxy/build.mjs +++ b/packages/provider-proxy/build.mjs @@ -28,7 +28,7 @@ const packageJSON = { name: '@dimensiondev/provider-proxy', version: `${version}-${now.getFullYear()}${String(now.getMonth() + 1).padStart(2)}${String(now.getDate()).padStart( 2, - )}-${String(now.getHours()).padStart(2)}${String(now.getMinutes()).padStart(2)}`, + )}-${String(now.getHours()).padStart(2)}${String(now.getMinutes()).padStart(2)}`.replace(' ', '0'), dependencies: { 'wallet.ts': '1.0.1', 'bignumber.js': '9.0.1', From ea51b50c8a7d25f8a9046a910575aa6e4826a1fd Mon Sep 17 00:00:00 2001 From: Lantt Date: Wed, 22 Dec 2021 01:33:29 +0800 Subject: [PATCH 13/59] feat: add zerion fallback for get fungible token asset list --- packages/provider-proxy/build.mjs | 1 + .../src/producers/fungibleTokenAsset.ts | 12 +- packages/web3-providers/package.json | 1 + packages/web3-providers/src/debank/index.ts | 2 +- packages/web3-providers/src/zerion/format.ts | 88 +++++++++++ packages/web3-providers/src/zerion/index.ts | 133 ++++++++++++++++ packages/web3-providers/src/zerion/type.ts | 146 ++++++++++++++++++ pnpm-lock.yaml | 2 + 8 files changed, 380 insertions(+), 5 deletions(-) create mode 100644 packages/web3-providers/src/zerion/format.ts create mode 100644 packages/web3-providers/src/zerion/index.ts create mode 100644 packages/web3-providers/src/zerion/type.ts diff --git a/packages/provider-proxy/build.mjs b/packages/provider-proxy/build.mjs index 1bc411f39e0c..fa5af53e3aff 100644 --- a/packages/provider-proxy/build.mjs +++ b/packages/provider-proxy/build.mjs @@ -32,6 +32,7 @@ const packageJSON = { dependencies: { 'wallet.ts': '1.0.1', 'bignumber.js': '9.0.1', + 'socket.io-client': '2.4.0', }, main: './output.js', } diff --git a/packages/provider-proxy/src/producers/fungibleTokenAsset.ts b/packages/provider-proxy/src/producers/fungibleTokenAsset.ts index 41787e99d067..3df84798aa3a 100644 --- a/packages/provider-proxy/src/producers/fungibleTokenAsset.ts +++ b/packages/provider-proxy/src/producers/fungibleTokenAsset.ts @@ -1,5 +1,5 @@ import type { Web3Plugin } from '@masknet/plugin-infra' -import { getDebankAssetsList } from '@masknet/web3-providers' +import { getAssetListByZerion, getAssetListFromDebank } from '@masknet/web3-providers' export interface FungibleTokenAssetArgs extends ProducerArgBase { address: string @@ -12,11 +12,15 @@ const fungibleTokenAsset = async ( ): Promise => { const { address } = args try { - const data = await getDebankAssetsList(address) + const data = await getAssetListFromDebank(address) await push(data) } catch { - // await zerion - throw new Error('Fetch failed from debank') + try { + const data = await getAssetListByZerion(address) + await push(data) + } catch { + throw new Error('Fetch failed from debank and zerion') + } } } diff --git a/packages/web3-providers/package.json b/packages/web3-providers/package.json index 95db3e0d67ff..9e8298c30d5e 100644 --- a/packages/web3-providers/package.json +++ b/packages/web3-providers/package.json @@ -11,6 +11,7 @@ "date-fns": "^2.27.0", "json-stable-stringify": "^1.0.1", "lodash-unified": "1.0.1", + "socket.io-client": "2.4.0", "uuid": "^8.3.2", "urlcat": "^2.0.4" } diff --git a/packages/web3-providers/src/debank/index.ts b/packages/web3-providers/src/debank/index.ts index 3844ac23a647..84b653aa079a 100644 --- a/packages/web3-providers/src/debank/index.ts +++ b/packages/web3-providers/src/debank/index.ts @@ -11,7 +11,7 @@ export async function getTransactionList(address: string, chain: string) { return (await response.json()) as HistoryResponse } -export async function getDebankAssetsList(address: string) { +export async function getAssetListFromDebank(address: string) { const response = await fetch( `${DEBANK_OPEN_API}/v1/user/token_list?is_all=true&has_balance=true&id=${address.toLowerCase()}`, ) diff --git a/packages/web3-providers/src/zerion/format.ts b/packages/web3-providers/src/zerion/format.ts new file mode 100644 index 000000000000..6778e1a00ae2 --- /dev/null +++ b/packages/web3-providers/src/zerion/format.ts @@ -0,0 +1,88 @@ +import { leftShift, multipliedBy } from '@masknet/web3-shared-base' +import { ChainId, createLookupTableResolver, getTokenConstants, NetworkType } from '@masknet/web3-shared-evm' +import BigNumber from 'bignumber.js' +import type { + SocketRequestAssetScope, + ZerionAddressAsset, + ZerionAddressCovalentAsset, + ZerionAsset, + ZerionCovalentAsset, +} from './type' +import type { Web3Plugin } from '@masknet/plugin-infra' +import { TokenType } from '@masknet/plugin-infra' + +type Asset = Web3Plugin.Asset + +export const resolveZerionAssetsScopeName = createLookupTableResolver( + { + [NetworkType.Ethereum]: 'assets', + [NetworkType.Binance]: 'bsc-assets', + [NetworkType.Polygon]: 'polygon-assets', + [NetworkType.Arbitrum]: 'arbitrum-assets', + [NetworkType.xDai]: 'xdai-assets', + }, + '', +) + +export const resolveChainByScope = createLookupTableResolver< + SocketRequestAssetScope, + { + chain: string + chainId?: ChainId + } +>( + { + assets: { + chain: 'eth', + chainId: ChainId.Mainnet, + }, + 'bsc-assets': { + chain: 'bsc', + chainId: ChainId.BSC, + }, + 'polygon-assets': { + chain: 'matic', + chainId: ChainId.Matic, + }, + }, + { + chain: 'unknown', + }, +) + +export function formatAssets( + data: ZerionAddressAsset[] | ZerionAddressCovalentAsset[], + scope: SocketRequestAssetScope, +) { + return data.map(({ asset, quantity }) => { + const balance = leftShift(quantity, asset.decimals).toNumber() + const value = (asset as ZerionAsset).price?.value ?? (asset as ZerionCovalentAsset).value ?? 0 + const isNativeToken = (symbol: string) => ['ETH', 'BNB', 'MATIC', 'ARETH'].includes(symbol) + const address = isNativeToken(asset.symbol) ? getTokenConstants().NATIVE_TOKEN_ADDRESS : asset.asset_code + const chainId = resolveChainByScope(scope).chainId + + return { + id: address, + chainId: chainId, + token: { + id: address, + name: asset.name, + symbol: asset.symbol, + decimals: asset.decimals, + address: address, + chainId: chainId, + type: TokenType.Fungible, + logoURI: asset.icon_url, + }, + chain: resolveChainByScope(scope).chain, + balance: quantity, + price: { + usd: new BigNumber(value).toString(), + }, + value: { + usd: multipliedBy(balance, value).toString(), + }, + logoURI: asset.icon_url, + } + }) as Asset[] +} diff --git a/packages/web3-providers/src/zerion/index.ts b/packages/web3-providers/src/zerion/index.ts new file mode 100644 index 000000000000..24ff7426e76d --- /dev/null +++ b/packages/web3-providers/src/zerion/index.ts @@ -0,0 +1,133 @@ +import io from 'socket.io-client' +import type { + SocketRequestBody, + SocketNameSpace, + SocketResponseBody, + ZerionTransactionResponseBody, + ZerionAssetResponseBody, + ZerionAddressAsset, + ZerionAddressCovalentAsset, + SocketRequestAssetScope, +} from './type' +import { values } from 'lodash-unified' +import { createLookupTableResolver, NetworkType } from '@masknet/web3-shared-evm' +import { formatAssets } from './format' +import type { Web3Plugin } from '@masknet/plugin-infra' + +const ZERION_API = 'wss://api-v4.zerion.io' +const ZERION_TOKEN = 'Mask.yEUEfDnoxgLBwNEcYPVussxxjdrGwapj' + +let socket: SocketIOClient.Socket | null = null + +export const resolveZerionAssetsScopeName = createLookupTableResolver( + { + [NetworkType.Ethereum]: 'assets', + [NetworkType.Binance]: 'bsc-assets', + [NetworkType.Polygon]: 'polygon-assets', + [NetworkType.Arbitrum]: 'arbitrum-assets', + [NetworkType.xDai]: 'xdai-assets', + }, + '', +) + +function createSocket() { + if (socket?.connected) return socket + if (socket) socket.removeAllListeners() + return (socket = io(`${ZERION_API}/address`, { + transports: ['websocket'], + query: { + api_token: ZERION_TOKEN, + }, + // disable the auto reconnection + reconnection: false, + })) +} + +function verify(request: SocketRequestBody, response: any) { + // each value in request payload must be found in response meta + return Object.keys(request.payload).every((key) => { + const requestValue = request.payload[key] + const responseMetaValue = response.meta[key] + if (typeof requestValue === 'object') { + return JSON.stringify(requestValue) === JSON.stringify(responseMetaValue) + } + return responseMetaValue === requestValue + }) +} + +function subscribeFromZerion(socketNamespace: SocketNameSpace, requestBody: SocketRequestBody) { + return new Promise((resolve) => { + const { socket, namespace } = socketNamespace + const model = requestBody.scope[0] + socket.emit('subscribe', requestBody) + socket.on(`received ${namespace} ${model}`, (data: SocketResponseBody) => { + if (verify(requestBody, data)) { + resolve(data) + } + }) + }) +} + +export async function getTransactionListFromZerion(address: string, scope: string, page?: number, size = 30) { + return (await subscribeFromZerion( + { + namespace: 'address', + socket: createSocket(), + }, + { + scope: [scope], + payload: { + address, + currency: 'usd', + transactions_limit: size, + transactions_offset: (page ?? 0) * size, + transactions_search_query: '', + }, + }, + )) as ZerionTransactionResponseBody +} + +export async function getAssetsList(address: string, scope: string) { + return (await subscribeFromZerion( + { + namespace: 'address', + socket: createSocket(), + }, + { + scope: [scope], + payload: { + address, + currency: 'usd', + }, + }, + )) as ZerionAssetResponseBody +} + +const filterAssetType = ['compound', 'trash', 'uniswap', 'uniswap-v2', 'nft'] + +type Asset = Web3Plugin.Asset + +export async function getAssetListByZerion(address: string) { + let result: Asset[] = [] + const scopes = ['assets', 'bsc-assets', 'polygon-assets', 'arbitrum-assets'] + for (const scope of scopes) { + const { meta, payload } = await getAssetsList(address, scope) + if (meta.status !== 'ok') throw new Error('Fail to load assets.') + + const assets = Object.entries(payload).map(([key, value]) => { + if (key === 'assets') { + const assetsList = (values(value) as ZerionAddressAsset[]).filter( + ({ asset }) => + asset.is_displayable && !filterAssetType.some((type) => type === asset.type) && asset.icon_url, + ) + return formatAssets(assetsList, key) + } + + return formatAssets(values(value) as ZerionAddressCovalentAsset[], key as SocketRequestAssetScope) + }) + + result = [...result, ...assets.flat()] + } + + return result +} diff --git a/packages/web3-providers/src/zerion/type.ts b/packages/web3-providers/src/zerion/type.ts new file mode 100644 index 000000000000..d39248b20e6e --- /dev/null +++ b/packages/web3-providers/src/zerion/type.ts @@ -0,0 +1,146 @@ +export type SocketNameSpace = { + namespace: string + socket: SocketIOClient.Socket +} + +export type SocketRequestAssetScope = 'assets' | 'bsc-assets' | 'polygon-assets' + +export type SocketRequestBody = { + scope: string[] + payload: Record +} + +export interface SocketResponseBody { + meta: { status: string } + payload: Record +} + +export enum ZerionRBDTransactionType { + SEND = 'send', + RECEIVE = 'receive', + TRADE = 'trade', + AUTHORIZE = 'authorize', + EXECUTION = 'execution', + DEPLOYMENT = 'deployment', + CANCEL = 'cancel', + DEPOSIT = 'deposit', + WITHDRAW = 'withdraw', + BORROW = 'borrow', + REPAY = 'repay', + STAKE = 'stake', + UNSTAKE = 'unstake', + CLAIM = 'claim', +} + +export enum ZerionTransactionStatus { + CONFIRMED = 'confirmed', + FAILED = 'failed', + PENDING = 'pending', +} + +export enum ZerionTransactionDirection { + IN = 'in', + OUT = 'out', + SELF = 'self', +} + +export interface ZerionPrice { + value: number + changed_at: number + relative_change_24h?: number +} + +export interface ZerionAsset { + asset_code: string + name: string + symbol: string + decimals: number + type: string + is_displayable: boolean + is_verified: boolean + icon_url?: string + price?: ZerionPrice +} + +export interface ZerionCovalentAsset { + asset_code: string + name: string + symbol: string + decimals: number + icon_url?: string + value: number +} + +export interface ZerionAddressAsset { + asset: ZerionAsset + quantity: string +} + +export interface ZerionAddressCovalentAsset { + asset: ZerionCovalentAsset + quantity: string +} + +export interface ZerionTransactionChange { + asset: ZerionAsset + value: number + direction: ZerionTransactionDirection + address_from: string + address_to: string + price?: number +} + +export interface ZerionTransactionFee { + value: number + price: number +} + +export interface ZerionTransactionItem { + id: string + type: ZerionRBDTransactionType + protocol: string + mined_at: number + block_number: number + status: ZerionTransactionStatus + hash: string + direction?: ZerionTransactionDirection + address_from?: string + address_to?: string + contract?: string + nonce?: number + changes?: ZerionTransactionChange[] + fee?: ZerionTransactionFee + meta?: string +} + +export interface ZerionTransactionResponseBody extends SocketResponseBody { + meta: { + status: string + address: string + addresses: string[] + currency: string + transactions_limit: number + transactions_offset: number + transactions_search_query: string + } + payload: { + transactions: ZerionTransactionItem[] + } +} + +type ZerionAssetResponseBodyPayload = { + assets: Record + 'bsc-assets': Record + 'polygon-assets': Record +} + +export interface ZerionAssetResponseBody extends SocketResponseBody { + meta: { + address: string + addresses: string[] + asset_codes: string[] + currency: string + status: string + } + payload: Partial +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e68beec2b560..d1c7d6cb2bee 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -972,6 +972,7 @@ importers: date-fns: ^2.27.0 json-stable-stringify: ^1.0.1 lodash-unified: 1.0.1 + socket.io-client: 2.4.0 urlcat: ^2.0.4 dependencies: '@masknet/plugin-infra': link:../plugin-infra @@ -981,6 +982,7 @@ importers: date-fns: 2.27.0 json-stable-stringify: 1.0.1 lodash-unified: 1.0.1_bd4845f8c1bf1d81a616b60758034d0a + socket.io-client: 2.4.0 urlcat: 2.0.4 packages/web3-shared/base: From b4833082150b8612bedea286a33b2dbb405d91bb Mon Sep 17 00:00:00 2001 From: Lantt Date: Wed, 22 Dec 2021 01:37:08 +0800 Subject: [PATCH 14/59] fix: update deploy ci --- .github/workflows/deploy-proxy.yml | 20 +++++--------------- packages/provider-proxy/package.json | 1 - 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/.github/workflows/deploy-proxy.yml b/.github/workflows/deploy-proxy.yml index 00aa3bb2167e..40c88ba2aa53 100644 --- a/.github/workflows/deploy-proxy.yml +++ b/.github/workflows/deploy-proxy.yml @@ -7,6 +7,9 @@ on: jobs: build: runs-on: ubuntu-20.04 + permissions: + packages: write + contents: read steps: - name: Get cache date id: get-date @@ -31,23 +34,10 @@ jobs: restore-keys: ${{ runner.os }}-extension-${{ hashFiles('pnpm-lock.yaml') }} - run: pnpm install - run: pnpm build - - name: Run Install + - name: Run provider-proxy package build working-directory: ./packages/provider-proxy run: pnpm install && pnpm build - publish-gpr: - needs: build - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: 16 - registry-url: https://npm.pkg.github.com/ - name: Publish to github npm pkg - working-directory: ./packages/provider-proxy/dist - run: npm publish --scope=@dimensiondev --registry=https://npm.pkg.github.com + run: npm publish ./packages/provider-proxy/dist env: NODE_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/packages/provider-proxy/package.json b/packages/provider-proxy/package.json index a99b917838df..9030a1864afd 100644 --- a/packages/provider-proxy/package.json +++ b/packages/provider-proxy/package.json @@ -1,7 +1,6 @@ { "name": "@masknet/provider-proxy", "version": "0.0.1", - "private": true, "main": "./dist/index.js", "types": "./dist/index.d.ts", "scripts": { From eb08e76fa009adc2c29a1e1a14109dc244967625 Mon Sep 17 00:00:00 2001 From: Lantt Date: Wed, 22 Dec 2021 11:35:02 +0800 Subject: [PATCH 15/59] fix: use dimension dev token action to publish --- .github/workflows/deploy-proxy.yml | 5 +++-- packages/provider-proxy/package.json | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-proxy.yml b/.github/workflows/deploy-proxy.yml index 40c88ba2aa53..8f0e62ee4678 100644 --- a/.github/workflows/deploy-proxy.yml +++ b/.github/workflows/deploy-proxy.yml @@ -37,7 +37,8 @@ jobs: - name: Run provider-proxy package build working-directory: ./packages/provider-proxy run: pnpm install && pnpm build + - uses: DimensionDev/github-token-action@latest + with: + registry: true - name: Publish to github npm pkg run: npm publish ./packages/provider-proxy/dist - env: - NODE_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/packages/provider-proxy/package.json b/packages/provider-proxy/package.json index 9030a1864afd..a99b917838df 100644 --- a/packages/provider-proxy/package.json +++ b/packages/provider-proxy/package.json @@ -1,6 +1,7 @@ { "name": "@masknet/provider-proxy", "version": "0.0.1", + "private": true, "main": "./dist/index.js", "types": "./dist/index.d.ts", "scripts": { From f5f977f3834a64b9cc963f9b39d72494646954ee Mon Sep 17 00:00:00 2001 From: Lantt Date: Wed, 22 Dec 2021 13:11:41 +0800 Subject: [PATCH 16/59] feat: update package to publish --- packages/provider-proxy/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/provider-proxy/package.json b/packages/provider-proxy/package.json index a99b917838df..37888897fa8a 100644 --- a/packages/provider-proxy/package.json +++ b/packages/provider-proxy/package.json @@ -1,7 +1,6 @@ { - "name": "@masknet/provider-proxy", + "name": "@dimensiondev/provider-proxy", "version": "0.0.1", - "private": true, "main": "./dist/index.js", "types": "./dist/index.d.ts", "scripts": { From 53b5bbdc3807e72d4a900d6af39ac2ea1b9ee966 Mon Sep 17 00:00:00 2001 From: Lantt Date: Wed, 22 Dec 2021 15:49:44 +0800 Subject: [PATCH 17/59] feat: generate ts type --- packages/provider-proxy/build.mjs | 6 +++++- packages/provider-proxy/package.json | 3 +-- packages/provider-proxy/rollup.config.js | 11 ++++++++++- packages/provider-proxy/src/index.ts | 6 ++---- packages/provider-proxy/src/producer.d.ts | 11 ----------- .../src/producers/fungibleTokenAsset.ts | 4 +++- packages/provider-proxy/src/typs.ts | 12 ++++++++++++ packages/provider-proxy/tsconfig.json | 6 +++++- 8 files changed, 38 insertions(+), 21 deletions(-) delete mode 100644 packages/provider-proxy/src/producer.d.ts create mode 100644 packages/provider-proxy/src/typs.ts diff --git a/packages/provider-proxy/build.mjs b/packages/provider-proxy/build.mjs index fa5af53e3aff..69ff90b4634e 100644 --- a/packages/provider-proxy/build.mjs +++ b/packages/provider-proxy/build.mjs @@ -1,6 +1,6 @@ // @ts-check import { spawn } from 'child_process' -import { readFile, writeFile } from 'fs/promises' +import { readFile, writeFile, unlink, rmdir } from 'fs/promises' import { join, resolve } from 'path' import { fileURLToPath } from 'url' @@ -35,6 +35,7 @@ const packageJSON = { 'socket.io-client': '2.4.0', }, main: './output.js', + types: './index.d.ts', } await writeFile(join(__dirname, './dist/package.json'), JSON.stringify(packageJSON, undefined, 4)) @@ -54,3 +55,6 @@ await awaitChildProcess( stdio: 'inherit', }), ) + +await unlink(join(__dirname, './dist/index.ts')) +await rmdir(join(__dirname, './dist/producers'), { recursive: true, force: true }) diff --git a/packages/provider-proxy/package.json b/packages/provider-proxy/package.json index 37888897fa8a..7897a67f703f 100644 --- a/packages/provider-proxy/package.json +++ b/packages/provider-proxy/package.json @@ -1,8 +1,6 @@ { "name": "@dimensiondev/provider-proxy", "version": "0.0.1", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", "scripts": { "build:rollup": "rollup -c", "build": "node ./build.mjs" @@ -20,6 +18,7 @@ "@rollup/plugin-json": "^4.1.0", "@rollup/plugin-node-resolve": "^13.0.6", "@rollup/plugin-sucrase": "^4.0.1", + "@rollup/plugin-typescript": "^8.3.0", "rollup": "^2.60.1" } } diff --git a/packages/provider-proxy/rollup.config.js b/packages/provider-proxy/rollup.config.js index 0aff6ee8d4d2..edf168b8b60c 100644 --- a/packages/provider-proxy/rollup.config.js +++ b/packages/provider-proxy/rollup.config.js @@ -3,8 +3,15 @@ import node from '@rollup/plugin-node-resolve' import sucrase from '@rollup/plugin-sucrase' import json from '@rollup/plugin-json' import alias from '@rollup/plugin-alias' +import typescript from '@rollup/plugin-typescript' -export default { +const typesBuildConfig = { + input: './src/index.ts', + output: [{ file: './dist/index.ts', format: 'cjs' }], + plugins: [typescript({ tsconfig: './tsconfig.json' })], +} + +const config = { input: './src/index.ts', output: { file: 'dist/output.js', @@ -30,3 +37,5 @@ export default { }, treeshake: { moduleSideEffects: false, propertyReadSideEffects: false }, } + +export default [config, typesBuildConfig] diff --git a/packages/provider-proxy/src/index.ts b/packages/provider-proxy/src/index.ts index 5371f1280cc9..578df134858a 100644 --- a/packages/provider-proxy/src/index.ts +++ b/packages/provider-proxy/src/index.ts @@ -1,8 +1,6 @@ -/// import fungibleTokenProducer from './producers/fungibleTokenAsset' +import type { RpcMethodRegistrationValue } from './typs' // TODO: unit test -export const producers = { - fungibleTokenAsset: fungibleTokenProducer, -} +export const producers: RpcMethodRegistrationValue[] = [fungibleTokenProducer] diff --git a/packages/provider-proxy/src/producer.d.ts b/packages/provider-proxy/src/producer.d.ts deleted file mode 100644 index b0515a350bda..000000000000 --- a/packages/provider-proxy/src/producer.d.ts +++ /dev/null @@ -1,11 +0,0 @@ -type ProducerPushFunction = (item: T[]) => Promise -type ProducerKeyFunction = (provider: string) => Promise - -interface ProducerArgBase { - size: number -} - -interface RpcMethodRegistrationValue { - producer(push: ProducerPushFunction, getKey: ProducerKeyFunction, args: TArgs): Promise - keyHasher(item: T): string -} diff --git a/packages/provider-proxy/src/producers/fungibleTokenAsset.ts b/packages/provider-proxy/src/producers/fungibleTokenAsset.ts index 3df84798aa3a..24cc758876eb 100644 --- a/packages/provider-proxy/src/producers/fungibleTokenAsset.ts +++ b/packages/provider-proxy/src/producers/fungibleTokenAsset.ts @@ -1,5 +1,6 @@ import type { Web3Plugin } from '@masknet/plugin-infra' import { getAssetListByZerion, getAssetListFromDebank } from '@masknet/web3-providers' +import type { ProducerArgBase, ProducerKeyFunction, ProducerPushFunction, RpcMethodRegistrationValue } from '../typs' export interface FungibleTokenAssetArgs extends ProducerArgBase { address: string @@ -25,8 +26,9 @@ const fungibleTokenAsset = async ( } const producer: RpcMethodRegistrationValue, FungibleTokenAssetArgs> = { + method: 'mask.fetchFungibleTokenAsset', producer: fungibleTokenAsset, - keyHasher: (item) => `${item.id}_${item.chainId}`, + distinctBy: (item) => `${item.id}_${item.chainId}`, } export default producer diff --git a/packages/provider-proxy/src/typs.ts b/packages/provider-proxy/src/typs.ts new file mode 100644 index 000000000000..d9cc4553c8dc --- /dev/null +++ b/packages/provider-proxy/src/typs.ts @@ -0,0 +1,12 @@ +export type ProducerPushFunction = (item: T[]) => Promise +export type ProducerKeyFunction = (provider: string) => Promise + +export interface ProducerArgBase { + size: number +} + +export interface RpcMethodRegistrationValue { + method: string + producer(push: ProducerPushFunction, getKey: ProducerKeyFunction, args: TArgs): Promise + distinctBy(item: T): string +} diff --git a/packages/provider-proxy/tsconfig.json b/packages/provider-proxy/tsconfig.json index 8d5e01442c2f..fc39e0ab9839 100644 --- a/packages/provider-proxy/tsconfig.json +++ b/packages/provider-proxy/tsconfig.json @@ -2,8 +2,12 @@ "extends": "../../tsconfig.json", "compilerOptions": { "rootDir": "./src/", - "outDir": "./dist/" + "outDir": "./dist/", + "declaration": true, + "declarationDir": ".", + "emitDeclarationOnly": true }, + "exclude": ["dist"], "include": ["./src", "./src/**/*.json"], "references": [ { "path": "../plugin-infra" }, From a210150cae196578522842105e91508a348e2b01 Mon Sep 17 00:00:00 2001 From: Lantt Date: Wed, 22 Dec 2021 15:52:28 +0800 Subject: [PATCH 18/59] feat: change ci --- .github/workflows/deploy-proxy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-proxy.yml b/.github/workflows/deploy-proxy.yml index 8f0e62ee4678..c9c01b70bcb0 100644 --- a/.github/workflows/deploy-proxy.yml +++ b/.github/workflows/deploy-proxy.yml @@ -41,4 +41,5 @@ jobs: with: registry: true - name: Publish to github npm pkg - run: npm publish ./packages/provider-proxy/dist + run: pnpm publish --no-git-checks + working-directory: packages/provider-proxy From 05e264d2cabcfcd47ac95d9551c4b4f93ff3402f Mon Sep 17 00:00:00 2001 From: Jack Works Date: Wed, 22 Dec 2021 23:26:55 +0800 Subject: [PATCH 19/59] feat: support dts rollup --- packages/provider-proxy/build.mjs | 64 +++++++++++----------- packages/provider-proxy/package.json | 4 +- packages/provider-proxy/rollup.config.js | 67 +++++++++++++----------- packages/provider-proxy/tsconfig.json | 4 +- tsconfig.json | 1 + 5 files changed, 71 insertions(+), 69 deletions(-) diff --git a/packages/provider-proxy/build.mjs b/packages/provider-proxy/build.mjs index 69ff90b4634e..cc5f576dd6b9 100644 --- a/packages/provider-proxy/build.mjs +++ b/packages/provider-proxy/build.mjs @@ -8,53 +8,51 @@ const __file = fileURLToPath(import.meta.url) const __dirname = resolve(__file, '../') function awaitChildProcess(child) { - return new Promise((resolve, reject) => { - child.on('error', () => reject(child.exitCode || 0)) - child.on('exit', (code) => resolve(code || 0)) - }) + return new Promise((resolve, reject) => { + child.on('error', () => reject(child.exitCode || 0)) + child.on('exit', (code) => resolve(code || 0)) + }) } const rollup = awaitChildProcess( - spawn('pnpm run build:rollup', { - cwd: __dirname, - shell: true, - stdio: 'inherit', - }), + spawn('pnpm run build:rollup', { + cwd: __dirname, + shell: true, + stdio: 'inherit', + }), ) await rollup const version = JSON.parse(await readFile(join(__dirname, './package.json'), 'utf-8')).version const now = new Date() const packageJSON = { - name: '@dimensiondev/provider-proxy', - version: `${version}-${now.getFullYear()}${String(now.getMonth() + 1).padStart(2)}${String(now.getDate()).padStart( - 2, - )}-${String(now.getHours()).padStart(2)}${String(now.getMinutes()).padStart(2)}`.replace(' ', '0'), - dependencies: { - 'wallet.ts': '1.0.1', - 'bignumber.js': '9.0.1', - 'socket.io-client': '2.4.0', - }, - main: './output.js', - types: './index.d.ts', + name: '@dimensiondev/provider-proxy', + version: `${version}-${now.getFullYear()}${String(now.getMonth() + 1).padStart(2)}${String(now.getDate()).padStart( + 2, + )}-${String(now.getHours()).padStart(2)}${String(now.getMinutes()).padStart(2)}`.replace(' ', '0'), + dependencies: { + 'wallet.ts': '1.0.1', + 'bignumber.js': '9.0.1', + 'socket.io-client': '2.4.0', + }, + main: './output.js', + types: './output.d.ts', + files: ['output.js', 'output.d.ts'], } await writeFile(join(__dirname, './dist/package.json'), JSON.stringify(packageJSON, undefined, 4)) const out = join(__dirname, './dist') // validate if there is dependency missing await awaitChildProcess( - spawn('npm install', { - cwd: out, - shell: true, - stdio: 'inherit', - }), + spawn('npm install', { + cwd: out, + shell: true, + stdio: 'inherit', + }), ) await awaitChildProcess( - spawn('node output.js', { - cwd: out, - shell: true, - stdio: 'inherit', - }), + spawn('node output.js', { + cwd: out, + shell: true, + stdio: 'inherit', + }), ) - -await unlink(join(__dirname, './dist/index.ts')) -await rmdir(join(__dirname, './dist/producers'), { recursive: true, force: true }) diff --git a/packages/provider-proxy/package.json b/packages/provider-proxy/package.json index 7897a67f703f..f0d10b6168c3 100644 --- a/packages/provider-proxy/package.json +++ b/packages/provider-proxy/package.json @@ -18,7 +18,7 @@ "@rollup/plugin-json": "^4.1.0", "@rollup/plugin-node-resolve": "^13.0.6", "@rollup/plugin-sucrase": "^4.0.1", - "@rollup/plugin-typescript": "^8.3.0", - "rollup": "^2.60.1" + "rollup": "^2.60.1", + "rollup-plugin-dts": "4.0.1" } } diff --git a/packages/provider-proxy/rollup.config.js b/packages/provider-proxy/rollup.config.js index edf168b8b60c..6c63b63337f4 100644 --- a/packages/provider-proxy/rollup.config.js +++ b/packages/provider-proxy/rollup.config.js @@ -3,39 +3,44 @@ import node from '@rollup/plugin-node-resolve' import sucrase from '@rollup/plugin-sucrase' import json from '@rollup/plugin-json' import alias from '@rollup/plugin-alias' -import typescript from '@rollup/plugin-typescript' +import dts from 'rollup-plugin-dts' -const typesBuildConfig = { - input: './src/index.ts', - output: [{ file: './dist/index.ts', format: 'cjs' }], - plugins: [typescript({ tsconfig: './tsconfig.json' })], +const config = { + input: './src/index.ts', + output: { + file: 'dist/output.js', + format: 'cjs', + }, + plugins: [ + commonjs(), + node(), + json(), + alias({ + entries: [ + { find: '@masknet/web3-shared-evm', replacement: '../web3-shared/evm/index.ts' }, + { find: '@masknet/web3-shared-base', replacement: '../web3-shared/base/index.ts' }, + ], + }), + sucrase({ transforms: ['typescript', 'jsx'] }), + ], + external: (id) => { + if (id.startsWith('.')) return false + if (id.includes('@masknet')) return false + if (id.includes('node_modules')) return true + return false + }, + treeshake: { moduleSideEffects: false, propertyReadSideEffects: false }, } -const config = { - input: './src/index.ts', - output: { - file: 'dist/output.js', - format: 'cjs', - }, - plugins: [ - commonjs(), - node(), - json(), - alias({ - entries: [ - { find: '@masknet/web3-shared-evm', replacement: '../web3-shared/evm/index.ts' }, - { find: '@masknet/web3-shared-base', replacement: '../web3-shared/base/index.ts' }, - ], - }), - sucrase({ transforms: ['typescript', 'jsx'] }), - ], - external: (id) => { - if (id.startsWith('.')) return false - if (id.includes('@masknet')) return false - if (id.includes('node_modules')) return true - return false - }, - treeshake: { moduleSideEffects: false, propertyReadSideEffects: false }, +const dtsConfig = { + input: './dist/index.d.ts', + output: [{ file: './dist/output.d.ts', format: 'es' }], + plugins: [ + dts({ + respectExternal: true, + }), + ], + external: config.external, } -export default [config, typesBuildConfig] +export default [config, dtsConfig] diff --git a/packages/provider-proxy/tsconfig.json b/packages/provider-proxy/tsconfig.json index fc39e0ab9839..fd8ba048eac1 100644 --- a/packages/provider-proxy/tsconfig.json +++ b/packages/provider-proxy/tsconfig.json @@ -3,11 +3,9 @@ "compilerOptions": { "rootDir": "./src/", "outDir": "./dist/", - "declaration": true, - "declarationDir": ".", + "tsBuildInfoFile": "./dist/.tsbuildinfo", "emitDeclarationOnly": true }, - "exclude": ["dist"], "include": ["./src", "./src/**/*.json"], "references": [ { "path": "../plugin-infra" }, diff --git a/tsconfig.json b/tsconfig.json index 97551e4c7172..791c7ec99e97 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,6 +7,7 @@ { "path": "./packages/mask-sdk/main" }, { "path": "./packages/mask-sdk/public-api" }, { "path": "./packages/dashboard" }, + { "path": "./packages/provider-proxy" }, // Storybooks are considered as a "project" not depended by any other project { "path": "./packages/dashboard/stories" }, { "path": "./packages/theme/stories" }, From 6efeb7dd359b22c3cc50cd2128c123418f31851a Mon Sep 17 00:00:00 2001 From: Lantt Date: Fri, 24 Dec 2021 10:03:03 +0800 Subject: [PATCH 20/59] feat: disable get asset from zerion --- packages/provider-proxy/build.mjs | 3 ++- .../src/producers/fungibleTokenAsset.ts | 17 +++++------------ packages/web3-providers/src/debank/format.ts | 5 ++--- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/packages/provider-proxy/build.mjs b/packages/provider-proxy/build.mjs index cc5f576dd6b9..d5b5e9f01e9c 100644 --- a/packages/provider-proxy/build.mjs +++ b/packages/provider-proxy/build.mjs @@ -30,9 +30,10 @@ const packageJSON = { 2, )}-${String(now.getHours()).padStart(2)}${String(now.getMinutes()).padStart(2)}`.replace(' ', '0'), dependencies: { - 'wallet.ts': '1.0.1', + // 'wallet.ts': '1.0.1', 'bignumber.js': '9.0.1', 'socket.io-client': '2.4.0', + 'lodash-unified': '1.0.1', }, main: './output.js', types: './output.d.ts', diff --git a/packages/provider-proxy/src/producers/fungibleTokenAsset.ts b/packages/provider-proxy/src/producers/fungibleTokenAsset.ts index 24cc758876eb..c7f67d23c9ae 100644 --- a/packages/provider-proxy/src/producers/fungibleTokenAsset.ts +++ b/packages/provider-proxy/src/producers/fungibleTokenAsset.ts @@ -1,5 +1,5 @@ import type { Web3Plugin } from '@masknet/plugin-infra' -import { getAssetListByZerion, getAssetListFromDebank } from '@masknet/web3-providers' +import { getAssetListFromDebank } from '@masknet/web3-providers' import type { ProducerArgBase, ProducerKeyFunction, ProducerPushFunction, RpcMethodRegistrationValue } from '../typs' export interface FungibleTokenAssetArgs extends ProducerArgBase { @@ -12,17 +12,10 @@ const fungibleTokenAsset = async ( args: FungibleTokenAssetArgs, ): Promise => { const { address } = args - try { - const data = await getAssetListFromDebank(address) - await push(data) - } catch { - try { - const data = await getAssetListByZerion(address) - await push(data) - } catch { - throw new Error('Fetch failed from debank and zerion') - } - } + const data = await getAssetListFromDebank(address) + await push(data) + // const data = await getAssetListByZerion(address) + // await push(data) } const producer: RpcMethodRegistrationValue, FungibleTokenAssetArgs> = { diff --git a/packages/web3-providers/src/debank/format.ts b/packages/web3-providers/src/debank/format.ts index c7abbed22552..aa8cda398f62 100644 --- a/packages/web3-providers/src/debank/format.ts +++ b/packages/web3-providers/src/debank/format.ts @@ -1,6 +1,6 @@ import BigNumber from 'bignumber.js' import type { WalletTokenRecord } from './type' -import { ChainId, formatEthereumAddress, getChainIdFromName } from '@masknet/web3-shared-evm' +import { ChainId, getChainIdFromName } from '@masknet/web3-shared-evm' import { CurrencyType, TokenType, Web3Plugin } from '@masknet/plugin-infra' import { multipliedBy, rightShift } from '@masknet/web3-shared-base' @@ -11,8 +11,7 @@ export function formatAssets(data: WalletTokenRecord[]): Asset[] { .filter((x) => x.is_verified) .map((y): Asset => { const chainIdFromChain = getChainIdFromName(y.chain) ?? ChainId.Mainnet - const chainIdFromId = getChainIdFromName(y.id) - const address = y.id === 'eth' ? '' : formatEthereumAddress(y.id) + const address = y.id === 'eth' ? '' : y.id return { id: address, From 3a14d4b9eba97e66e876be232adaefffd9f7fef1 Mon Sep 17 00:00:00 2001 From: Lantt Date: Fri, 24 Dec 2021 10:14:54 +0800 Subject: [PATCH 21/59] feat: add fetch non fungible token from opensea --- packages/provider-proxy/src/index.ts | 1 + .../src/producers/nonFungibleTokenAsset.ts | 32 +++++ packages/web3-providers/src/opensea/format.ts | 50 ++++++++ packages/web3-providers/src/opensea/index.ts | 12 ++ packages/web3-providers/src/opensea/type.ts | 118 ++++++++++++++++++ 5 files changed, 213 insertions(+) create mode 100644 packages/provider-proxy/src/producers/nonFungibleTokenAsset.ts create mode 100644 packages/web3-providers/src/opensea/format.ts create mode 100644 packages/web3-providers/src/opensea/type.ts diff --git a/packages/provider-proxy/src/index.ts b/packages/provider-proxy/src/index.ts index 578df134858a..287417c7e091 100644 --- a/packages/provider-proxy/src/index.ts +++ b/packages/provider-proxy/src/index.ts @@ -1,3 +1,4 @@ +/// import fungibleTokenProducer from './producers/fungibleTokenAsset' import type { RpcMethodRegistrationValue } from './typs' diff --git a/packages/provider-proxy/src/producers/nonFungibleTokenAsset.ts b/packages/provider-proxy/src/producers/nonFungibleTokenAsset.ts new file mode 100644 index 000000000000..452b4a67b0e2 --- /dev/null +++ b/packages/provider-proxy/src/producers/nonFungibleTokenAsset.ts @@ -0,0 +1,32 @@ +import { getOpenSeaNFTList } from '@masknet/web3-providers' +import type { ERC721TokenDetailed } from '@masknet/web3-shared-evm' +import type { ProducerArgBase, ProducerKeyFunction, ProducerPushFunction, RpcMethodRegistrationValue } from '../typs' + +export interface NonFungibleTokenAssetArgs extends ProducerArgBase { + address: string +} + +const nonFungibleTokenAsset = async ( + push: ProducerPushFunction, + getKeys: ProducerKeyFunction, + args: NonFungibleTokenAssetArgs, +): Promise => { + const { address } = args + // todo: key word + const openSeaApiKey = await getKeys('OpenSea') + + let openSeaHasNextPage = true + while (openSeaHasNextPage) { + const dataFromOpenSea = await getOpenSeaNFTList(openSeaApiKey, address) + await push(dataFromOpenSea.assets) + openSeaHasNextPage = !!dataFromOpenSea.hasNextPage + } +} + +const producer: RpcMethodRegistrationValue = { + method: 'mask.fetchNonFungibleTokenAsset', + producer: nonFungibleTokenAsset, + distinctBy: (item) => `${item.tokenId}_${item.contractDetailed.address}`, +} + +export default producer diff --git a/packages/web3-providers/src/opensea/format.ts b/packages/web3-providers/src/opensea/format.ts new file mode 100644 index 000000000000..df376634f699 --- /dev/null +++ b/packages/web3-providers/src/opensea/format.ts @@ -0,0 +1,50 @@ +import { + ChainId, + ERC721ContractDetailed, + ERC721TokenDetailed, + ERC721TokenInfo, + EthereumTokenType, +} from '@masknet/web3-shared-evm' +import type { Asset } from './type' + +export function createERC721Token( + contractDetailed: ERC721ContractDetailed, + info: ERC721TokenInfo, + tokenId: string, +): ERC721TokenDetailed { + return { + contractDetailed, + info, + tokenId, + } +} + +export function format(address: string, size: number, assets: Asset[]) { + return { + assets: assets + .filter( + (x) => + ['non-fungible', 'semi-fungible'].includes(x.asset_contract.asset_contract_type) || + ['ERC721', 'ERC1155'].includes(x.asset_contract.schema_name), + ) + .map((x) => + createERC721Token( + { + chainId: ChainId.Mainnet, + type: EthereumTokenType.ERC721, + name: x.asset_contract.name, + symbol: x.asset_contract.symbol, + address: x.asset_contract.address, + }, + { + owner: address, + name: x.name || x.asset_contract.name, + description: x.description || x.asset_contract.symbol, + mediaUrl: x.image_url || x.image_preview_url || x.asset_contract.image_url || '', + }, + x.token_id, + ), + ), + hasNextPage: assets.length === size, + } +} diff --git a/packages/web3-providers/src/opensea/index.ts b/packages/web3-providers/src/opensea/index.ts index 28dce5da7804..eb57f71db747 100644 --- a/packages/web3-providers/src/opensea/index.ts +++ b/packages/web3-providers/src/opensea/index.ts @@ -339,3 +339,15 @@ export class OpenSeaAPI implements NonFungibleTokenAPI.Provider { } } } + +export async function getOpenSeaNFTList( + apiKey: string, + address: string, + page?: number, + size?: number, + collection?: string, +) { + const { assets } = await getAssetsListFromOpenSea(address, { page, size, collection }, apiKey) + return format(address, size ?? 50, assets) +} + diff --git a/packages/web3-providers/src/opensea/type.ts b/packages/web3-providers/src/opensea/type.ts new file mode 100644 index 000000000000..c8b86996bc99 --- /dev/null +++ b/packages/web3-providers/src/opensea/type.ts @@ -0,0 +1,118 @@ +interface AssetContract { + address: string + asset_contract_type: string + created_date: string + name: string + nft_version: string + opensea_version: unknown | null + owner: unknown | null + schema_name: 'ERC721' | 'ERC1155' + symbol: string + total_supply: string + description: string | null + external_link: string | null + image_url: string | null + default_to_fiat: boolean + dev_buyer_fee_basis_points: number + dev_seller_fee_basis_points: number + only_proxied_transfers: boolean + opensea_buyer_fee_basis_points: number + opensea_seller_fee_basis_points: number + buyer_fee_basis_points: number + seller_fee_basis_points: number + payout_address: string | null +} + +interface AssetOwner { + user: { + username: string | null + } + profile_img_url: string | null + address: string | null + config: string | null + discord_id: string | null +} + +interface AssetCollection { + banner_image_url: string | null + chat_url: string | null + created_date: string + default_to_fiat: boolean + description: string | null + dev_buyer_fee_basis_points: string + dev_seller_fee_basis_points: string + discord_url: string | null + display_data: { + card_display_style: 'contain' | 'padded' + images: unknown[] + } + external_url: string | null + featured: boolean + featured_image_url: string | null + hidden: boolean + safelist_request_status: 'not_requested' | 'verified' + image_url: string | null + is_subject_to_whitelist: boolean + large_image_url: string | null + medium_username: string | null + name: string + only_proxied_transfers: boolean + opensea_buyer_fee_basis_points: string + opensea_seller_fee_basis_points: string + payout_address: string | null + require_email: boolean + short_description: string | null + slug: string + telegram_url: string | null + twitter_username: string | null + instagram_username: string | null + wiki_url: string | null +} + +interface AssetCreator { + user: { + username: string | null + } + profile_img_url: string + address: string + config: string + discord_id: string +} + +export interface Asset { + id: number + token_id: string + num_sales: number + background_color: string | null + image_url: string | null + image_preview_url: string | null + image_thumbnail_url: string | null + image_original_url: string | null + animation_url: string | null + animation_original_url: string | null + name: string + description: string + external_link: string + asset_contract: AssetContract + owner: AssetOwner + permalink: string + collection: AssetCollection + decimals: number + sell_orders: unknown[] + creator: AssetCreator + traits: unknown[] + last_sale: unknown | null + top_bid: unknown | null + listing_date: unknown | null + is_presale: boolean + transfer_fee_payment_token: unknown | null + transfer_fee: unknown | null +} + +export interface AssetsListResponse { + assets: Asset[] +} + +export interface CollectionsResponse { + collections: AssetCollection[] +} From ac93b2011677ef2ad0342dec24ada4b2a96a700d Mon Sep 17 00:00:00 2001 From: Lantt Date: Fri, 24 Dec 2021 12:12:06 +0800 Subject: [PATCH 22/59] feat: add fetch non fungible collectible from opensea --- packages/provider-proxy/src/helper/request.ts | 16 +++++ packages/provider-proxy/src/index.ts | 10 +++- .../producers/nonFungibleCollectibleAsset.ts | 29 +++++++++ .../src/producers/nonFungibleTokenAsset.ts | 2 +- packages/web3-providers/src/opensea/index.ts | 59 +++++++++++++++++++ packages/web3-providers/src/opensea/type.ts | 2 +- 6 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 packages/provider-proxy/src/helper/request.ts create mode 100644 packages/provider-proxy/src/producers/nonFungibleCollectibleAsset.ts diff --git a/packages/provider-proxy/src/helper/request.ts b/packages/provider-proxy/src/helper/request.ts new file mode 100644 index 000000000000..1191655aefc9 --- /dev/null +++ b/packages/provider-proxy/src/helper/request.ts @@ -0,0 +1,16 @@ +interface Response { + data: T[] + hasNextPage?: boolean +} + +export async function collectAllPageDate(fetcher: () => Promise>) { + let data: T[] = [] + let hasNextPage = true + while (hasNextPage) { + const result = await fetcher() + data = [...data, ...result.data] + hasNextPage = !!result.hasNextPage + } + + return data +} diff --git a/packages/provider-proxy/src/index.ts b/packages/provider-proxy/src/index.ts index 287417c7e091..299d8acd55b6 100644 --- a/packages/provider-proxy/src/index.ts +++ b/packages/provider-proxy/src/index.ts @@ -1,7 +1,11 @@ -/// import fungibleTokenProducer from './producers/fungibleTokenAsset' import type { RpcMethodRegistrationValue } from './typs' +import nonFungibleTokenAsset from './producers/nonFungibleTokenAsset' +import nonFungibleCollectibleAsset from './producers/nonFungibleCollectibleAsset' // TODO: unit test - -export const producers: RpcMethodRegistrationValue[] = [fungibleTokenProducer] +export const producers: RpcMethodRegistrationValue[] = [ + fungibleTokenProducer, + nonFungibleTokenAsset, + nonFungibleCollectibleAsset, +] diff --git a/packages/provider-proxy/src/producers/nonFungibleCollectibleAsset.ts b/packages/provider-proxy/src/producers/nonFungibleCollectibleAsset.ts new file mode 100644 index 000000000000..07453770b27e --- /dev/null +++ b/packages/provider-proxy/src/producers/nonFungibleCollectibleAsset.ts @@ -0,0 +1,29 @@ +import { Collection, getOpenSeaCollectionList } from '@masknet/web3-providers' +import type { ProducerArgBase, ProducerKeyFunction, ProducerPushFunction, RpcMethodRegistrationValue } from '../typs' +import { collectAllPageDate } from '../helper/request' + +interface NonFungibleCollectibleAssetArgs extends ProducerArgBase { + address: string +} + +const nonFungibleCollectibleAsset = async ( + push: ProducerPushFunction, + getKeys: ProducerKeyFunction, + args: NonFungibleCollectibleAssetArgs, +): Promise => { + const { address } = args + const openSeaApiKey = await getKeys('opensea') + + const collectFromOpenSea = await collectAllPageDate(() => + getOpenSeaCollectionList(openSeaApiKey, address), + ) + await push(collectFromOpenSea) +} + +const producer: RpcMethodRegistrationValue = { + method: 'mask.fetchNonFungibleCollectibleAsset', + producer: nonFungibleCollectibleAsset, + distinctBy: (item) => item.name, +} + +export default producer diff --git a/packages/provider-proxy/src/producers/nonFungibleTokenAsset.ts b/packages/provider-proxy/src/producers/nonFungibleTokenAsset.ts index 452b4a67b0e2..a7ba16136857 100644 --- a/packages/provider-proxy/src/producers/nonFungibleTokenAsset.ts +++ b/packages/provider-proxy/src/producers/nonFungibleTokenAsset.ts @@ -13,7 +13,7 @@ const nonFungibleTokenAsset = async ( ): Promise => { const { address } = args // todo: key word - const openSeaApiKey = await getKeys('OpenSea') + const openSeaApiKey = await getKeys('opensea') let openSeaHasNextPage = true while (openSeaHasNextPage) { diff --git a/packages/web3-providers/src/opensea/index.ts b/packages/web3-providers/src/opensea/index.ts index eb57f71db747..e9d5adc4a227 100644 --- a/packages/web3-providers/src/opensea/index.ts +++ b/packages/web3-providers/src/opensea/index.ts @@ -340,6 +340,49 @@ export class OpenSeaAPI implements NonFungibleTokenAPI.Provider { } } +export async function getAssetsListFromOpenSea( + from: string, + opts: { chainId?: ChainId; page?: number; size?: number; collection?: string }, + apiKey: string, +) { + const { page = 0, size = 50, collection } = opts + const params = new URLSearchParams() + params.append('owner', from.toLowerCase()) + params.append('limit', String(size)) + params.append('offset', String(size * page)) + if (collection) { + params.append('collection', collection) + } + + const response = await fetch(`${OPENSEA_API}/api/v1/assets?${params.toString()}`, { + method: 'GET', + mode: 'cors', + headers: { + 'x-api-key': apiKey ?? OPENSEA_API_KEY, + }, + }) + return (await response.json()) as AssetsListResponse +} + +export async function getCollectionsFromOpenSea(owner: string, opts: { page?: number; size?: number }) { + const { page = 0, size = 300 } = opts + const params = new URLSearchParams() + params.append('asset_owner', owner.toLowerCase()) + params.append('limit', String(size)) + params.append('offset', String(size * page)) + + const response = await fetch(`${OPENSEA_API}/api/v1/collections?${params.toString()}`, { + method: 'GET', + mode: 'cors', + headers: { + 'x-api-key': OPENSEA_API_KEY, + }, + }) + const collections = (await response.json()) as AssetCollection[] + + return { collections } +} + export async function getOpenSeaNFTList( apiKey: string, address: string, @@ -351,3 +394,19 @@ export async function getOpenSeaNFTList( return format(address, size ?? 50, assets) } +export interface Collection { + name: string + image: string + slug: string +} +export async function getOpenSeaCollectionList(apiKey: string, address: string, page?: number, size?: number) { + const { collections } = await getCollectionsFromOpenSea(address, { page, size }) + return { + data: collections.map((x) => ({ + name: x.name, + image: x.image_url || undefined, + slug: x.slug, + })) as Collection[], + hasNextPage: collections.length === size, + } +} diff --git a/packages/web3-providers/src/opensea/type.ts b/packages/web3-providers/src/opensea/type.ts index c8b86996bc99..44298a0f6959 100644 --- a/packages/web3-providers/src/opensea/type.ts +++ b/packages/web3-providers/src/opensea/type.ts @@ -33,7 +33,7 @@ interface AssetOwner { discord_id: string | null } -interface AssetCollection { +export interface AssetCollection { banner_image_url: string | null chat_url: string | null created_date: string From 4c9212c0c34a98df3260ac5569beb4826bc833e8 Mon Sep 17 00:00:00 2001 From: Lantt Date: Fri, 24 Dec 2021 14:11:07 +0800 Subject: [PATCH 23/59] feat: should clear websocket pool data --- packages/dashboard/src/web3/context.ts | 4 +- packages/mask/src/web3/context.ts | 2 +- packages/plugins/Wallet/src/messages.ts | 8 ++- packages/web3-shared/base/proxy/index.ts | 53 ++++++++++++++----- packages/web3-shared/base/proxy/proxy.test.ts | 29 +++++++--- 5 files changed, 73 insertions(+), 23 deletions(-) diff --git a/packages/dashboard/src/web3/context.ts b/packages/dashboard/src/web3/context.ts index 3bd12093209e..9dee12f86bc3 100644 --- a/packages/dashboard/src/web3/context.ts +++ b/packages/dashboard/src/web3/context.ts @@ -115,8 +115,8 @@ export const Web3Context: Web3ProviderType = { getTransactionList: PluginServices.Wallet.getTransactionList, fetchERC20TokensFromTokenLists: Services.Ethereum.fetchERC20TokensFromTokenLists, // TODO: send to local - providerSocketInstance: getProxyWebsocketInstance((id) => - PluginMessages.Wallet.events.websocketDataUpdate.sendToAll(id), + providerSocketInstance: getProxyWebsocketInstance((event) => + PluginMessages.Wallet.events.websocketDataUpdate.sendToAll(event), ), } diff --git a/packages/mask/src/web3/context.ts b/packages/mask/src/web3/context.ts index f7893706f11c..10c97710321a 100644 --- a/packages/mask/src/web3/context.ts +++ b/packages/mask/src/web3/context.ts @@ -135,7 +135,7 @@ function createWeb3Context(disablePopup = false, isMask = false): Web3ProviderTy getAddressNamesList: WalletRPC.getAddressNames, getTransactionList: WalletRPC.getTransactionList, fetchERC20TokensFromTokenLists: Services.Ethereum.fetchERC20TokensFromTokenLists, - providerSocketInstance: getProxyWebsocketInstance(WalletMessages.events.websocketDataUpdate.sendToLocal), + providerSocketInstance: getProxyWebsocketInstance(WalletMessages.events.websocketDataUpdate.sendToAll), } } diff --git a/packages/plugins/Wallet/src/messages.ts b/packages/plugins/Wallet/src/messages.ts index 0c4125bde866..13582929e17b 100644 --- a/packages/plugins/Wallet/src/messages.ts +++ b/packages/plugins/Wallet/src/messages.ts @@ -137,6 +137,12 @@ export type SelectNftContractDialogEvent = { contract?: ERC721ContractDetailed } +export type WebsocketDataUpdateEvent = { + id: string + done: boolean + error?: unknown +} + export interface WalletMessage { /** * Transaction dialog @@ -219,7 +225,7 @@ export interface WalletMessage { /** true: Now locked; false: Now unlocked */ walletLockStatusUpdated: boolean /** request id */ - websocketDataUpdate: string + websocketDataUpdate: WebsocketDataUpdateEvent rpc: unknown } diff --git a/packages/web3-shared/base/proxy/index.ts b/packages/web3-shared/base/proxy/index.ts index 08745113ba41..3496d3831467 100644 --- a/packages/web3-shared/base/proxy/index.ts +++ b/packages/web3-shared/base/proxy/index.ts @@ -1,4 +1,6 @@ import differenceInSeconds from 'date-fns/differenceInSeconds' +import { first } from 'lodash-unified' +import compareAsc from 'date-fns/compareAsc' export interface MessageBase { id: string } @@ -21,6 +23,8 @@ export interface PollItem { done?: boolean } +export type NotifyFn = (event: { id: string; done: boolean; error?: unknown }) => void + // todo: // 1. avoid duplicate instance // 2. auto reconnection @@ -28,12 +32,12 @@ export interface PollItem { // 4. handle open status export class ProviderProxy { private readonly _socket: WebSocket - private readonly _poll: Map - private readonly _notify: (id: string) => void + private readonly _pool: Map + private readonly _notify: NotifyFn - constructor(point: string, notifyFn: (id: string) => void) { + constructor(point: string, notifyFn: NotifyFn) { this._socket = new WebSocket(point) - this._poll = new Map() + this._pool = new Map() this._notify = notifyFn } @@ -48,16 +52,21 @@ export class ProviderProxy { this._socket.addEventListener('message', (event: MessageEvent) => { console.debug('Message from server ', event.data) const { id, results, error } = JSON.parse(event.data) as PayloadMessage + this.clearPool() if (error || !id) { - // TODO: handle errors + this._notify({ id, done: true, error: error }) } if (results) { + if (results.length === 0) { + this._notify({ id, done: true }) + return + } const updatedAt = new Date() - const itemInPoll = this._poll.get(id) ?? { data: [], createdAt: updatedAt } + const itemInPoll = this._pool.get(id) ?? { data: [], createdAt: updatedAt } const dataInPool = itemInPoll?.data ?? [] const patchData = [...dataInPool, ...(results ?? [])] - this._poll.set(id, { ...itemInPoll, updatedAt, data: patchData }) - this._notify(id) + this._pool.set(id, { ...itemInPoll, updatedAt, data: patchData }) + this._notify({ id, done: false }) } }) } @@ -67,11 +76,11 @@ export class ProviderProxy { * @param message */ send(message: RequestMessage) { - const cache = this._poll.get(message.id) + const cache = this._pool.get(message.id) if (cache && !this.isExpired(cache!)) return this._socket.send(JSON.stringify(message)) - this._poll.set(message.id, { data: [], createdAt: new Date() }) + this._pool.set(message.id, { data: [], createdAt: new Date() }) } get socket(): WebSocket { @@ -80,7 +89,11 @@ export class ProviderProxy { getResult(id: string): T[] { // todo: update pick time - return (this._poll.get(id)?.data ?? []) as T[] + const item = this._pool.get(id) + if (!item) return [] + const newItem = { ...item, pickedAt: new Date() } + this._pool.set(id, newItem) + return item.data as T[] } /** @@ -96,8 +109,22 @@ export class ProviderProxy { // lasted pick time > 30s return !!item.pickedAt && differenceInSeconds(now, item.pickedAt) > 30 } + + private clearPool() { + const entities = Array.from(this._pool.entries()) + if (entities.length <= 10) { + return + } + + const firstPick = first( + entities.sort((a, b) => compareAsc(a[1].pickedAt || a[1].createdAt, b[1].pickedAt || b[1].createdAt)), + ) + if (!firstPick) return + this._pool.delete(firstPick[0]) + } } +// TODO: Production const DEV = 'wss://hyper-proxy-development.mask-reverse-proxy.workers.dev' enum SocketState { @@ -107,9 +134,9 @@ enum SocketState { CLOSED = 3, } -function getProxyWebsocketInstanceWrapper(): (notify: (id: string) => void) => Promise { +function getProxyWebsocketInstanceWrapper(): (notify: NotifyFn) => Promise { let cached: ProviderProxy - return async (notify: (id: string) => void) => { + return async (notify: NotifyFn) => { const state = cached?.socket.readyState if (!cached || state === SocketState.CLOSING || state === SocketState.CLOSED) { cached = new ProviderProxy(DEV, notify) diff --git a/packages/web3-shared/base/proxy/proxy.test.ts b/packages/web3-shared/base/proxy/proxy.test.ts index 4b75f8751808..8e25d9b33a1d 100644 --- a/packages/web3-shared/base/proxy/proxy.test.ts +++ b/packages/web3-shared/base/proxy/proxy.test.ts @@ -36,7 +36,7 @@ describe('Proxy websocket', () => { const data = client.getResult(id) expect(data).toEqual(['eth', 'bsc']) expect(mockNotifyCallback?.mock.calls.length).toBe(1) - expect(mockNotifyCallback?.mock.calls[0]).toEqual([id]) + expect(mockNotifyCallback?.mock.calls[0]).toEqual([{ id, done: false }]) }) test('should merge cache data when server push two times data', async () => { @@ -54,8 +54,8 @@ describe('Proxy websocket', () => { const data = client.getResult(id) expect(data).toEqual(['eth', 'bsc', 'matic']) expect(mockNotifyCallback?.mock.calls.length).toBe(2) - expect(mockNotifyCallback?.mock.calls[0]).toEqual([id]) - expect(mockNotifyCallback?.mock.calls[1]).toEqual([id]) + expect(mockNotifyCallback?.mock.calls[0]).toEqual([{ id, done: false }]) + expect(mockNotifyCallback?.mock.calls[1]).toEqual([{ id, done: false }]) }) test('should cache divide data when server push different response', async () => { @@ -82,9 +82,9 @@ describe('Proxy websocket', () => { const request2Data = client.getResult(requestID2) expect(request2Data).toEqual([1]) expect(mockNotifyCallback?.mock.calls.length).toBe(3) - expect(mockNotifyCallback?.mock.calls[0]).toEqual([requestID1]) - expect(mockNotifyCallback?.mock.calls[1]).toEqual([requestID2]) - expect(mockNotifyCallback?.mock.calls[2]).toEqual([requestID1]) + expect(mockNotifyCallback?.mock.calls[0]).toEqual([{ id: requestID1, done: false }]) + expect(mockNotifyCallback?.mock.calls[1]).toEqual([{ id: requestID2, done: false }]) + expect(mockNotifyCallback?.mock.calls[2]).toEqual([{ id: requestID1, done: false }]) }) test('should use cache when last pick within 30 second', async () => { @@ -98,6 +98,23 @@ describe('Proxy websocket', () => { await expect(server).toReceiveMessage(testMethod) }) + test('should clear cache ', async () => { + Array.from({ length: 11 }, (v, i) => i).forEach((x) => { + const id = `mask.fetchAsset_${x}` + const mockData = { id, results: ['eth', 'bsc'] } + const testMethod = { method: 'fetchAsset', params: [], id } + client.send(testMethod) + pushToClientMockData(mockData) + + // @ts-ignore + const data = client.getResult(id) + expect(data).toEqual(['eth', 'bsc']) + }) + + const data = client.getResult('mask.fetchAsset_0') + expect(data).toEqual([]) + }) + xtest('should merge cache data when server push two times data and uniq', async () => { const id = 'fetchAsset3' const testMethod = { method: 'fetchAsset2', params: [], id } From 00081fab0e1fc80145c4a2dba527011fd0088dc5 Mon Sep 17 00:00:00 2001 From: Lantt Date: Fri, 24 Dec 2021 17:34:25 +0800 Subject: [PATCH 24/59] feat: add opensea data in producer --- packages/web3-providers/src/opensea/format.ts | 11 +++++++++++ packages/web3-providers/src/opensea/index.ts | 2 -- packages/web3-shared/evm/types/index.ts | 5 +++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/web3-providers/src/opensea/format.ts b/packages/web3-providers/src/opensea/format.ts index df376634f699..2b6463b3d630 100644 --- a/packages/web3-providers/src/opensea/format.ts +++ b/packages/web3-providers/src/opensea/format.ts @@ -11,11 +11,17 @@ export function createERC721Token( contractDetailed: ERC721ContractDetailed, info: ERC721TokenInfo, tokenId: string, + collection?: { + name: string + image?: string + slug: string + }, ): ERC721TokenDetailed { return { contractDetailed, info, tokenId, + collection, } } @@ -43,6 +49,11 @@ export function format(address: string, size: number, assets: Asset[]) { mediaUrl: x.image_url || x.image_preview_url || x.asset_contract.image_url || '', }, x.token_id, + { + name: x.collection.name, + image: x.collection.image_url || undefined, + slug: x.collection.slug, + }, ), ), hasNextPage: assets.length === size, diff --git a/packages/web3-providers/src/opensea/index.ts b/packages/web3-providers/src/opensea/index.ts index e9d5adc4a227..6f35f409b032 100644 --- a/packages/web3-providers/src/opensea/index.ts +++ b/packages/web3-providers/src/opensea/index.ts @@ -356,7 +356,6 @@ export async function getAssetsListFromOpenSea( const response = await fetch(`${OPENSEA_API}/api/v1/assets?${params.toString()}`, { method: 'GET', - mode: 'cors', headers: { 'x-api-key': apiKey ?? OPENSEA_API_KEY, }, @@ -373,7 +372,6 @@ export async function getCollectionsFromOpenSea(owner: string, opts: { page?: nu const response = await fetch(`${OPENSEA_API}/api/v1/collections?${params.toString()}`, { method: 'GET', - mode: 'cors', headers: { 'x-api-key': OPENSEA_API_KEY, }, diff --git a/packages/web3-shared/evm/types/index.ts b/packages/web3-shared/evm/types/index.ts index 98a5802b490b..0116fd92c4b7 100644 --- a/packages/web3-shared/evm/types/index.ts +++ b/packages/web3-shared/evm/types/index.ts @@ -181,6 +181,11 @@ export interface ERC721TokenDetailed { tokenId: string info: ERC721TokenInfo contractDetailed: ERC721ContractDetailed + collection?: { + name: string + image?: string + slug: string + } } export interface ERC721TokenRecordInDatabase extends ERC721TokenDetailed { From 364ea7cd46645e316ee05facf248c345bd9b36e2 Mon Sep 17 00:00:00 2001 From: Lantt Date: Sat, 25 Dec 2021 02:02:59 +0800 Subject: [PATCH 25/59] feat: use socket to loading nft in web3 tab --- .../components/FungibleTokenTable/index.tsx | 2 +- .../CollectibleList/index.tsx | 47 ++++---- .../plugins/Avatar/SNSAdaptor/NFTAvatar.tsx | 45 +------ .../plugins/EVM/UI/Web3State/getAssetsFn.ts | 3 +- .../src/producers/nonFungibleTokenAsset.ts | 1 - packages/web3-providers/src/opensea/index.ts | 6 +- packages/web3-shared/base/proxy/index.ts | 62 +++++----- packages/web3-shared/evm/hooks/index.ts | 1 + .../web3-shared/evm/hooks/useCollectibles.ts | 110 +++++++----------- packages/web3-shared/evm/hooks/useSocket.ts | 32 +++++ packages/web3-shared/evm/package.json | 1 + pnpm-lock.yaml | 2 + 12 files changed, 146 insertions(+), 166 deletions(-) create mode 100644 packages/web3-shared/evm/hooks/useSocket.ts diff --git a/packages/dashboard/src/pages/Wallets/components/FungibleTokenTable/index.tsx b/packages/dashboard/src/pages/Wallets/components/FungibleTokenTable/index.tsx index 148f1f64fda8..bcf38a9508d9 100644 --- a/packages/dashboard/src/pages/Wallets/components/FungibleTokenTable/index.tsx +++ b/packages/dashboard/src/pages/Wallets/components/FungibleTokenTable/index.tsx @@ -89,7 +89,7 @@ export const FungibleTokenTable = memo(({ selectedChainId }) => PluginMessages.Wallet.events.erc20TokensUpdated.on(() => setTimeout(() => setTokenUpdateCount((prev) => prev + 1), 100), ) - PluginMessages.Wallet.events.websocketDataUpdate.on((id) => { + PluginMessages.Wallet.events.websocketDataUpdate.on(() => { if ('fetchFungibleTokenAsset' + account + network?.chainId) { setTimeout(() => setTokenUpdateCount((prev) => prev + 1), 100) } diff --git a/packages/mask/src/extension/options-page/DashboardComponents/CollectibleList/index.tsx b/packages/mask/src/extension/options-page/DashboardComponents/CollectibleList/index.tsx index 4c7d31f722eb..f79d871bb627 100644 --- a/packages/mask/src/extension/options-page/DashboardComponents/CollectibleList/index.tsx +++ b/packages/mask/src/extension/options-page/DashboardComponents/CollectibleList/index.tsx @@ -1,10 +1,8 @@ -import { createContext, useState, useEffect } from 'react' -import { useUpdateEffect } from 'react-use' +import { createContext, useState } from 'react' import { useValueRef } from '@masknet/shared' import { ChainId, NonFungibleAssetProvider, - ERC721TokenCollectionInfo, ERC721TokenDetailed, useCollectibles, useCollections, @@ -15,7 +13,6 @@ import { makeStyles, useStylesExtends } from '@masknet/theme' import { currentNonFungibleAssetDataProviderSettings } from '../../../../plugins/Wallet/settings' import { useI18N } from '../../../../utils' import { CollectibleCard } from './CollectibleCard' -import { WalletMessages } from '../../../../plugins/Wallet/messages' import { Image } from '../../../../components/shared/Image' import { uniqBy } from 'lodash-unified' @@ -112,7 +109,7 @@ interface CollectibleListUIProps extends withClasses<'empty' | 'button' | 'text' collectibles: ERC721TokenDetailed[] loading: boolean collectiblesRetry: () => void - error: Error | undefined + error: string | undefined readonly?: boolean hasRetry?: boolean } @@ -121,7 +118,8 @@ function CollectibleListUI(props: CollectibleListUIProps) { const { t } = useI18N() const classes = useStylesExtends(useStyles(), props) - useEffect(() => WalletMessages.events.erc721TokensUpdated.on(collectiblesRetry)) + // TODO: should we listen erc721 event + // useEffect(() => WalletMessages.events.erc721TokensUpdated.on(collectiblesRetry)) if (loading) return ( @@ -177,13 +175,13 @@ export interface CollectibleListProps extends withClasses<'empty' | 'button'> { address: string collection?: string setCount: (count: number) => void + collectibles: ERC721TokenDetailed[] + error?: string } export function CollectibleList(props: CollectibleListProps) { - const { address, collection, setCount } = props + const { address, collectibles, error } = props const provider = useValueRef(currentNonFungibleAssetDataProviderSettings) - const chainId = ChainId.Mainnet - const [page, setPage] = useState(0) const classes = props.classes ?? {} const { @@ -200,15 +198,20 @@ export function CollectibleList(props: CollectibleListProps) { }, [provider, address]) useEffect(() => { - if (!collectibles.length) return - setRendCollectibles([...rendCollectibles, ...collectibles]) - if (!hasNextPage) return - const timer = setTimeout(() => { - setPage(page + 1) - }, 1000) - return () => { - clearTimeout(timer) + if (collectibles.length) { + setRendCollectibles([...rendCollectibles, ...collectibles]) + if (hasNextPage) { + const timer = setTimeout(() => { + setPage(page + 1) + }, 1000) + + return () => { + clearTimeout(timer) + } + } } + + return () => {} }, [collectibles]) useEffect(() => { @@ -220,9 +223,10 @@ export function CollectibleList(props: CollectibleListProps) { classes={classes} provider={provider} collectibles={collectibles} - loading={collectiblesLoading} - collectiblesRetry={collectiblesRetry} - error={collectiblesError} + // TODO: fix loading statue + loading={false} + collectiblesRetry={() => {}} + error={error} readonly hasRetry={!!address} /> @@ -268,7 +272,7 @@ export function CollectionList({ address }: { address: string }) { return ( - {rendCollections.map((x, i) => ( + {(collections ?? []).map((x, i) => ( @@ -290,6 +294,7 @@ export function CollectionList({ address }: { address: string }) { counts[i] = count setCounts(counts) }} + collectibles={collectible.filter((c) => c.collection?.slug === x.slug)} /> ))} diff --git a/packages/mask/src/plugins/Avatar/SNSAdaptor/NFTAvatar.tsx b/packages/mask/src/plugins/Avatar/SNSAdaptor/NFTAvatar.tsx index 20aef9b42ba6..f90d9830a84d 100644 --- a/packages/mask/src/plugins/Avatar/SNSAdaptor/NFTAvatar.tsx +++ b/packages/mask/src/plugins/Avatar/SNSAdaptor/NFTAvatar.tsx @@ -1,7 +1,7 @@ import { useCallback, useState } from 'react' import { uniqBy } from 'lodash-unified' import { WalletMessages } from '@masknet/plugin-wallet' -import { useRemoteControlledDialog, useValueRef } from '@masknet/shared' +import { useRemoteControlledDialog } from '@masknet/shared' import { makeStyles, useStylesExtends } from '@masknet/theme' import { ChainId } from '@masknet/web3-shared-evm' import { @@ -11,8 +11,7 @@ import { useChainId, useCollectibles, } from '@masknet/web3-shared-evm' -import { Box, Button, Skeleton, TablePagination, Typography } from '@mui/material' -import { currentNonFungibleAssetDataProviderSettings } from '../../../plugins/Wallet/settings' +import { Box, Button, Skeleton, Typography } from '@mui/material' import { useI18N } from '../../../utils' import { EthereumChainBoundary } from '../../../web3/UI/EthereumChainBoundary' import { AddNFT } from './AddNFT' @@ -91,22 +90,11 @@ export function NFTAvatar(props: NFTAvatarProps) { const classes = useStylesExtends(useStyles(), props) const account = useAccount() const chainId = useChainId() - const provider = useValueRef(currentNonFungibleAssetDataProviderSettings) - const [page, setPage] = useState(0) const [selectedToken, setSelectedToken] = useState() const [open_, setOpen_] = useState(false) const [collectibles_, setCollectibles_] = useState([]) const { t } = useI18N() - const { - value = { - collectibles: [], - hasNextPage: false, - }, - loading, - retry, - error, - } = useCollectibles(account, ChainId.Mainnet, provider, page, 50) - const { collectibles, hasNextPage } = value + const { data: collectibles, error, retry } = useCollectibles(account, ChainId.Mainnet) const onClick = useCallback(async () => { if (!selectedToken) return @@ -123,6 +111,7 @@ export function NFTAvatar(props: NFTAvatarProps) { WalletMessages.events.selectProviderDialogUpdated, ) + // TODO: merger loading const LoadStatus = Array.from({ length: 8 }) .fill(0) .map((_, i) => ( @@ -157,9 +146,7 @@ export function NFTAvatar(props: NFTAvatarProps) { - {loading - ? LoadStatus - : error || (collectibles.length === 0 && collectibles_.length === 0) + {!error || (collectibles.length === 0 && collectibles_.length === 0) ? Retry : uniqBy( [...collectibles_, ...collectibles], @@ -179,28 +166,6 @@ export function NFTAvatar(props: NFTAvatarProps) { /> ))} - - {hasNextPage || page > 0 ? ( - {}} - page={page} - rowsPerPage={30} - rowsPerPageOptions={[30]} - labelDisplayedRows={() => null} - backIconButtonProps={{ - onClick: () => setPage(page - 1), - size: 'small', - disabled: page === 0, - }} - nextIconButtonProps={{ - onClick: () => setPage(page + 1), - disabled: !hasNextPage, - size: 'small', - }} - /> - ) : null}