client: optimistic (beacon) sync (#1878)

* beacon sync (optimistic sync) implementation

Co-authored-by: harkamal <gajinder@g11.in>
Co-authored-by: acolytec3 <17355484+acolytec3@users.noreply.github.com>

Author: 3 people

packages/client/bin/cli.ts

@@ -11,7 +11,7 @@ import { _getInitializedChains } from '@ethereumjs/common/dist/chains'
 import { Address, toBuffer, BN } from 'ethereumjs-util'
 import { parseMultiaddrs, parseGenesisState, parseCustomParams } from '../lib/util'
 import EthereumClient from '../lib/client'
-import { Config, DataDirectory } from '../lib/config'
+import { Config, DataDirectory, SyncMode } from '../lib/config'
 import { Logger, getLogger } from '../lib/logging'
 import { startRPCServers, helprpc } from './startRpc'
 import type { Chain as IChain, GenesisState } from '@ethereumjs/common/dist/types'
@@ -39,7 +39,7 @@ const args = yargs(hideBin(process.argv))
   })
   .option('syncmode', {
     describe: 'Blockchain sync mode (light sync experimental)',
-    choices: ['light', 'full'],
+    choices: Object.values(SyncMode),
     default: Config.SYNCMODE_DEFAULT,
   })
   .option('lightserv', {
@@ -250,6 +250,11 @@ const args = yargs(hideBin(process.argv))
       'Save tx receipts and logs in the meta db (warning: may use a large amount of storage). With `--rpc` allows querying via eth_getLogs (max 10000 logs per request) and eth_getTransactionReceipt (within `--txLookupLimit`)',
     boolean: true,
   })
+  .option('disableBeaconSync', {
+    describe:
+      'Disables beacon (optimistic) sync if the CL provides blocks at the head of the chain',
+    boolean: true,
+  })
   .option('txLookupLimit', {
     describe:
       'Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)',
@@ -276,13 +281,10 @@ function initDBs(config: Config) {
   ensureDirSync(stateDataDir)
   const stateDB = level(stateDataDir)
 
-  // Meta DB (receipts, logs, indexes)
-  let metaDB
-  if (args.saveReceipts) {
-    const metaDataDir = config.getDataDirectory(DataDirectory.Meta)
-    ensureDirSync(metaDataDir)
-    metaDB = level(metaDataDir)
-  }
+  // Meta DB (receipts, logs, indexes, skeleton chain)
+  const metaDataDir = config.getDataDirectory(DataDirectory.Meta)
+  ensureDirSync(metaDataDir)
+  const metaDB = level(metaDataDir)
 
   return { chainDB, stateDB, metaDB }
 }
@@ -621,6 +623,7 @@ async function run() {
     port: args.port,
     saveReceipts: args.saveReceipts,
     syncmode: args.syncmode,
+    disableBeaconSync: args.disableBeaconSync,
     transports: args.transports,
     txLookupLimit: args.txLookupLimit,
   })

packages/client/kiln/README.md

@@ -6,10 +6,9 @@ Kiln v2 public testnet has been bootstrapped:
 
 The config files can be downloaded from the [merge-testnets](https://github.com/eth-clients/merge-testnets/tree/main/kiln) config GitHub repository.
 
-
 ## Execution - EthereumJS Setup
 
-Please ensure you have Node 12.x+ installed.
+Please ensure you have Node 16.x installed.
 
 ### Client Installation
 
@@ -28,7 +27,7 @@ The `v0.4.0` client release (respectively follow-up bug fix releases) is ready f
 npm install -g @ethereumjs/client
 ```
 
-Note that you eventually want to adopt the `config` download and accordingly modify the config file access paths as well as start the client just with `ethereumjs` instad of using the `npm run client:start` GitHub start command (also leave the inbetween `--` to forward the client config options).
+Note that you eventually want to adopt the `config` download and accordingly modify the config file access paths as well as start the client just with `ethereumjs` instead of using the `npm run client:start` GitHub start command (also leave the in-between `--` to forward the client config options).
 
 #### Docker
 
@@ -67,7 +66,7 @@ To prevent the secret to be re-generated next time you restart the client, pass
 1. Use lodestar branch `master` and run `yarn && yarn build`
 2. Export path of the downloaded config dir `export CONFIG_PATH=/path/to/ethereumjs-monorepo/packages/client/kiln/config`
 3. Export path of the written jwt secret file `export JWT_SECRET_PATH=/path/to/ethereumjs-monorepo/packages/client/kiln/datadir/jwtsecret`
-4. Run cmd: `./lodestar beacon --rootDir kiln/temp --paramsFile $CONFIG_PATH/config.yaml --genesisStateFile $CONFIG_PATH/genesis.ssz --bootnodesFile $CONFIG_PATH/boot_enr.yaml --network.connectToDiscv5Bootnodes --network.discv5.enabled true --eth1.enabled true --eth1.providerUrls=http://localhost:8545 --execution.urls=http://localhost:8551 --eth1.disableEth1DepositDataTracker true --jwt-secret $JWT_SECRET_PATH`
+4. Run cmd: `./lodestar beacon --rootDir kiln/temp --paramsFile $CONFIG_PATH/config.yaml --genesisStateFile $CONFIG_PATH/genesis.ssz --bootnodesFile $CONFIG_PATH/boot_enr.yaml --network.connectToDiscv5Bootnodes --network.discv5.enabled true --eth1.enabled true --eth1.providerUrls=http://localhost:8545 --execution.urls=http://localhost:8551 --eth1.disableEth1DepositDataTracker true --jwt-secret $JWT_SECRET_PATH --weakSubjectivityServerUrl https://lodestar-kiln.chainsafe.io --weakSubjectivitySyncLatest`
 
 #### Validator
 
@@ -77,8 +76,26 @@ Also, one will need to remove `--eth1.disableEth1DepositDataTracker true` and in
 
 ### Lighthouse
 
-### Beacon
+#### Beacon
 
 1. Use lighthouse branch `unstable` and run `make`
-1. Make dir `lighthouse/kiln` and copy in from the downloaded config dir: `config.yaml`, `genesis.ssz`, `deploy_block.txt`, `deposit_contract.txt`, `deposit_contract_block.txt`
-1. Run cmd: `lighthouse --debug-level=info --datadir=kiln/datadir --testnet-dir=kiln beacon_node --disable-enr-auto-update --dummy-eth1 --boot-nodes="enr:" --merge --http-allow-sync-stalled --metrics --disable-packet-filter --execution-endpoints=http://127.0.0.1:8551 --terminal-total-difficulty-override=`
+2. Make dir `lighthouse/kiln` and copy in from the downloaded config dir: `config.yaml`, `genesis.ssz`, `deploy_block.txt`, `deposit_contract.txt`, `deposit_contract_block.txt`
+3. Run cmd: `lighthouse --debug-level=info --datadir=kiln/datadir --testnet-dir=kiln beacon_node --disable-enr-auto-update --dummy-eth1 --boot-nodes="enr:" --merge --http-allow-sync-stalled --metrics --disable-packet-filter --execution-endpoints=http://127.0.0.1:8551 --terminal-total-difficulty-override=20000000000000`
+4. Run cmd (with checkpoint sync - tested this with a locally running syncd Nimbus client): `lighthouse bn --network kiln --terminal-total-difficulty-override=20000000000000 --merge --http-allow-sync-stalled --checkpoint-sync-url "http://localhost:5052" --logfile logs.txt --logfile-debug-level trace`
+
+### Teku
+
+#### Beacon
+
+1. Download latest Teku binary [here](https://github.com/ConsenSys/teku/releases) and extract from archive
+2. Run cmd (with checkpoint sync): `./teku --data-path "./data" --network kiln --initial-state="https://lodestar-kiln.chainsafe.io/eth/v2/debug/beacon/states/finalized" --ee-endpoint http://localhost:8551 --ee-jwt-secret-file "/path/to/ethereumjs-monorepo/packages/client/kiln/datadir/jwtsecret" --logging=DEBUG`
+
+### Nimbus
+
+#### Beacon
+
+1. Build Nimbus following the [kiln instructions](https://nimbus.guide/kiln.html#3-nimbus)
+2. Get your hands on a SSZ encoded finalized state/block snapshot from a synced client. Assuming you have a synced Teku (or other CL node running locally that exposes the Beacon REST API), you can use the below `curl` commands to get it.
+   `curl -H 'Accept: application/octet-stream' http://127.0.0.1:5051/eth/v2/debug/beacon/states/finalized > state.ssz`
+   `curl -H 'Accept: application/octet-stream' http://127.0.0.1:5051/eth/v2/beacon/blocks/[block number corresponding to finalized state above] > block.ssz`
+3. Run cmd (with checkpoint sync and adjust ports/paths accordingly for your setup): `build/nimbus_beacon_node --network=vendor/merge-testnets/kiln --web3-url=ws://127.0.0.1:8551 --log-level=DEBUG --jwt-secret="/path/to/ethereumjs-monorepo/packages/client/kiln/datadir/jwtsecret" --data-dir=build/kiln --data-dir:trusted --finalized-checkpoint-state=state.ssz --finalized-checkpoint-block=block.ssz`

packages/client/lib/blockchain/chain.ts

@@ -182,7 +182,7 @@ export class Chain {
     await this.blockchain.db.open()
     await this.blockchain.initPromise
     this.opened = true
-    await this.update()
+    await this.update(false)
 
     this.config.chainCommon.on('hardforkChanged', async (hardfork: string) => {
       if (hardfork !== Hardfork.Merge) {

packages/client/lib/client.ts

@@ -31,8 +31,7 @@ export interface EthereumClientOptions {
    * Database to store tx receipts, logs, and indexes.
    * Should be an abstract-leveldown compliant store.
    *
-   * Default: Database created in datadir folder when
-   * `--saveReceipts` is enabled, otherwise undefined
+   * Default: Database created in datadir folder
    */
   metaDB?: LevelUp
 

packages/client/lib/config.ts

@@ -38,6 +38,14 @@ export interface ConfigOptions {
    */
   syncmode?: SyncMode
 
+  /**
+   * Whether to disable beacon (optimistic) sync if CL provides
+   * blocks at the head of chain.
+   *
+   * Default: false
+   */
+  disableBeaconSync?: boolean
+
   /**
    * Provide a custom VM instance to process blocks
    *
@@ -259,6 +267,7 @@ export class Config {
   public readonly accounts: [address: Address, privKey: Buffer][]
   public readonly minerCoinbase?: Address
   public readonly safeReorgDistance: number
+  public readonly disableBeaconSync: boolean
 
   public synchronized: boolean
   public lastSyncDate: number
@@ -295,6 +304,7 @@ export class Config {
     this.accounts = options.accounts ?? []
     this.minerCoinbase = options.minerCoinbase
     this.safeReorgDistance = options.safeReorgDistance ?? Config.SAFE_REORG_DISTANCE
+    this.disableBeaconSync = options.disableBeaconSync ?? false
 
     this.synchronized = false
     this.lastSyncDate = 0

packages/client/lib/execution/receipt.ts

@@ -3,7 +3,7 @@ import { Log } from '@ethereumjs/vm/dist/evm/types'
 import Bloom from '@ethereumjs/vm/dist/bloom'
 import { TypedTransaction } from '@ethereumjs/tx'
 import { rlp, intToBuffer, bufferToInt } from 'ethereumjs-util'
-import { MetaDBManager, DBKey } from './metaDBManager'
+import { MetaDBManager, DBKey } from '../util/metaDBManager'
 import type { Block } from '@ethereumjs/block'
 
 /**

packages/client/lib/execution/vmexecution.ts

@@ -53,7 +53,7 @@ export class VMExecution extends Execution {
       ;(this.vm as any).blockchain = this.chain.blockchain
     }
 
-    if (this.metaDB) {
+    if (this.metaDB && this.config.saveReceipts) {
       this.receiptsManager = new ReceiptsManager({
         chain: this.chain,
         config: this.config,

packages/client/lib/miner/miner.ts

@@ -6,6 +6,7 @@ import { Event } from '../types'
 import { Config } from '../config'
 import { FullEthereumService } from '../service'
 import { VMExecution } from '../execution'
+import type { FullSynchronizer } from '../sync'
 
 const level = require('level-mem')
 
@@ -300,7 +301,7 @@ export class Miner {
     this.assembling = false
     if (interrupt) return
     // Put block in blockchain
-    await this.service.synchronizer.handleNewBlock(block)
+    await (this.service.synchronizer as FullSynchronizer).handleNewBlock(block)
     // Remove included txs from TxPool
     this.service.txPool.removeNewBlockTxs([block])
     this.config.events.removeListener(Event.CHAIN_UPDATED, _boundSetInterruptHandler)

packages/client/lib/net/protocol/ethprotocol.ts

@@ -66,15 +66,15 @@ export interface EthProtocolMethods {
   getReceipts: (opts: GetReceiptsOpts) => Promise<[BN, TxReceipt[]]>
 }
 
-const id = new BN(0)
-
 /**
  * Implements eth/66 protocol
  * @memberof module:net/protocol
  */
 export class EthProtocol extends Protocol {
   private chain: Chain
+  private nextReqId = new BN(0)
 
+  /* eslint-disable no-invalid-this */
   private protocolMessages: Message[] = [
     {
       name: 'NewBlockHashes',
@@ -108,7 +108,7 @@ export class EthProtocol extends Protocol {
       code: 0x03,
       response: 0x04,
       encode: ({ reqId, block, max, skip = 0, reverse = false }: GetBlockHeadersOpts) => [
-        (reqId === undefined ? id.iaddn(1) : new BN(reqId)).toArrayLike(Buffer),
+        (reqId === undefined ? this.nextReqId.iaddn(1) : new BN(reqId)).toArrayLike(Buffer),
         [BN.isBN(block) ? block.toArrayLike(Buffer) : block, max, skip, !reverse ? 0 : 1],
       ],
       decode: ([reqId, [block, max, skip, reverse]]: any) => ({
@@ -135,7 +135,7 @@ export class EthProtocol extends Protocol {
           // and we look backwards for the correct block)
           BlockHeader.fromValuesArray(h, {
             hardforkByBlockNumber: true,
-            common: this.config.chainCommon, // eslint-disable-line no-invalid-this
+            common: this.config.chainCommon,
           })
         ),
       ],
@@ -145,7 +145,7 @@ export class EthProtocol extends Protocol {
       code: 0x05,
       response: 0x06,
       encode: ({ reqId, hashes }: GetBlockBodiesOpts) => [
-        (reqId === undefined ? id.iaddn(1) : new BN(reqId)).toArrayLike(Buffer),
+        (reqId === undefined ? this.nextReqId.iaddn(1) : new BN(reqId)).toArrayLike(Buffer),
         hashes,
       ],
       decode: ([reqId, hashes]: [Buffer, Buffer[]]) => ({
@@ -168,7 +168,6 @@ export class EthProtocol extends Protocol {
       encode: ([block, td]: [Block, BN]) => [block.raw(), td.toArrayLike(Buffer)],
       decode: ([block, td]: [BlockBuffer, Buffer]) => [
         Block.fromValuesArray(block, {
-          // eslint-disable-next-line no-invalid-this
           common: this.config.chainCommon,
           hardforkByBlockNumber: true,
         }),
@@ -184,7 +183,7 @@ export class EthProtocol extends Protocol {
       code: 0x09,
       response: 0x0a,
       encode: ({ reqId, hashes }: GetPooledTransactionsOpts) => [
-        (reqId === undefined ? id.iaddn(1) : new BN(reqId)).toArrayLike(Buffer),
+        (reqId === undefined ? this.nextReqId.iaddn(1) : new BN(reqId)).toArrayLike(Buffer),
         hashes,
       ],
       decode: ([reqId, hashes]: [Buffer, Buffer[]]) => ({
@@ -219,7 +218,7 @@ export class EthProtocol extends Protocol {
       code: 0x0f,
       response: 0x10,
       encode: ({ reqId, hashes }: { reqId: BN; hashes: Buffer[] }) => [
-        (reqId === undefined ? id.iaddn(1) : new BN(reqId)).toArrayLike(Buffer),
+        (reqId === undefined ? this.nextReqId.iaddn(1) : new BN(reqId)).toArrayLike(Buffer),
         hashes,
       ],
       decode: ([reqId, hashes]: [Buffer, Buffer[]]) => ({
@@ -319,7 +318,6 @@ export class EthProtocol extends Protocol {
         : this.chain.blocks.td.toArrayLike(Buffer),
       bestHash: this.chain.blocks.latest!.hash(),
       genesisHash: this.chain.genesis.hash,
-      latestBlock: this.chain.blocks.latest!.header.number.toArrayLike(Buffer),
     }
   }