merge: kintsugi updates (#1565)

* * Add EIP-4399: Supplant DIFFICULTY opcode with RANDOM
* EIP-3675 changes: validate extraData does not exceed 32 bytes
* Remove pithos files

* implement updated Engine API spec

* Apply suggestions from code review

* add files for merge-devnet-0, small fixes to successfully sync to head

* update readme, add kintsugi docker files, move log timestamp to start of log

* v3 spec update: rename message field to validationError

* throw proper rpc error message shape on internal error

* allow ws and http on the same port

* update instructions to merge-devnet-3

* receiptRoot -> receiptsRoot
feeReceipient -> suggestedFeeReceipient

* use mergeForkBlock if provided

* wip: respect message ordering in forkchoiceUpdated

* update instructions to latest

* update docker-compose (closes #1644)

* update docker-compose: use bootnode from config/el_bootnode.txt

* Rename coinbase to feeRecipient as per v1.0.0-alpha.5 spec (#1659)

* add preMerge hardfork to implement setting FORK_NEXT value from EIP-3675

* remove mergeForkBlock logic from common (solved by preMerge hardfork)

* set preMerge value to null (this is set when parsing the genesis.json file in the client)

* fix getUncleCountByBlockNumber validation

* update lodestar kintsugi instructions

* kintsugi config update (#1682)

* engine: pass feeRecipient as coinbase

* vm: simplify 0x44 opcode for eip-4399

* engine: temporarily comment out following message order (need to find a way to get access to the request id)

* execution: only skipBlockValidation if consensus is PoA (based on comment)

* client: fix tests with enabled block verification (removed hardcoded gasLimit, set timestamp based on parent block plus one)

* client: add tests for new pendingBlock class

* client: add two more tests to pendingBlock for increased coverage

* kintsugi docker: remove extra WORKDIR, move instructions up in readme

* fix root Dockerfile typo (should match version 16 above)

* engine: improve while condition and getting stateRoot for executing previous blocks prior to head

* turn off debug logs in command for quicker exec

* remove default --logFile param (to use less space)

* add missing vm changelog entry

* simplify exec log post merge by removing td output (since it is constant)

* clean up TODOs (message ordering removed in kiln spec, so can be removed here)

Co-authored-by: g11tech <76567250+g11tech@users.noreply.github.com>

Author: ryanio

Dockerfile

@@ -6,7 +6,7 @@ ARG VERSION=latest
 ENV VERSION=$VERSION
 RUN npm install @ethereumjs/client@$VERSION
 
-FROM node:14-alpine
+FROM node:16-alpine
 WORKDIR /usr/app
 COPY --from=build /usr/app .
 

docker/docker-compose.yml

@@ -8,5 +8,5 @@ services:
     volumes:
       - ./datadir:/datadir
     command: >
-      --datadir=/datadir --logFile
+      --datadir=/datadir
     network_mode: host

packages/block/src/header.ts

@@ -65,6 +65,19 @@ export class BlockHeader {
     return this.logsBloom
   }
 
+  /**
+   * EIP-4399: Post-PoS merge, `mixHash` supplanted as `random`
+   */
+  get random() {
+    if (!this._common.isActivatedEIP(4399)) {
+      const msg = this._errorMsg(
+        'The random parameter can only be accessed when EIP-4399 is activated'
+      )
+      throw new Error(msg)
+    }
+    return this.mixHash
+  }
+
   /**
    * Static constructor to create a block header from a header data dictionary
    *
@@ -352,7 +365,7 @@ export class BlockHeader {
       transactionsTrie,
       receiptTrie,
       difficulty,
-      // extraData,
+      extraData,
       mixHash,
       nonce,
     } = this
@@ -397,7 +410,7 @@ export class BlockHeader {
       }
     }
 
-    // Check for constant values for PoS blocks
+    // Validation for PoS blocks (EIP-3675)
     if (this._common.consensusType() === ConsensusType.ProofOfStake) {
       let error = false
       let errorMsg = ''
@@ -412,14 +425,10 @@ export class BlockHeader {
         errorMsg += `, difficulty: ${difficulty} (expected: 0)`
         error = true
       }
-      // WIP from merge interop event:
-      // extraData behavior pending consideration, for now allowed to be non-empty
-      // if (!extraData.equals(Buffer.from([]))) {
-      //   errorMsg += `, extraData: ${extraData.toString('hex')} (expected: '')`
-      //   error = true
-      // }
-      if (!mixHash.equals(zeros(32))) {
-        errorMsg += `, mixHash: ${mixHash.toString('hex')} (expected: ${zeros(32).toString('hex')})`
+      if (extraData.length > 32) {
+        errorMsg += `, extraData: ${extraData.toString(
+          'hex'
+        )} (cannot exceed 32 bytes length, received ${extraData.length} bytes)`
         error = true
       }
       if (!nonce.equals(zeros(8))) {

packages/block/test/mergeBlock.spec.ts

@@ -22,8 +22,8 @@ function validateMergeHeader(st: tape.Test, header: BlockHeader) {
   st.ok(header.gasLimit.eq(new BN(Buffer.from('ffffffffffffff', 'hex'))), 'gasLimit')
   st.ok(header.gasUsed.isZero(), 'gasUsed')
   st.ok(header.timestamp.isZero(), 'timestamp')
-  st.ok(header.extraData.equals(Buffer.from([])), 'extraData')
-  st.ok(header.mixHash.equals(zeros(32)), 'mixHash')
+  st.ok(header.extraData.length <= 32, 'extraData')
+  st.ok(header.mixHash.length === 32, 'mixHash')
   st.ok(header.nonce.equals(zeros(8)), 'nonce')
 }
 
@@ -60,26 +60,24 @@ tape('[Header]: Casper PoS / The Merge Functionality', function (t) {
       st.pass('should throw on wrong difficulty')
     }
 
-    // WIP from merge interop event:
-    // extraData behavior pending consideration, for now allowed to be non-empty
-    // try {
-    //   const headerData = {
-    //     extraData: Buffer.from('123abc', 'hex'),
-    //   }
-    //   BlockHeader.fromHeaderData(headerData, { common })
-    //   st.fail('should throw')
-    // } catch (e: any) {
-    //   st.pass('should throw on wrong extraData')
-    // }
+    try {
+      const headerData = {
+        extraData: Buffer.alloc(33).fill(1),
+      }
+      BlockHeader.fromHeaderData(headerData, { common })
+      st.fail('should throw')
+    } catch (e: any) {
+      st.pass('should throw on invalid extraData length')
+    }
 
     try {
       const headerData = {
-        mixHash: Buffer.alloc(32).fill(1),
+        mixHash: Buffer.alloc(30).fill(1),
       }
       BlockHeader.fromHeaderData(headerData, { common })
       st.fail('should throw')
     } catch (e: any) {
-      st.pass('should throw on wrong mixHash')
+      st.pass('should throw on invalid mixHash length')
     }
 
     try {
@@ -106,4 +104,21 @@ tape('[Header]: Casper PoS / The Merge Functionality', function (t) {
     }
     st.end()
   })
+
+  t.test('EIP-4399: random should return mixHash value', function (st) {
+    const mixHash = Buffer.alloc(32, 3)
+    let block = Block.fromBlockData({ header: { mixHash } }, { common })
+    st.ok(block.header.random.equals(mixHash), 'random should return mixHash value')
+
+    const commonLondon = common.copy()
+    commonLondon.setHardfork(Hardfork.London)
+    block = Block.fromBlockData({ header: { mixHash } }, { common: commonLondon })
+    try {
+      block.header.random
+      st.fail('should have thrown')
+    } catch (e: any) {
+      st.pass('random should throw if EIP-4399 is not activated')
+    }
+    st.end()
+  })
 })

packages/client/bin/cli.ts

@@ -376,7 +376,14 @@ function startRPCServers(client: EthereumClient) {
       )
     }
     if (ws) {
-      server.websocket({ port: wsPort })
+      const opts: any = { port: wsPort }
+      if (rpcaddr === wsAddr && rpcport === wsPort) {
+        // If http and ws are listening on the same port,
+        // pass in the existing server to prevent a listening error
+        delete opts.port
+        opts.server = server
+      }
+      server.websocket(opts)
       config.logger.info(
         `Started JSON RPC Server address=ws://${wsAddr}:${wsPort} namespaces=${namespaces}`
       )
@@ -673,4 +680,4 @@ async function run() {
   })
 }
 
-run().catch((err) => logger?.error(err) ?? console.error(err))
+run().catch((err) => logger?.error(err.message.toString()) ?? console.error(err))

packages/client/pithos/Dockerfile packages/client/kintsugi/Dockerfilepackages/client/pithos/Dockerfile renamed to packages/client/kintsugi/Dockerfile

@@ -1,14 +1,13 @@
-FROM node:14-alpine as build
+FROM node:16-alpine as build
 WORKDIR /usr/app
-RUN apk update && apk add --no-cache g++ make python bash git && rm -rf /var/cache/apk/*
+RUN apk update && apk add --no-cache bash git && rm -rf /var/cache/apk/*
+
+RUN git clone --depth 1 --branch merge-kintsugi https://github.com/ethereumjs/ethereumjs-monorepo.git
 
-WORKDIR /usr/app
-RUN git clone --depth 1 --branch merge-pithos https://github.com/ethereumjs/ethereumjs-monorepo.git
 WORKDIR /usr/app/ethereumjs-monorepo
-RUN npm i -g npm@7
 RUN npm i
 
-FROM node:14-alpine
+FROM node:16-alpine
 WORKDIR /usr/app
 COPY --from=build /usr/app .
 

packages/client/kintsugi/README.md

@@ -0,0 +1,43 @@
+# merge-devnet-3 instructions
+
+## Execution - EthereumJS Setup
+
+Please ensure you have Node 12.x+ installed
+
+1. `git clone --depth 1 --branch merge-kintsugi https://github.com/ethereumjs/ethereumjs-monorepo.git`
+1. `cd ethereumjs-monorepo`
+1. `npm i`
+1. `cd packages/client`
+
+### Run client
+
+1. `npm run client:start -- --datadir kintsugi/datadir --gethGenesis kintsugi/config/genesis.json --saveReceipts --rpc --ws --rpcEngine --rpcEnginePort=8545 --bootnodes=64.225.4.226:30303`
+
+#### Docker
+
+Or try it in Docker.
+
+In `packages/client/kintsugi` run:
+
+`docker-compose --file docker-compose.ethereumjs.yml up`
+
+## Consensus
+
+### Lodestar
+
+#### Beacon
+
+1. Use lodestar branch `master` and run `yarn && yarn build`
+1. Run cmd: `./lodestar beacon --rootDir kintsugi/temp --network=kintsugi --eth1.providerUrls=http://localhost:8545 --execution.urls=http://localhost:8545`
+
+#### Validator
+
+1. Run cmd: `./lodestar validator --rootDir=kintsugi/temp_validatordata --paramsFile=kintsugi/config/config.yaml --keystoresDir=kintsugi/keystores --secretsDir=kintsugi/secrets`
+
+### Lighthouse
+
+### Beacon
+
+1. Use lighthouse branch `kintsugi` and run `make`
+1. Make dir `lighthouse/kintsugi` and copy in from this dir: `config.yaml`, `genesis.ssz`, `deploy_block.txt`, `deposit_contract.txt`, `deposit_contract_block.txt`
+1. Run cmd: `lighthouse --debug-level=info --datadir=kintsugi/datadir --testnet-dir=kintsugi beacon_node --disable-enr-auto-update --dummy-eth1 --boot-nodes="enr:-Iq4QKuNB_wHmWon7hv5HntHiSsyE1a6cUTK1aT7xDSU_hNTLW3R4mowUboCsqYoh1kN9v3ZoSu_WuvW9Aw0tQ0Dxv6GAXxQ7Nv5gmlkgnY0gmlwhLKAlv6Jc2VjcDI1NmsxoQK6S-Cii_KmfFdUJL2TANL3ksaKUnNXvTCv1tLwXs0QgIN1ZHCCIyk" --merge --http-allow-sync-stalled --metrics --disable-packet-filter --execution-endpoints=http://127.0.0.1:8545 --terminal-total-difficulty-override=5000000000`

packages/client/kintsugi/config.yaml

@@ -0,0 +1,68 @@
+# Extends the mainnet preset
+PRESET_BASE: 'mainnet'
+# Genesis
+# ---------------------------------------------------------------
+MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 72100
+# Dec 16th, 2021, 13:00 UTC
+MIN_GENESIS_TIME: 1639659600
+# Gensis fork
+GENESIS_FORK_VERSION: 0x60000069
+# 300 seconds (5 min)
+GENESIS_DELAY: 300
+
+
+# Forking
+# ---------------------------------------------------------------
+# Some forks are disabled for now:
+#  - These may be re-assigned to another fork-version later
+#  - Temporarily set to max uint64 value: 2**64 - 1
+
+# Altair
+ALTAIR_FORK_VERSION: 0x61000070
+ALTAIR_FORK_EPOCH: 10
+# Bellatrix
+BELLATRIX_FORK_VERSION: 0x62000071
+BELLATRIX_FORK_EPOCH: 20
+TERMINAL_TOTAL_DIFFICULTY: 5000000000
+TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000
+TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615
+
+# Sharding
+SHARDING_FORK_VERSION: 0x03000000
+SHARDING_FORK_EPOCH: 18446744073709551615
+
+
+# Time parameters
+# ---------------------------------------------------------------
+# 12 seconds
+SECONDS_PER_SLOT: 12
+# 14 (estimate from Eth1 mainnet)
+SECONDS_PER_ETH1_BLOCK: 14
+# 2**8 (= 256) epochs ~27 hours
+MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256
+# 2**8 (= 256) epochs ~27 hours
+SHARD_COMMITTEE_PERIOD: 256
+# 16 blocks is ~190s
+ETH1_FOLLOW_DISTANCE: 16
+
+
+# Validator cycle
+# ---------------------------------------------------------------
+# 2**2 (= 4)
+INACTIVITY_SCORE_BIAS: 4
+# 2**4 (= 16)
+INACTIVITY_SCORE_RECOVERY_RATE: 16
+# 2**4 * 10**9 (= 16,000,000,000) Gwei
+EJECTION_BALANCE: 16000000000
+# 2**2 (= 4)
+MIN_PER_EPOCH_CHURN_LIMIT: 4
+# 2**16 (= 65,536)
+CHURN_LIMIT_QUOTIENT: 65536
+
+
+# Deposit contract
+# ---------------------------------------------------------------
+# Custom Ethereum testnet
+DEPOSIT_CHAIN_ID: 1337702
+DEPOSIT_NETWORK_ID: 1337702
+DEPOSIT_CONTRACT_ADDRESS: 0x4242424242424242424242424242424242424242

packages/client/kintsugi/config/boot_enr.yaml

@@ -0,0 +1,5 @@
+- enr:-Iq4QKuNB_wHmWon7hv5HntHiSsyE1a6cUTK1aT7xDSU_hNTLW3R4mowUboCsqYoh1kN9v3ZoSu_WuvW9Aw0tQ0Dxv6GAXxQ7Nv5gmlkgnY0gmlwhLKAlv6Jc2VjcDI1NmsxoQK6S-Cii_KmfFdUJL2TANL3ksaKUnNXvTCv1tLwXs0QgIN1ZHCCIyk
+- enr:-Ly4QOMHhGw2RIq9sHGHdSrhpjBJ421HplSN80sVyaNJkdUDSACc2_CXb1B-PJs9TR5yuV5xItUR4fcsaZZNHKCSC1QSh2F0dG5ldHOI__________-EZXRoMpDucelzYgAAcf__________gmlkgnY0gmlwhKXouc-Jc2VjcDI1NmsxoQO_Mk0rN4Q4vheqeFkz0bA9uWO8Cq51rNAxfq3rhNRKZIhzeW5jbmV0cw-DdGNwgiMog3VkcIIjKA
+- enr:-L24QNzlHJhdNf8OXLMdc5qrCI9iwEoTOJHz66YxtVStBVmXEwOFzWqFitxNfc6_ckUhhBZKWT6J-BO5JmikXu3fcOWBk4dhdHRuZXRziP__________hGV0aDKQ7nHpc2IAAHH__________4JpZIJ2NIJpcIRA4QTfiXNlY3AyNTZrMaED4DY2_MH1Q66-9YRxcxhSO5hqaUd-IGqOsMXJfcCfnW2Ic3luY25ldHMPg3RjcIIjKIN1ZHCCIyg
+- enr:-Ly4QCz_0NonV_RhGRbI8lezNtdzkK9gEvfQsvQdCiuZcqYGTuoCrRdhpKINp90_nClvi7of2Sq0AcNMk1-BMO91rqMOh2F0dG5ldHOI__________-EZXRoMpDucelzYgAAcf__________gmlkgnY0gmlwhKXosXmJc2VjcDI1NmsxoQJuQwF3fUjF432CpNvXqv5R6c4ZKr7jFX2Bzjdy9hW79IhzeW5jbmV0cw-DdGNwgiMog3VkcIIjKA
+- enr:-L24QJu98clS2Xl1L80iDeSCr4D_VDFl7S-63fmuAcxTy3RwThEy1JrHy2dkCq_7RYZz_y4D04o8C4iOIHd0Bb86PQCBgYdhdHRuZXRziO309Lg2pfflhGV0aDKQ7nHpc2IAAHH__________4JpZIJ2NIJpcIShI0tOiXNlY3AyNTZrMaEDH7tt6r2IO9tq5vSfHJtdi1wTBb37n29yVaQ2iAZSE3qIc3luY25ldHMPg3RjcIIjKIN1ZHCCIyg

packages/client/kintsugi/config/bootstrap_nodes.txt

@@ -0,0 +1,5 @@
+enr:-Iq4QKuNB_wHmWon7hv5HntHiSsyE1a6cUTK1aT7xDSU_hNTLW3R4mowUboCsqYoh1kN9v3ZoSu_WuvW9Aw0tQ0Dxv6GAXxQ7Nv5gmlkgnY0gmlwhLKAlv6Jc2VjcDI1NmsxoQK6S-Cii_KmfFdUJL2TANL3ksaKUnNXvTCv1tLwXs0QgIN1ZHCCIyk
+enr:-Ly4QOMHhGw2RIq9sHGHdSrhpjBJ421HplSN80sVyaNJkdUDSACc2_CXb1B-PJs9TR5yuV5xItUR4fcsaZZNHKCSC1QSh2F0dG5ldHOI__________-EZXRoMpDucelzYgAAcf__________gmlkgnY0gmlwhKXouc-Jc2VjcDI1NmsxoQO_Mk0rN4Q4vheqeFkz0bA9uWO8Cq51rNAxfq3rhNRKZIhzeW5jbmV0cw-DdGNwgiMog3VkcIIjKA
+enr:-L24QNzlHJhdNf8OXLMdc5qrCI9iwEoTOJHz66YxtVStBVmXEwOFzWqFitxNfc6_ckUhhBZKWT6J-BO5JmikXu3fcOWBk4dhdHRuZXRziP__________hGV0aDKQ7nHpc2IAAHH__________4JpZIJ2NIJpcIRA4QTfiXNlY3AyNTZrMaED4DY2_MH1Q66-9YRxcxhSO5hqaUd-IGqOsMXJfcCfnW2Ic3luY25ldHMPg3RjcIIjKIN1ZHCCIyg
+enr:-Ly4QCz_0NonV_RhGRbI8lezNtdzkK9gEvfQsvQdCiuZcqYGTuoCrRdhpKINp90_nClvi7of2Sq0AcNMk1-BMO91rqMOh2F0dG5ldHOI__________-EZXRoMpDucelzYgAAcf__________gmlkgnY0gmlwhKXosXmJc2VjcDI1NmsxoQJuQwF3fUjF432CpNvXqv5R6c4ZKr7jFX2Bzjdy9hW79IhzeW5jbmV0cw-DdGNwgiMog3VkcIIjKA
+enr:-L24QJu98clS2Xl1L80iDeSCr4D_VDFl7S-63fmuAcxTy3RwThEy1JrHy2dkCq_7RYZz_y4D04o8C4iOIHd0Bb86PQCBgYdhdHRuZXRziO309Lg2pfflhGV0aDKQ7nHpc2IAAHH__________4JpZIJ2NIJpcIShI0tOiXNlY3AyNTZrMaEDH7tt6r2IO9tq5vSfHJtdi1wTBb37n29yVaQ2iAZSE3qIc3luY25ldHMPg3RjcIIjKIN1ZHCCIyg