rlp: v3 updates from integration (#1648)

* rlp updates
* util: add arrToBufArr and bufArrToArr and tests
* util updates, add deprecation notice for re-exports
* rlp tsconfig: include rootDir for composite option
* remove util exports deprecation notices
* rlp: add readme note for buffer compatibility
* undo capitalization

Author: ryanio

packages/rlp/CHANGELOG.md

@@ -27,7 +27,26 @@ RLP.encode(1)
 
 ### Uint8Array
 
-Buffers were replaced in favor of using Uint8Arrays for greater compatibility with browsers.
+Buffers were replaced in favor of using Uint8Arrays for improved performance and greater compatibility with browsers.
+
+When upgrading from rlp v2 to v3, you must convert your Buffers to Uint8Arrays before passing in. To help, two new utility methods were added to `ethereumjs-util v7.1.4`: `arrToBufArr` and `bufArrToArr`. These will recursively step through your arrays to replace Buffers with Uint8Arrays, or vise versa.
+
+Example:
+
+```typescript
+// Old, rlp v2
+import * as rlp from 'rlp'
+const bufArr = [Buffer.from('123', 'hex'), Buffer.from('456', 'hex')]
+const encoded = rlp.encode(bufArr)
+const decoded = rlp.decode(encoded)
+
+// New, rlp v3
+import RLP from 'rlp'
+const encoded: Uint8Array = RLP.encode(bufArrToArr(bufArr))
+const encodedAsBuffer = Buffer.from(encoded)
+const decoded: Uint8Array[] = RLP.decode(encoded)
+const decodedAsBuffers = arrToBufArr(decoded)
+```
 
 ### Invalid RLPs
 

packages/rlp/README.md

@@ -32,6 +32,23 @@ assert.deepEqual(nestedList, decoded)
 
 `RLP.decode(encoded, [stream=false])` - Decodes an RLP encoded `Uint8Array`, `Array` or `String` and returns a `Uint8Array` or `NestedUint8Array`. If `stream` is enabled, it will just decode the first rlp sequence in the Uint8Array. By default, it would throw an error if there are more bytes in Uint8Array than used by the rlp sequence.
 
+### Buffer compatibility
+
+If you would like to continue using Buffers like in rlp v2, you can use:
+
+```typescript
+import assert from 'assert'
+import { arrToBufArr, bufArrToArr } from 'ethereumjs-util'
+import RLP from 'rlp'
+
+const bufferList = [Buffer.from('123', 'hex'), Buffer.from('456', 'hex')]
+const encoded = RLP.encode(bufArrToArr(bufferList))
+const encodedAsBuffer = Buffer.from(encoded)
+const decoded = RLP.decode(Uint8Array.from(encodedAsBuffer)) // or RLP.decode(encoded)
+const decodedAsBuffers = arrToBufArr(decoded)
+assert.deepEqual(bufferList, decodedAsBuffers)
+```
+
 ## CLI
 
 `rlp encode <JSON string>`\

packages/rlp/src/index.ts

@@ -1,16 +1,12 @@
-export type Input = string | number | bigint | Uint8Array | List | null | undefined
+export type Input = string | number | bigint | Uint8Array | Array<Input> | null | undefined
 
-// Use interface extension instead of type alias to
-// make circular declaration possible.
-export interface List extends Array<Input> {}
+export type NestedUint8Array = Array<Uint8Array | NestedUint8Array>
 
 export interface Decoded {
   data: Uint8Array | NestedUint8Array
   remainder: Uint8Array
 }
 
-export interface NestedUint8Array extends Array<Uint8Array | NestedUint8Array> {}
-
 /**
  * RLP Encoding based on https://eth.wiki/en/fundamentals/rlp
  * This function takes in data, converts it to Uint8Array if not,
@@ -51,13 +47,11 @@ function safeSlice(input: Uint8Array, start: number, end: number) {
 /**
  * Parse integers. Check if there is no leading zeros
  * @param v The value to parse
- * @param base The base to parse the integer into
  */
 function decodeLength(v: Uint8Array): number {
-  if (v[0] === 0 && v[1] === 0) {
+  if (v[0] === 0) {
     throw new Error('invalid RLP: extra zeros')
   }
-
   return parseHexByte(bytesToHex(v))
 }
 
@@ -147,12 +141,12 @@ function _decode(input: Uint8Array): Decoded {
       remainder: input.slice(length + llength),
     }
   } else if (firstByte <= 0xf7) {
-    // a list between  0-55 bytes long
+    // a list between 0-55 bytes long
     length = firstByte - 0xbf
     innerRemainder = safeSlice(input, 1, length)
     while (innerRemainder.length) {
       d = _decode(innerRemainder)
-      decoded.push(d.data as Uint8Array)
+      decoded.push(d.data)
       innerRemainder = d.remainder
     }
 
@@ -161,7 +155,7 @@ function _decode(input: Uint8Array): Decoded {
       remainder: input.slice(length),
     }
   } else {
-    // a list  over 55 bytes long
+    // a list over 55 bytes long
     llength = firstByte - 0xf6
     length = decodeLength(safeSlice(input, 1, llength))
     if (length < 56) {
@@ -176,7 +170,7 @@ function _decode(input: Uint8Array): Decoded {
 
     while (innerRemainder.length) {
       d = _decode(innerRemainder)
-      decoded.push(d.data as Uint8Array)
+      decoded.push(d.data)
       innerRemainder = d.remainder
     }
 
@@ -198,7 +192,6 @@ function bytesToHex(uint8a: Uint8Array): string {
 }
 
 function parseHexByte(hexByte: string): number {
-  if (hexByte.length !== 2) throw new Error('Invalid byte sequence')
   const byte = Number.parseInt(hexByte, 16)
   if (Number.isNaN(byte)) throw new Error('Invalid byte sequence')
   return byte

packages/rlp/tsconfig.prod.json

@@ -1,7 +1,10 @@
 {
   "extends": "../../config/tsconfig.prod.json",
   "compilerOptions": {
+    "target": "es2020",
+    "rootDir": "./src",
     "outDir": "./dist",
+    "composite": true
   },
   "include": ["src/*.ts"]
-}
+}

packages/util/package.json

@@ -6,7 +6,8 @@
   "author": "mjbecze <mjbecze@gmail.com>",
   "keywords": [
     "ethereum",
-    "utilties"
+    "utilities",
+    "utils"
   ],
   "engines": {
     "node": ">=10.0.0"

packages/util/src/bytes.ts

@@ -1,6 +1,12 @@
 import { BN } from './externals'
 import { stripHexPrefix, padToEven, isHexString, isHexPrefixed } from './internal'
-import { PrefixedHexString, TransformableToArray, TransformableToBuffer } from './types'
+import {
+  PrefixedHexString,
+  TransformableToArray,
+  TransformableToBuffer,
+  NestedBufferArray,
+  NestedUint8Array,
+} from './types'
 import { assertIsBuffer, assertIsArray, assertIsHexString } from './helpers'
 
 /**
@@ -300,3 +306,29 @@ export const validateNoLeadingZeroes = function (values: { [key: string]: Buffer
     }
   }
 }
+
+/**
+ * Converts a {@link Uint8Array} or {@link NestedUint8Array} to {@link Buffer} or {@link NestedBufferArray}
+ */
+export function arrToBufArr(arr: Uint8Array): Buffer
+export function arrToBufArr(arr: NestedUint8Array): NestedBufferArray
+export function arrToBufArr(arr: Uint8Array | NestedUint8Array): Buffer | NestedBufferArray
+export function arrToBufArr(arr: Uint8Array | NestedUint8Array): Buffer | NestedBufferArray {
+  if (!Array.isArray(arr)) {
+    return Buffer.from(arr)
+  }
+  return arr.map((a) => arrToBufArr(a))
+}
+
+/**
+ * Converts a {@link Buffer} or {@link NestedBufferArray} to {@link Uint8Array} or {@link NestedUint8Array}
+ */
+export function bufArrToArr(arr: Buffer): Uint8Array
+export function bufArrToArr(arr: NestedBufferArray): NestedUint8Array
+export function bufArrToArr(arr: Buffer | NestedBufferArray): Uint8Array | NestedUint8Array
+export function bufArrToArr(arr: Buffer | NestedBufferArray): Uint8Array | NestedUint8Array {
+  if (!Array.isArray(arr)) {
+    return Uint8Array.from(arr ?? [])
+  }
+  return arr.map((a) => bufArrToArr(a))
+}

packages/util/src/index.ts

@@ -34,7 +34,7 @@ export * from './bytes'
 export * from './object'
 
 /**
- * External exports (BN, rlp, secp256k1)
+ * External exports (BN, rlp)
  */
 export * from './externals'
 

packages/util/src/types.ts

@@ -47,6 +47,9 @@ export interface TransformableToBuffer {
   toArray?(): Uint8Array
 }
 
+export type NestedUint8Array = Array<Uint8Array | NestedUint8Array>
+export type NestedBufferArray = Array<Buffer | NestedBufferArray>
+
 /**
  * Convert BN to 0x-prefixed hex string.
  */

packages/util/test/bytes.spec.ts

@@ -1,7 +1,9 @@
 import tape from 'tape'
 import {
+  arrToBufArr,
   Address,
   BN,
+  bufArrToArr,
   zeros,
   zeroAddress,
   isZeroAddress,
@@ -397,3 +399,47 @@ tape('validateNoLeadingZeroes', function (st) {
   st.throws(() => validateNoLeadingZeroes(onlyZeroes), 'throws when value has only zeroes')
   st.end()
 })
+
+tape('arrToBufArr', function (st) {
+  const uint8 = Uint8Array.from([0, 1, 2])
+  const uint8Arr = [
+    Uint8Array.from([1, 2, 3]),
+    Uint8Array.from([4, 5, 6]),
+    [Uint8Array.from([7, 8, 9]), Uint8Array.from([1, 0, 0]), [Uint8Array.from([1, 1, 1])]],
+  ]
+  const buf = Buffer.from(uint8)
+  const bufArr = [
+    Buffer.from(Uint8Array.from([1, 2, 3])),
+    Buffer.from(Uint8Array.from([4, 5, 6])),
+    [
+      Buffer.from(Uint8Array.from([7, 8, 9])),
+      Buffer.from(Uint8Array.from([1, 0, 0])),
+      [Buffer.from(Uint8Array.from([1, 1, 1]))],
+    ],
+  ]
+  st.deepEqual(arrToBufArr(uint8), buf)
+  st.deepEqual(arrToBufArr(uint8Arr), bufArr)
+  st.end()
+})
+
+tape('bufArrToArr', function (st) {
+  const buf = Buffer.from('123', 'hex')
+  const bufArr = [
+    Buffer.from('123', 'hex'),
+    Buffer.from('456', 'hex'),
+    [Buffer.from('789', 'hex'), Buffer.from('100', 'hex'), [Buffer.from('111', 'hex')]],
+  ]
+  const uint8 = Uint8Array.from(buf)
+  const uint8Arr = [
+    Uint8Array.from(Buffer.from('123', 'hex')),
+    Uint8Array.from(Buffer.from('456', 'hex')),
+    [
+      Uint8Array.from(Buffer.from('789', 'hex')),
+      Uint8Array.from(Buffer.from('100', 'hex')),
+      [Uint8Array.from(Buffer.from('111', 'hex'))],
+    ],
+  ]
+  st.deepEqual(bufArrToArr(buf), uint8)
+  st.deepEqual(bufArrToArr(bufArr), uint8Arr)
+  st.end()
+})