Stream dependency replacement with walk all value nodes

packages/statemanager/src/stateManager.ts

@@ -9,7 +9,6 @@ import {
   KECCAK256_RLP,
   KECCAK256_RLP_S,
   bigIntToHex,
-  bytesToBigInt,
   bytesToHex,
   bytesToUnprefixedHex,
   concatBytes,
@@ -231,7 +230,6 @@ export class DefaultStateManager implements EVMStateManagerInterface {
       type: opts.accountCacheOpts?.type ?? CacheType.ORDERED_MAP,
       size: opts.accountCacheOpts?.size ?? 100000,
     }
-
     if (!this._accountCacheSettings.deactivate) {
       this._accountCache = new AccountCache({
         size: this._accountCacheSettings.size,
@@ -245,7 +243,6 @@ export class DefaultStateManager implements EVMStateManagerInterface {
       type: opts.storageCacheOpts?.type ?? CacheType.ORDERED_MAP,
       size: opts.storageCacheOpts?.size ?? 20000,
     }
-
     if (!this._storageCacheSettings.deactivate) {
       this._storageCache = new StorageCache({
         size: this._storageCacheSettings.size,
@@ -259,7 +256,6 @@ export class DefaultStateManager implements EVMStateManagerInterface {
       type: opts.codeCacheOpts?.type ?? CacheType.ORDERED_MAP,
       size: opts.codeCacheOpts?.size ?? 20000,
     }
-
     if (!this._codeCacheSettings.deactivate) {
       this._codeCache = new CodeCache({
         size: this._codeCacheSettings.size,
@@ -991,19 +987,8 @@ export class DefaultStateManager implements EVMStateManagerInterface {
     }
     const trie = this._getStorageTrie(address, account)
 
-    return new Promise((resolve, reject) => {
-      const storage: StorageDump = {}
-      const stream = trie.createReadStream()
-
-      stream.on('data', (val: any) => {
-        storage[bytesToHex(val.key)] = bytesToHex(val.value)
-      })
-      stream.on('end', () => {
-        resolve(storage)
-      })
-      stream.on('error', (e) => {
-        reject(e)
-      })
+    return trie.getValueMap().then((value) => {
+      return value.values
     })
   }
 
@@ -1026,44 +1011,24 @@ export class DefaultStateManager implements EVMStateManagerInterface {
     if (!account) {
       throw new Error(`Account does not exist.`)
     }
-    const trie = this._getStorageTrie(address, account)
 
-    return new Promise((resolve, reject) => {
-      let inRange = false
-      let i = 0
-
-      /** Object conforming to {@link StorageRange.storage}. */
-      const storageMap: StorageRange['storage'] = {}
-      const stream = trie.createReadStream()
-
-      stream.on('data', (val: any) => {
-        if (!inRange) {
-          // Check if the key is already in the correct range.
-          if (bytesToBigInt(val.key) >= startKey) {
-            inRange = true
-          } else {
-            return
-          }
-        }
+    const trie = this._getStorageTrie(address, account)
 
-        if (i < limit) {
-          storageMap[bytesToHex(val.key)] = { key: null, value: bytesToHex(val.value) }
-          i++
-        } else if (i === limit) {
-          resolve({
-            storage: storageMap,
-            nextKey: bytesToHex(val.key),
-          })
+    return trie.getValueMap(startKey, limit).then((value) => {
+      const values = value.values
+      const dump = Object.create(null)
+      for (const key of Object.keys(values)) {
+        const val = values[key]
+        dump[key] = {
+          key: null,
+          value: val,
         }
-      })
+      }
 
-      stream.on('end', () => {
-        resolve({
-          storage: storageMap,
-          nextKey: null,
-        })
-      })
-      stream.on('error', (e) => reject(e))
+      return {
+        storage: dump,
+        nextKey: value.nextKey,
+      }
     })
   }
 

packages/trie/package.json

@@ -59,8 +59,7 @@
     "@types/readable-stream": "^2.3.13",
     "debug": "^4.3.4",
     "lru-cache": "10.1.0",
-    "ethereum-cryptography": "^2.2.1",
-    "readable-stream": "^3.6.0"
+    "ethereum-cryptography": "^2.2.1"
   },
   "devDependencies": {
     "@ethereumjs/genesis": "^0.2.2",

packages/trie/src/trie.ts

@@ -3,11 +3,13 @@
 
 import { RLP } from '@ethereumjs/rlp'
 import {
+  BIGINT_0,
   KeyEncoding,
   Lock,
   MapDB,
   RLP_EMPTY_STRING,
   ValueEncoding,
+  bytesToBigInt,
   bytesToHex,
   bytesToUnprefixedHex,
   bytesToUtf8,
@@ -29,8 +31,8 @@ import {
 import { verifyRangeProof } from './proof/range.js'
 import { ROOT_DB_KEY } from './types.js'
 import { _walkTrie } from './util/asyncWalk.js'
+import { nibbleTypeToPackedBytes } from './util/encoding.js'
 import { bytesToNibbles, matchingNibbleLength } from './util/nibbles.js'
-import { TrieReadStream as ReadStream } from './util/readStream.js'
 import { WalkController } from './util/walkController.js'
 
 import type {
@@ -1081,14 +1083,6 @@ export class Trie {
     return true
   }
 
-  /**
-   * The `data` event is given an `Object` that has two properties; the `key` and the `value`. Both should be Uint8Arrays.
-   * @return Returns a [stream](https://nodejs.org/dist/latest-v12.x/docs/api/stream.html#stream_class_stream_readable) of the contents of the `trie`
-   */
-  createReadStream(): ReadStream {
-    return new ReadStream(this)
-  }
-
   /**
    * Returns a copy of the underlying trie.
    *
@@ -1227,4 +1221,45 @@ export class Trie {
       this.debug(`Deleting ${this._db.checkpoints.length} checkpoints.`, ['FLUSH_CHECKPOINTS'])
     this._db.checkpoints = []
   }
+
+  /**
+   * Returns a list of values stored in the trie
+   * @param startKey first unhashed key in the range to be returned (defaults to 0)
+   * @param limit - the number of keys to be returned (undefined means all keys)
+   * @returns an object with two properties (a map of all key/value pairs in the trie - or in the specified range) and then a `nextKey` reference if a range is specified
+   */
+  async getValueMap(
+    startKey = BIGINT_0,
+    limit?: number
+  ): Promise<{ values: { [key: string]: string }; nextKey: null | string }> {
+    // If limit is undefined, all keys are inRange
+    let inRange = limit !== undefined ? false : true
+    let i = 0
+    const values: { [key: string]: string } = {}
+    let nextKey: string | null = null
+    await this.walkAllValueNodes(async (node: TrieNode, currentKey: number[]) => {
+      if (node instanceof LeafNode) {
+        const keyBytes = nibbleTypeToPackedBytes(currentKey.concat(node.key()))
+        if (!inRange) {
+          // Check if the key is already in the correct range.
+          if (bytesToBigInt(keyBytes) >= startKey) {
+            inRange = true
+          } else {
+            return
+          }
+        }
+
+        if (limit === undefined || i < limit) {
+          values[bytesToHex(keyBytes)] = bytesToHex(node._value)
+          i++
+        } else if (i === limit) {
+          nextKey = bytesToHex(keyBytes)
+        }
+      }
+    })
+    return {
+      values,
+      nextKey,
+    }
+  }
 }

packages/trie/src/util/index.ts

@@ -1,5 +1,4 @@
 export * from './encoding.js'
 export * from './genesisState.js'
-export * from './readStream.js'
 export * from './tasks.js'
 export * from './walkController.js'