core-utils: add GenesisJsonProvider and fix tests

The `GenesisJsonProvider` implements the `ethers.Provider`
interface and is constructed with a geth genesis file, either
as an object or as a file to be read from disk. It implements
a subset of the RPC methods that use the genesis file
as the backing storage. It includes tests for its correctness.
Not all methods are implemented, just the ones for the regenesis
testing.

This PR also moves the tests around in the `core-utils` package
as some of the tests were being skipped. The `tests` directory is
flattened, having so many subdirectories was not needed. The
`package.json` test script is updated to ensure that all tests
are run.

Also add some deps that are required for the `GenesisJsonProvider`.

.changeset/funny-kings-swim.md

+---
+'@eth-optimism/core-utils': patch
+---
+
+Add GenesisJsonProvider

packages/core-utils/package.json

@@ -17,8 +17,8 @@
     "lint:check": "eslint .",
     "lint:fix": "yarn lint:check --fix",
     "pre-commit": "lint-staged",
-    "test": "ts-mocha test/**/*.spec.ts",
-    "test:coverage": "nyc ts-mocha test/**/*.spec.ts && nyc merge .nyc_output coverage.json"
+    "test": "ts-mocha test/*.spec.ts",
+    "test:coverage": "nyc ts-mocha test/*.spec.ts && nyc merge .nyc_output coverage.json"
   },
   "devDependencies": {
     "@types/chai": "^4.2.18",
@@ -46,8 +46,12 @@
   },
   "dependencies": {
     "@ethersproject/abstract-provider": "^5.4.1",
+    "@ethersproject/bignumber": "^5.5.0",
+    "@ethersproject/bytes": "^5.5.0",
+    "@ethersproject/properties": "^5.5.0",
     "@ethersproject/providers": "^5.4.5",
     "chai": "^4.3.4",
+    "ethereumjs-util": "^7.1.3",
     "ethers": "^5.4.5",
     "lodash": "^4.17.21"
   }

packages/core-utils/src/common/hex-strings.ts

 /* Imports: External */
 import { BigNumber, ethers } from 'ethers'
+import { hexZeroPad } from '@ethersproject/bytes'
 
 /**
  * Removes "0x" from start of a string if it exists.
@@ -92,3 +93,7 @@ export const hexStringEquals = (stringA: string, stringB: string): boolean => {
 
   return stringA.toLowerCase() === stringB.toLowerCase()
 }
+
+export const bytes32ify = (value: number | BigNumber): string => {
+  return hexZeroPad(BigNumber.from(value).toHexString(), 32)
+}

packages/core-utils/src/common/test-utils.ts

 import { expect } from 'chai'
 import { BigNumber } from 'ethers'
-import { sleep }  from './misc'
+import { sleep } from './misc'
 
 interface deviationRanges {
   percentUpperDeviation?: number

packages/core-utils/src/index.ts

@@ -8,3 +8,4 @@ export * from './bcfg'
 export * from './fees'
 export * from './provider'
 export * from './alias'
+export * from './types'

packages/core-utils/src/provider.ts

@@ -3,7 +3,26 @@
  */
 
 import { ethers } from 'ethers'
+import { BigNumber, BigNumberish } from '@ethersproject/bignumber'
+import { Deferrable } from '@ethersproject/properties'
 import { Provider } from '@ethersproject/providers'
+import {
+  Provider as AbstractProvider,
+  EventType,
+  TransactionRequest,
+  TransactionResponse,
+  TransactionReceipt,
+  Filter,
+  Log,
+  Block,
+  BlockWithTransactions,
+  BlockTag,
+  Listener,
+} from '@ethersproject/abstract-provider'
+import { KECCAK256_RLP_S, KECCAK256_NULL_S } from 'ethereumjs-util'
+
+import { State, Genesis } from './types'
+import { bytes32ify, remove0x, add0x } from './common/hex-strings'
 
 // Copied from @ethersproject/providers since it is not
 // currently exported
@@ -38,3 +57,200 @@ export const FallbackProvider = (config: string | FallbackProviderConfig[]) => {
   }
   return new ethers.providers.FallbackProvider(config)
 }
+
+export class GenesisJsonProvider implements AbstractProvider {
+  genesis: Genesis
+
+  constructor(genesis: string | Genesis) {
+    if (typeof genesis === 'string') {
+      this.genesis = require(genesis)
+    } else if (typeof genesis === 'object') {
+      this.genesis = genesis
+    }
+
+    if (this.genesis === null) {
+      throw new Error('Must initialize with genesis object')
+    }
+  }
+
+  async getBalance(
+    addressOrName: string,
+    blockTag?: BlockTag
+  ): Promise<BigNumber> {
+    const address = remove0x(addressOrName)
+    const account = this.genesis.alloc[address]
+    if (!account) {
+      return BigNumber.from(0)
+    }
+    return BigNumber.from(account.balance)
+  }
+
+  async getTransactionCount(
+    addressOrName: string,
+    blockTag?: BlockTag
+  ): Promise<number> {
+    const address = remove0x(addressOrName)
+    const account = this.genesis.alloc[address]
+    if (!account) {
+      return 0
+    }
+    return account.nonce
+  }
+
+  async getCode(addressOrName: string, blockTag?: BlockTag): Promise<string> {
+    const address = remove0x(addressOrName)
+    const account = this.genesis.alloc[address]
+    if (!account) {
+      return '0x'
+    }
+    return add0x(account.code)
+  }
+
+  async getStorageAt(
+    addressOrName: string,
+    position: BigNumber | number,
+    blockTag?: BlockTag
+  ): Promise<string> {
+    const address = remove0x(addressOrName)
+    const account = this.genesis.alloc[address]
+    if (!account) {
+      return '0x'
+    }
+    const bytes32 = bytes32ify(position)
+    const storage = account.storage[remove0x(bytes32)]
+    if (!storage) {
+      return '0x'
+    }
+    return add0x(storage)
+  }
+
+  async call(
+    transaction: Deferrable<TransactionRequest>,
+    blockTag?: BlockTag | Promise<BlockTag>
+  ): Promise<string> {
+    throw new Error('Unsupported Method: call')
+  }
+
+  async send(method: string, args: Array<any>): Promise<any> {
+    switch (method) {
+      case 'eth_getProof': {
+        const address = args[0]
+        if (!address) {
+          throw new Error('Must pass address as first arg')
+        }
+        const account = this.genesis.alloc[remove0x(address)]
+        // The account doesn't exist or is an EOA
+        if (!account || !account.code || account.code === '0x') {
+          return {
+            codeHash: add0x(KECCAK256_NULL_S),
+            storageHash: add0x(KECCAK256_RLP_S),
+          }
+        }
+        return {
+          codeHash: ethers.utils.keccak256('0x' + account.code),
+          storageHash: add0x(account.root),
+        }
+      }
+
+      default:
+        throw new Error(`Unsupported Method: send ${method}`)
+    }
+  }
+
+  async getNetwork() {
+    return undefined
+  }
+
+  async getBlockNumber(): Promise<number> {
+    return 0
+  }
+  async getGasPrice(): Promise<BigNumber> {
+    return BigNumber.from(0)
+  }
+
+  async getFeeData() {
+    return undefined
+  }
+
+  async sendTransaction(
+    signedTransaction: string | Promise<string>
+  ): Promise<TransactionResponse> {
+    throw new Error('Unsupported Method: sendTransaction')
+  }
+
+  async estimateGas(
+    transaction: Deferrable<TransactionRequest>
+  ): Promise<BigNumber> {
+    return BigNumber.from(0)
+  }
+
+  async getBlock(
+    blockHashOrBlockTag: BlockTag | string | Promise<BlockTag | string>
+  ): Promise<Block> {
+    throw new Error('Unsupported Method: getBlock')
+  }
+  async getBlockWithTransactions(
+    blockHashOrBlockTag: BlockTag | string | Promise<BlockTag | string>
+  ): Promise<BlockWithTransactions> {
+    throw new Error('Unsupported Method: getBlockWithTransactions')
+  }
+  async getTransaction(transactionHash: string): Promise<TransactionResponse> {
+    throw new Error('Unsupported Method: getTransaction')
+  }
+  async getTransactionReceipt(
+    transactionHash: string
+  ): Promise<TransactionReceipt> {
+    throw new Error('Unsupported Method: getTransactionReceipt')
+  }
+
+  async getLogs(filter: Filter): Promise<Array<Log>> {
+    throw new Error('Unsupported Method: getLogs')
+  }
+
+  async resolveName(name: string | Promise<string>): Promise<null | string> {
+    throw new Error('Unsupported Method: resolveName')
+  }
+  async lookupAddress(
+    address: string | Promise<string>
+  ): Promise<null | string> {
+    throw new Error('Unsupported Method: lookupAddress')
+  }
+
+  on(eventName: EventType, listener: Listener): Provider {
+    throw new Error('Unsupported Method: on')
+  }
+  once(eventName: EventType, listener: Listener): Provider {
+    throw new Error('Unsupported Method: once')
+  }
+  emit(eventName: EventType, ...args: Array<any>): boolean {
+    throw new Error('Unsupported Method: emit')
+  }
+  listenerCount(eventName?: EventType): number {
+    throw new Error('Unsupported Method: listenerCount')
+  }
+  listeners(eventName?: EventType): Array<Listener> {
+    throw new Error('Unsupported Method: listeners')
+  }
+  off(eventName: EventType, listener?: Listener): Provider {
+    throw new Error('Unsupported Method: off')
+  }
+  removeAllListeners(eventName?: EventType): Provider {
+    throw new Error('Unsupported Method: removeAllListeners')
+  }
+  addListener(eventName: EventType, listener: Listener): Provider {
+    throw new Error('Unsupported Method: addListener')
+  }
+  removeListener(eventName: EventType, listener: Listener): Provider {
+    throw new Error('Unsupported Method: removeListener')
+  }
+
+  async waitForTransaction(
+    transactionHash: string,
+    confirmations?: number,
+    timeout?: number
+  ): Promise<TransactionReceipt> {
+    throw new Error('Unsupported Method: waitForTransaction')
+  }
+
+  readonly _isProvider: boolean
+}

packages/core-utils/src/types.ts

+// Optimism PBC 2021
+
+// Represents the ethereum state
+export interface State {
+  [address: string]: {
+    nonce: number
+    balance: string
+    codeHash: string
+    root: string
+    code?: string
+    storage?: {
+      [key: string]: string
+    }
+  }
+}
+
+// Represents a genesis file that geth can consume
+export interface Genesis {
+  config: {
+    chainId: number
+    homesteadBlock: number
+    eip150Block: number
+    eip155Block: number
+    eip158Block: number
+    byzantiumBlock: number
+    constantinopleBlock: number
+    petersburgBlock: number
+    istanbulBlock: number
+    muirGlacierBlock: number
+    clique: {
+      period: number
+      epoch: number
+    }
+  }
+  difficulty: string
+  gasLimit: string
+  extraData: string
+  alloc: State
+}

packages/core-utils/test/coders/batch-encoder.spec.ts → packages/core-utils/test/batch-encoder.spec.ts

-import '../setup'
+import './setup'
 
 /* Internal Imports */
 import {
   encodeAppendSequencerBatch,
   decodeAppendSequencerBatch,
   sequencerBatch,
-} from '../../src'
+} from '../src'
 import { expect } from 'chai'
 
 describe('BatchEncoder', () => {
@@ -43,7 +43,7 @@ describe('BatchEncoder', () => {
 
     it('should work with mainnet calldata', () => {
       // eslint-disable-next-line @typescript-eslint/no-var-requires
-      const data = require('../fixtures/appendSequencerBatch.json')
+      const data = require('./fixtures/appendSequencerBatch.json')
       for (const calldata of data.calldata) {
         const decoded = sequencerBatch.decode(calldata)
         const encoded = sequencerBatch.encode(decoded)

packages/core-utils/test/fees/fees.spec.ts → packages/core-utils/test/fees.spec.ts

-import { expect } from '../setup'
-import * as fees from '../../src/fees'
+import { expect } from './setup'
+import * as fees from '../src/fees'
 import { BigNumber, utils } from 'ethers'
 
 const hundredBillion = 10 ** 11

packages/core-utils/test/common/hex-utils.spec.ts → packages/core-utils/test/hex-utils.spec.ts

-import { expect } from '../setup'
+import { expect } from './setup'
 import { BigNumber } from 'ethers'
 
 /* Imports: Internal */
@@ -9,7 +9,7 @@ import {
   fromHexString,
   toHexString,
   padHexString,
-} from '../../src'
+} from '../src'
 
 describe('remove0x', () => {
   it('should return undefined', () => {

packages/core-utils/test/common/misc.spec.ts → packages/core-utils/test/misc.spec.ts

-import { expect } from '../setup'
+import { expect } from './setup'
 
 /* Imports: Internal */
-import { sleep } from '../../src'
+import { sleep } from '../src'
 
 describe('sleep', async () => {
   it('should return wait input amount of ms', async () => {

packages/core-utils/test/provider.spec.ts

+import { expect } from './setup'
+import { ethers, BigNumber } from 'ethers'
+import { GenesisJsonProvider } from '../src/provider'
+import { Genesis } from '../src/types'
+import { remove0x, add0x } from '../src/common/hex-strings'
+import { KECCAK256_RLP_S, KECCAK256_NULL_S } from 'ethereumjs-util'
+
+const account = '0x66a84544bed4ca45b3c024776812abf87728fbaf'
+
+const genesis: Genesis = {
+  config: {
+    chainId: 0,
+    homesteadBlock: 0,
+    eip150Block: 0,
+    eip155Block: 0,
+    eip158Block: 0,
+    byzantiumBlock: 0,
+    constantinopleBlock: 0,
+    petersburgBlock: 0,
+    istanbulBlock: 0,
+    muirGlacierBlock: 0,
+    clique: {
+      period: 0,
+      epoch: 0,
+    },
+  },
+  difficulty: '0x',
+  gasLimit: '0x',
+  extraData: '0x',
+  alloc: {
+    [remove0x(account)]: {
+      nonce: 101,
+      balance: '234',
+      codeHash: ethers.utils.keccak256('0x6080'),
+      root: '0x',
+      code: '6080',
+      storage: {
+        '0000000000000000000000000000000000000000000000000000000000000002':
+          '989680',
+        '0000000000000000000000000000000000000000000000000000000000000003':
+          '536f6d65205265616c6c7920436f6f6c20546f6b656e204e616d650000000036',
+        '7d55c28652d09dd36b33c69e81e67cbe8d95f51dc46ab5b17568d616d481854d':
+          '989680',
+      },
+    },
+  },
+}
+
+describe.only('GenesisJsonProvider', () => {
+  let provider
+  before(() => {
+    provider = new GenesisJsonProvider(genesis)
+  })
+
+  it('should get nonce', async () => {
+    const nonce = await provider.getTransactionCount(account)
+    expect(nonce).to.deep.eq(101)
+  })
+
+  it('should get nonce on missing account', async () => {
+    const nonce = await provider.getTransactionCount('0x')
+    expect(nonce).to.deep.eq(0)
+  })
+
+  it('should get code', async () => {
+    const code = await provider.getCode(account)
+    expect(code).to.deep.eq('0x6080')
+  })
+
+  it('should get code on missing account', async () => {
+    const code = await provider.getCode('0x')
+    expect(code).to.deep.eq('0x')
+  })
+
+  it('should get balance', async () => {
+    const balance = await provider.getBalance(account)
+    expect(balance.toString()).to.deep.eq(BigNumber.from(234).toString())
+  })
+
+  it('should get balance on missing account', async () => {
+    const balance = await provider.getBalance('0x')
+    expect(balance.toString()).to.deep.eq('0')
+  })
+
+  it('should get storage', async () => {
+    const storage = await provider.getStorageAt(account, 2)
+    expect(storage).to.deep.eq('0x989680')
+  })
+
+  it('should get storage of missing account', async () => {
+    const storage = await provider.getStorageAt('0x', 0)
+    expect(storage).to.deep.eq('0x')
+  })
+
+  it('should get storage of missing slot', async () => {
+    const storage = await provider.getStorageAt(account, 9999999999999)
+    expect(storage).to.deep.eq('0x')
+  })
+
+  it('should call eth_getProof', async () => {
+    const proof = await provider.send('eth_getProof', [account])
+    // There is code at the account, so it shouldn't be the null code hash
+    expect(proof.codeHash).to.not.eq(add0x(KECCAK256_NULL_S))
+    // There is storage so it should not be the null storage hash
+    expect(proof.storageHash).to.not.eq(add0x(KECCAK256_RLP_S))
+  })
+
+  it('should call eth_getProof on missing account', async () => {
+    const proof = await provider.send('eth_getProof', ['0x'])
+    expect(proof.codeHash).to.eq(add0x(KECCAK256_NULL_S))
+    expect(proof.storageHash).to.eq(add0x(KECCAK256_RLP_S))
+  })
+})

packages/core-utils/test/common/test-utils.spec.ts → packages/core-utils/test/test-utils.spec.ts

-import { expect } from '../setup'
+import { expect } from './setup'
 
 /* Imports: Internal */
-import { expectApprox, awaitCondition } from '../../src'
+import { expectApprox, awaitCondition } from '../src'
 import { assert } from 'chai'
 
 describe('awaitCondition', () => {
@@ -12,7 +12,7 @@ describe('awaitCondition', () => {
       return Promise.resolve(i === 2)
     }
 
-    await awaitCondition(condFn, 50, 3);
+    await awaitCondition(condFn, 50, 3)
     expect(i).to.equal(2)
   })
 
@@ -24,12 +24,12 @@ describe('awaitCondition', () => {
     }
 
     try {
-      await awaitCondition(condFn, 50, 1);
+      await awaitCondition(condFn, 50, 1)
     } catch (e) {
-      return;
+      return
     }
 
-    assert.fail('Condition never failed, but it should have.');
+    assert.fail('Condition never failed, but it should have.')
   })
 })
 

yarn.lock

@@ -792,6 +792,15 @@
     "@ethersproject/logger" "^5.4.0"
     bn.js "^4.11.9"
 
+"@ethersproject/bignumber@^5.5.0":
+  version "5.5.0"
+  resolved "https://registry.yarnpkg.com/@ethersproject/bignumber/-/bignumber-5.5.0.tgz#875b143f04a216f4f8b96245bde942d42d279527"
+  integrity sha512-6Xytlwvy6Rn3U3gKEc1vP7nR92frHkv6wtVr95LFR3jREXiCPzdWxKQ1cx4JGQBXxcguAwjA8murlYN2TSiEbg==
+  dependencies:
+    "@ethersproject/bytes" "^5.5.0"
+    "@ethersproject/logger" "^5.5.0"
+    bn.js "^4.11.9"
+
 "@ethersproject/bytes@5.4.0", "@ethersproject/bytes@>=5.0.0-beta.129", "@ethersproject/bytes@^5.0.0", "@ethersproject/bytes@^5.0.4", "@ethersproject/bytes@^5.4.0":
   version "5.4.0"
   resolved "https://registry.yarnpkg.com/@ethersproject/bytes/-/bytes-5.4.0.tgz#56fa32ce3bf67153756dbaefda921d1d4774404e"
@@ -799,6 +808,13 @@
   dependencies:
     "@ethersproject/logger" "^5.4.0"
 
+"@ethersproject/bytes@^5.5.0":
+  version "5.5.0"
+  resolved "https://registry.yarnpkg.com/@ethersproject/bytes/-/bytes-5.5.0.tgz#cb11c526de657e7b45d2e0f0246fb3b9d29a601c"
+  integrity sha512-ABvc7BHWhZU9PNM/tANm/Qx4ostPGadAuQzWTr3doklZOhDlmcBqclrQe/ZXUIj3K8wC28oYeuRa+A37tX9kog==
+  dependencies:
+    "@ethersproject/logger" "^5.5.0"
+
 "@ethersproject/constants@5.4.0", "@ethersproject/constants@>=5.0.0-beta.128", "@ethersproject/constants@^5.0.0", "@ethersproject/constants@^5.0.4", "@ethersproject/constants@^5.4.0":
   version "5.4.0"
   resolved "https://registry.yarnpkg.com/@ethersproject/constants/-/constants-5.4.0.tgz#ee0bdcb30bf1b532d2353c977bf2ef1ee117958a"
@@ -898,6 +914,11 @@
   resolved "https://registry.yarnpkg.com/@ethersproject/logger/-/logger-5.4.0.tgz#f39adadf62ad610c420bcd156fd41270e91b3ca9"
   integrity sha512-xYdWGGQ9P2cxBayt64d8LC8aPFJk6yWCawQi/4eJ4+oJdMMjEBMrIcIMZ9AxhwpPVmnBPrsB10PcXGmGAqgUEQ==
 
+"@ethersproject/logger@^5.5.0":
+  version "5.5.0"
+  resolved "https://registry.yarnpkg.com/@ethersproject/logger/-/logger-5.5.0.tgz#0c2caebeff98e10aefa5aef27d7441c7fd18cf5d"
+  integrity sha512-rIY/6WPm7T8n3qS2vuHTUBPdXHl+rGxWxW5okDfo9J4Z0+gRRZT0msvUdIJkE4/HS29GUMziwGaaKO2bWONBrg==
+
 "@ethersproject/networks@5.4.2", "@ethersproject/networks@^5.0.0", "@ethersproject/networks@^5.4.0":
   version "5.4.2"
   resolved "https://registry.yarnpkg.com/@ethersproject/networks/-/networks-5.4.2.tgz#2247d977626e97e2c3b8ee73cd2457babde0ce35"
@@ -920,6 +941,13 @@
   dependencies:
     "@ethersproject/logger" "^5.4.0"
 
+"@ethersproject/properties@^5.5.0":
+  version "5.5.0"
+  resolved "https://registry.yarnpkg.com/@ethersproject/properties/-/properties-5.5.0.tgz#61f00f2bb83376d2071baab02245f92070c59995"
+  integrity sha512-l3zRQg3JkD8EL3CPjNK5g7kMx4qSwiR60/uk5IVjd3oq1MZR5qUg40CNOoEJoX5wc3DyY5bt9EbMk86C7x0DNA==
+  dependencies:
+    "@ethersproject/logger" "^5.5.0"
+
 "@ethersproject/providers@5.4.4", "@ethersproject/providers@^5.0.0":
   version "5.4.4"
   resolved "https://registry.yarnpkg.com/@ethersproject/providers/-/providers-5.4.4.tgz#6729120317942fc0ab0ecdb35e944ec6bbedb795"
@@ -6824,6 +6852,17 @@ ethereumjs-util@^7.1.2:
     ethjs-util "0.1.6"
     rlp "^2.2.4"
 
+ethereumjs-util@^7.1.3:
+  version "7.1.3"
+  resolved "https://registry.yarnpkg.com/ethereumjs-util/-/ethereumjs-util-7.1.3.tgz#b55d7b64dde3e3e45749e4c41288238edec32d23"
+  integrity sha512-y+82tEbyASO0K0X1/SRhbJJoAlfcvq8JbrG4a5cjrOks7HS/36efU/0j2flxCPOUM++HFahk33kr/ZxyC4vNuw==
+  dependencies:
+    "@types/bn.js" "^5.1.0"
+    bn.js "^5.1.2"
+    create-hash "^1.1.2"
+    ethereum-cryptography "^0.1.3"
+    rlp "^2.2.4"
+
 ethereumjs-vm@4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/ethereumjs-vm/-/ethereumjs-vm-4.2.0.tgz#e885e861424e373dbc556278f7259ff3fca5edab"