regenesis-surgery: refactor tests

Update the regenesis surgery tests to use the `GenesisJsonProvider`.
Remove hardhat to test with and use `ts-mocha` instead.

packages/regenesis-surgery/package.json

@@ -10,7 +10,7 @@
     "lint:fix": "yarn lint:check --fix",
     "lint:check": "eslint .",
     "pre-commit": "lint-staged",
-    "test:surgery": "hardhat --config test/config/hardhat.config.ts test",
+    "test:surgery": "ts-mocha --timeout 50000000 test/*",
     "start": "ts-node ./scripts/surgery.ts"
   },
   "devDependencies": {
@@ -41,11 +41,11 @@
     "eslint-plugin-unicorn": "^32.0.1",
     "ethereumjs-util": "^7.1.3",
     "ethers": "^5.4.5",
-    "hardhat": "^2.6.5",
     "lint-staged": "11.0.0",
     "mocha": "^9.1.2",
     "node-fetch": "2.6.5",
     "solc": "0.8.7-fixed",
-    "ts-node": "^10.0.0"
+    "ts-node": "^10.0.0",
+    "ts-mocha": "^8.0.0"
   }
 }

packages/regenesis-surgery/test/config/hardhat.config.ts

-import { HardhatUserConfig } from 'hardhat/config'
-
-const config: HardhatUserConfig = {
-  // All paths relative to ** this file **.
-  paths: {
-    tests: '../../test',
-    cache: '../temp/cache',
-    artifacts: '../temp/artifacts',
-  },
-  mocha: {
-    timeout: 100000,
-  },
-}
-
-export default config

packages/regenesis-surgery/test/integration/delete.spec.ts → packages/regenesis-surgery/test/delete.spec.ts

 import { KECCAK256_RLP_S, KECCAK256_NULL_S } from 'ethereumjs-util'
 import { add0x } from '@eth-optimism/core-utils'
 import { ethers } from 'ethers'
-import { expect, env } from '../setup'
-import { AccountType } from '../../scripts/types'
+import { expect, env } from './setup'
+import { AccountType } from '../scripts/types'
 
 describe('deleted contracts', () => {
+  let accs
   before(async () => {
-    const accs = env.getAccountsByType(AccountType.DELETE)
+    await env.init()
+    accs = env.getAccountsByType(AccountType.DELETE)
+  })
 
+  it('accounts', async () => {
     for (const [i, acc] of accs.entries()) {
       describe(`account ${i}/${accs.length} (${acc.address})`, () => {
         it('should not have any code', async () => {
@@ -45,8 +49,4 @@ describe('deleted contracts', () => {
       })
     }
   })
-
-  // Hack for dynamically generating tests based on async data.
-  // eslint-disable-next-line @typescript-eslint/no-empty-function
-  it('stub', async () => {})
 })

packages/regenesis-surgery/test/integration/eoa.spec.ts → packages/regenesis-surgery/test/eoa.spec.ts

 import { KECCAK256_RLP_S, KECCAK256_NULL_S } from 'ethereumjs-util'
 import { add0x } from '@eth-optimism/core-utils'
-import { expect, env, NUM_ACCOUNTS_DIVISOR } from '../setup'
-import { AccountType, Account } from '../../scripts/types'
+import { expect, env } from './setup'
+import { AccountType, Account } from '../scripts/types'
 
 describe('EOAs', () => {
   describe('standard EOA', () => {
+    let eoas
     before(async () => {
-      const eoas = env.getAccountsByType(AccountType.EOA)
+      await env.init()
+      eoas = env.getAccountsByType(AccountType.EOA)
+    })
 
+    it('EOAs', () => {
       for (const [i, eoa] of eoas.entries()) {
-        if (i % NUM_ACCOUNTS_DIVISOR === 0) {
         describe(`account ${i}/${eoas.length} (${eoa.address})`, () => {
           it('should not have any code', async () => {
             const code = await env.postL2Provider.getCode(eoa.address)
@@ -35,9 +38,7 @@ describe('EOAs', () => {
             )
 
             // Balance after can come from the latest block.
-              const postBalance = await env.postL2Provider.getBalance(
-                eoa.address
-              )
+            const postBalance = await env.postL2Provider.getBalance(eoa.address)
 
             expect(preBalance).to.deep.eq(postBalance)
           })
@@ -58,12 +59,7 @@ describe('EOAs', () => {
           })
         })
       }
-      }
     })
-
-    // Hack for dynamically generating tests based on async data.
-    // eslint-disable-next-line @typescript-eslint/no-empty-function
-    it('stub', async () => {})
   })
 
   // Does not exist on Kovan?

packages/regenesis-surgery/test/provider.ts

@@ -16,6 +16,7 @@ import {
   Listener,
 } from '@ethersproject/abstract-provider'
 import { KECCAK256_RLP_S, KECCAK256_NULL_S } from 'ethereumjs-util'
+import path from 'path'
 
 import { bytes32ify, remove0x, add0x } from '@eth-optimism/core-utils'
 
@@ -63,7 +64,7 @@ export class GenesisJsonProvider implements AbstractProvider {
   constructor(dump: string | Genesis | State) {
     let input
     if (typeof dump === 'string') {
-      input = require(dump)
+      input = require(path.resolve(dump))
     } else if (typeof dump === 'object') {
       input = dump
     }
@@ -73,46 +74,73 @@ export class GenesisJsonProvider implements AbstractProvider {
     if (this.state === null) {
       throw new Error('Must initialize with genesis or state object')
     }
+
+    this._isProvider = false
   }
 
-  async getBalance(addressOrName: string): Promise<BigNumber> {
+  async getBalance(
+    addressOrName: string,
+    // eslint-disable-next-line
+    blockTag?: number | string
+  ): Promise<BigNumber> {
+    addressOrName = addressOrName.toLowerCase()
     const address = remove0x(addressOrName)
-    const account = this.state[address]
-    if (!account) {
+    const account = this.state[address] || this.state[addressOrName]
+    if (!account || account.balance === '') {
       return BigNumber.from(0)
     }
     return BigNumber.from(account.balance)
   }
 
-  async getTransactionCount(addressOrName: string): Promise<number> {
+  async getTransactionCount(
+    addressOrName: string,
+    // eslint-disable-next-line
+    blockTag?: number | string
+  ): Promise<number> {
+    addressOrName = addressOrName.toLowerCase()
     const address = remove0x(addressOrName)
-    const account = this.state[address]
+    const account = this.state[address] || this.state[addressOrName]
     if (!account) {
       return 0
     }
+    if (typeof account.nonce === 'number') {
       return account.nonce
     }
+    if (account.nonce === '') {
+      return 0
+    }
+    if (typeof account.nonce === 'string') {
+      return BigNumber.from(account.nonce).toNumber()
+    }
+    return 0
+  }
 
   async getCode(addressOrName: string): Promise<string> {
+    addressOrName = addressOrName.toLowerCase()
     const address = remove0x(addressOrName)
-    const account = this.state[address]
+    const account = this.state[address] || this.state[addressOrName]
     if (!account) {
       return '0x'
     }
+    if (typeof account.code === 'string') {
       return add0x(account.code)
     }
+    return '0x'
+  }
 
   async getStorageAt(
     addressOrName: string,
     position: BigNumber | number
   ): Promise<string> {
+    addressOrName = addressOrName.toLowerCase()
     const address = remove0x(addressOrName)
-    const account = this.state[address]
+    const account = this.state[address] || this.state[addressOrName]
     if (!account) {
       return '0x'
     }
     const bytes32 = bytes32ify(position)
-    const storage = account.storage[remove0x(bytes32)]
+    const storage =
+      account.storage[remove0x(bytes32)] || account.storage[bytes32]
     if (!storage) {
       return '0x'
     }
@@ -135,7 +163,7 @@ export class GenesisJsonProvider implements AbstractProvider {
         if (!address) {
           throw new Error('Must pass address as first arg')
         }
-        const account = this.state[remove0x(address)]
+        const account = this.state[remove0x(address)] || this.state[address]
         // The account doesn't exist or is an EOA
         if (!account || !account.code || account.code === '0x') {
           return {
@@ -144,7 +172,7 @@ export class GenesisJsonProvider implements AbstractProvider {
           }
         }
         return {
-          codeHash: ethers.utils.keccak256('0x' + account.code),
+          codeHash: ethers.utils.keccak256(add0x(account.code)),
           storageHash: add0x(account.root),
         }
       }

packages/regenesis-surgery/test/setup.ts

@@ -3,11 +3,12 @@ import chai = require('chai')
 import Mocha from 'mocha'
 import chaiAsPromised from 'chai-as-promised'
 import * as dotenv from 'dotenv'
-import { reqenv, getenv } from '@eth-optimism/core-utils'
-import { providers } from 'ethers'
+import { getenv, remove0x } from '@eth-optimism/core-utils'
+import { providers, BigNumber } from 'ethers'
 import { SurgeryDataSources, Account, AccountType } from '../scripts/types'
 import { loadSurgeryData } from '../scripts/data'
 import { classify } from '../scripts/classifiers'
+import { GenesisJsonProvider } from './provider'
 
 // Chai plugins go here.
 chai.use(chaiAsPromised)
@@ -20,16 +21,20 @@ dotenv.config()
 export const NUM_ACCOUNTS_DIVISOR = 4096
 
 interface TestEnvConfig {
-  preL2ProviderUrl: string
-  postL2ProviderUrl: string
+  preL2ProviderUrl: string | null
+  postL2ProviderUrl: string | null
+  postSurgeryGenesisFilePath: string
   stateDumpHeight: string | number
 }
 
 const config = (): TestEnvConfig => {
   const height = getenv('REGEN__STATE_DUMP_HEIGHT')
   return {
-    preL2ProviderUrl: reqenv('REGEN__PRE_L2_PROVIDER_URL'),
-    postL2ProviderUrl: reqenv('REGEN__POST_L2_PROVIDER_URL'),
+    // Optional config params for running against live nodes
+    preL2ProviderUrl: getenv('REGEN__PRE_L2_PROVIDER_URL'),
+    postL2ProviderUrl: getenv('REGEN__POST_L2_PROVIDER_URL'),
+    // File path to the post regenesis file to read
+    postSurgeryGenesisFilePath: getenv('REGEN__POST_GENESIS_FILE_PATH'),
     stateDumpHeight: parseInt(height, 10) || 'latest',
   }
 }
@@ -46,12 +51,12 @@ class TestEnv {
   // An L2 provider configured to be able to query a pre
   // regenesis L2 node. This node should be synced to the
   // height that the state dump was taken
-  preL2Provider: providers.StaticJsonRpcProvider
+  preL2Provider: providers.StaticJsonRpcProvider | GenesisJsonProvider
 
   // An L2 provider configured to be able to query a post
   // regenesis L2 node. This L2 node was initialized with
   // the results of the state surgery script
-  postL2Provider: providers.StaticJsonRpcProvider
+  postL2Provider: providers.StaticJsonRpcProvider | GenesisJsonProvider
 
   // The datasources used for doing state surgery
   surgeryDataSources: SurgeryDataSources
@@ -61,12 +66,27 @@ class TestEnv {
 
   constructor(opts: TestEnvConfig) {
     this.config = opts
+    // If the pre provider url is provided, use a json rpc provider.
+    // Otherwise, initialize a preL2Provider in the init function
+    // since it depends on suregery data sources
+    if (opts.preL2ProviderUrl) {
       this.preL2Provider = new providers.StaticJsonRpcProvider(
         opts.preL2ProviderUrl
       )
+    }
+    if (opts.postL2ProviderUrl) {
       this.postL2Provider = new providers.StaticJsonRpcProvider(
         opts.postL2ProviderUrl
       )
+    } else {
+      if (!opts.postSurgeryGenesisFilePath) {
+        throw new Error('Must configure REGEN__POST_GENESIS_FILE_PATH')
+      }
+      console.log('Using GenesisJsonProvider for postL2Provider')
+      this.postL2Provider = new GenesisJsonProvider(
+        opts.postSurgeryGenesisFilePath
+      )
+    }
   }
 
   // Read the big files from disk. Without bumping the size of the nodejs heap,
@@ -76,6 +96,40 @@ class TestEnv {
     if (this.surgeryDataSources === undefined) {
       this.surgeryDataSources = await loadSurgeryData()
 
+      if (!this.preL2Provider) {
+        console.log('Initializing pre GenesisJsonProvider...')
+        // Convert the genesis dump into a genesis file format
+        const genesis = { ...this.surgeryDataSources.genesis }
+        for (const account of this.surgeryDataSources.dump) {
+          let nonce = account.nonce
+          if (typeof nonce === 'string') {
+            if (nonce === '') {
+              nonce = 0
+            } else {
+              nonce = BigNumber.from(nonce).toNumber()
+            }
+          }
+          genesis.alloc[remove0x(account.address).toLowerCase()] = {
+            nonce,
+            balance: account.balance,
+            codeHash: remove0x(account.codeHash),
+            root: remove0x(account.root),
+            code: remove0x(account.code),
+            storage: {},
+          }
+          // Fill in the storage if it exists
+          if (account.storage) {
+            for (const [key, value] of Object.entries(account.storage)) {
+              genesis.alloc[remove0x(account.address).toLowerCase()].storage[
+                remove0x(key)
+              ] = remove0x(value)
+            }
+          }
+        }
+        // Create the pre L2 provider using the build genesis object
+        this.preL2Provider = new GenesisJsonProvider(genesis)
+      }
+
       // Classify the accounts once, this takes a while so it's better to cache it.
       console.log(`Classifying accounts...`)
       for (const account of this.surgeryDataSources.dump) {
@@ -88,6 +142,11 @@ class TestEnv {
     }
   }
 
+  // isProvider is false when it is not live
+  hasLiveProviders(): boolean {
+    return this.postL2Provider._isProvider
+  }
+
   getAccountsByType(type: AccountType) {
     return this.accounts.filter((account) => account.type === type)
   }

packages/regenesis-surgery/test/integration/uniswap.spec.ts → packages/regenesis-surgery/test/uniswap.spec.ts

 import { ethers } from 'ethers'
 import { abi as UNISWAP_POOL_ABI } from '@uniswap/v3-core/artifacts/contracts/UniswapV3Pool.sol/UniswapV3Pool.json'
-import { UNISWAP_V3_NFPM_ADDRESS } from '../../scripts/constants'
-import { getUniswapV3Factory, replaceWETH } from '../../scripts/utils'
-import { expect, env } from '../setup'
-import { AccountType } from '../../scripts/types'
+import { UNISWAP_V3_NFPM_ADDRESS } from '../scripts/constants'
+import { getUniswapV3Factory, replaceWETH } from '../scripts/utils'
+import { expect, env } from './setup'
+import { AccountType } from '../scripts/types'
 
 const ERC20_ABI = ['function balanceOf(address owner) view returns (uint256)']
 
 describe('uniswap contracts', () => {
-  describe('V3 factory', () => {
+  before(async () => {
+    await env.init()
+  })
+
+  it('V3 factory', () => {
+    if (!env.hasLiveProviders()) {
+      console.log('Cannot run factory tests without live provider')
+      return
+    }
+
     let preUniswapV3Factory: ethers.Contract
     let postUniswapV3Factory: ethers.Contract
     before(async () => {
@@ -17,12 +26,22 @@ describe('uniswap contracts', () => {
     })
 
     it('should have the same owner', async () => {
+      if (!env.hasLiveProviders()) {
+        console.log('Cannot run factory tests without live provider')
+        return
+      }
+
       const preOwner = await preUniswapV3Factory.owner()
       const postOwner = await postUniswapV3Factory.owner()
       expect(preOwner).to.equal(postOwner)
     })
 
     it('should have the same feeAmountTickSpacing map values', async () => {
+      if (!env.hasLiveProviders()) {
+        console.log('Cannot run factory tests without live provider')
+        return
+      }
+
       for (const fee of [500, 3000, 10000]) {
         const preValue = await preUniswapV3Factory.feeAmountTickSpacing(fee)
         const postValue = await postUniswapV3Factory.feeAmountTickSpacing(fee)
@@ -31,6 +50,11 @@ describe('uniswap contracts', () => {
     })
 
     it('should have the right pool addresses', async () => {
+      if (!env.hasLiveProviders()) {
+        console.log('Cannot run factory tests without live provider')
+        return
+      }
+
       for (const pool of env.surgeryDataSources.pools) {
         const remotePoolAddress1 = await postUniswapV3Factory.getPool(
           pool.token0,
@@ -49,10 +73,12 @@ describe('uniswap contracts', () => {
       }
     })
 
+    // Debug this one...
     it('should have the same code as on mainnet', async () => {
-      const l2Code = await env.postL2Provider.getCode(
+      let l2Code = await env.postL2Provider.getCode(
         postUniswapV3Factory.address
       )
+      l2Code = replaceWETH(l2Code)
       const l1Code = await env.surgeryDataSources.ethProvider.getCode(
         postUniswapV3Factory.address
       )
@@ -63,12 +89,12 @@ describe('uniswap contracts', () => {
 
   describe('V3 NFPM', () => {
     it('should have the same code as on mainnet', async () => {
-      let l2Code = await env.postL2Provider.getCode(UNISWAP_V3_NFPM_ADDRESS)
-      const l1Code = await env.surgeryDataSources.ethProvider.getCode(
+      const l2Code = await env.postL2Provider.getCode(UNISWAP_V3_NFPM_ADDRESS)
+      let l1Code = await env.surgeryDataSources.ethProvider.getCode(
         UNISWAP_V3_NFPM_ADDRESS
       )
+      l1Code = replaceWETH(l1Code)
       expect(l2Code).to.not.equal('0x')
-      l2Code = replaceWETH(l2Code)
       expect(l2Code).to.equal(l1Code)
     })
 
@@ -76,7 +102,26 @@ describe('uniswap contracts', () => {
   })
 
   describe('V3 pools', () => {
-    before(async () => {
+    it('Pools code', () => {
+      for (const pool of env.surgeryDataSources.pools) {
+        describe(`pool at address ${pool.newAddress}`, () => {
+          it('should have the same code as on testnet', async () => {
+            const l2Code = await env.postL2Provider.getCode(pool.newAddress)
+            const l1Code = await env.surgeryDataSources.ropstenProvider.getCode(
+              pool.newAddress
+            )
+            expect(l2Code).to.not.equal('0x')
+            expect(l2Code).to.equal(l1Code)
+          })
+        })
+      }
+    })
+
+    it('Pools contract', () => {
+      if (!env.hasLiveProviders()) {
+        console.log('Cannot run pool contract tests without live provider')
+        return
+      }
       for (const pool of env.surgeryDataSources.pools) {
         describe(`pool at address ${pool.newAddress}`, () => {
           let prePoolContract: ethers.Contract
@@ -172,32 +217,29 @@ describe('uniswap contracts', () => {
 
       // TODO: add a test for minting positions?
     })
-
-    // Hack for dynamically generating tests based on async data.
-    // eslint-disable-next-line @typescript-eslint/no-empty-function
-    it('stub', async () => {})
   })
 
   describe('other', () => {
+    let accs
     before(async () => {
-      const accs = env.getAccountsByType(AccountType.UNISWAP_V3_OTHER)
+      accs = env.getAccountsByType(AccountType.UNISWAP_V3_OTHER)
+    })
 
+    // TODO: for some reason these tests fail
+    it('Other uniswap contracts', () => {
       for (const acc of accs) {
         describe(`uniswap contract at address ${acc.address}`, () => {
           it('should have the same code as on mainnet', async () => {
             const l2Code = await env.postL2Provider.getCode(acc.address)
-            const l1Code = await env.surgeryDataSources.ethProvider.getCode(
+            let l1Code = await env.surgeryDataSources.ethProvider.getCode(
               acc.address
             )
+            l1Code = replaceWETH(l1Code)
             expect(l2Code).to.not.equal('0x')
             expect(l2Code).to.equal(l1Code)
           })
         })
       }
     })
-
-    // Hack for dynamically generating tests based on async data.
-    // eslint-disable-next-line @typescript-eslint/no-empty-function
-    it('stub', async () => {})
   })
 })

packages/regenesis-surgery/test/integration/verified.spec.ts → packages/regenesis-surgery/test/verified.spec.ts

 /* eslint-disable @typescript-eslint/no-empty-function */
-import { expect, env, NUM_ACCOUNTS_DIVISOR } from '../setup'
-import { AccountType } from '../../scripts/types'
+import { expect, env, NUM_ACCOUNTS_DIVISOR } from './setup'
+import { AccountType } from '../scripts/types'
 
 describe('verified', () => {
+  let verified
   before(async () => {
-    const verified = env.getAccountsByType(AccountType.VERIFIED)
+    await env.init()
+    verified = env.getAccountsByType(AccountType.VERIFIED)
+  })
 
+  it('accounts', async () => {
     for (const [i, account] of verified.entries()) {
       if (i % NUM_ACCOUNTS_DIVISOR === 0) {
         const preBytecode = await env.preL2Provider.getCode(account.address)
@@ -41,6 +45,4 @@ describe('verified', () => {
       }
     }
   })
-
-  it('stub', async () => {})
 })