feat(md): introduce Migration Data package (#3739)

Introduces the Migration Data package, a new temporary package meant for
collecting data related to the Bedrock migration. Package can be deleted
once the migration is complete.

packages/migration-data/.depcheckrc

+ignores: [
+  "@babel/eslint-parser",
+  "@typescript-eslint/parser",
+  "eslint-plugin-import",
+  "eslint-plugin-unicorn",
+  "eslint-plugin-jsdoc",
+  "eslint-plugin-prefer-arrow",
+  "eslint-plugin-react",
+  "@typescript-eslint/eslint-plugin",
+  "eslint-config-prettier",
+  "eslint-plugin-prettier",
+  "chai"
+]

packages/migration-data/.eslintrc.js

+module.exports = {
+  extends: '../../.eslintrc.js',
+}

packages/migration-data/.gitignore

+/data/evm-messages.json
+/data/slots.json

packages/migration-data/.lintstagedrc.yml

+"*.{ts,js}":
+  - eslint

packages/migration-data/.prettierrc.js

+module.exports = {
+  ...require('../../.prettierrc.js'),
+};
\ No newline at end of file

packages/migration-data/LICENSE

+(The MIT License)
+
+Copyright 2020-2021 Optimism
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

packages/migration-data/README.md

+# @eth-optimism/migration-data
+
+This package is a temporary space for scripts and tools being built to collect data for the upcoming Bedrock upgrade migration!
+We will not be publishing this package publicly as it is not meant for end-user consumption.

packages/migration-data/bin/cli.ts

+import fs from 'fs'
+
+import { Command } from 'commander'
+import { ethers } from 'ethers'
+import { getContractInterface } from '@eth-optimism/contracts'
+
+import { version } from '../package.json'
+import { advancedQueryFilter } from '../src/advanced-query'
+
+const program = new Command()
+
+program
+  .name('migration-data-query')
+  .description('CLI for querying Bedrock migration data')
+  .version(version)
+
+program
+  .command('evm-sent-messages')
+  .description('queries messages sent after the EVM upgrade')
+  .option('--rpc <rpc>', 'rpc url to use')
+  .action(async (options) => {
+    const provider = new ethers.providers.JsonRpcProvider(options.rpc)
+
+    const xdm = new ethers.Contract(
+      '0x4200000000000000000000000000000000000007',
+      getContractInterface('L2CrossDomainMessenger'),
+      provider
+    )
+
+    const sent: any[] = await advancedQueryFilter(xdm, {
+      queryFilter: xdm.filters.SentMessage(),
+    })
+
+    const messages: any[] = []
+    for (const s of sent) {
+      messages.push({
+        who: '0x4200000000000000000000000000000000000007',
+        msg: xdm.interface.encodeFunctionData('relayMessage', [
+          s.args.target,
+          s.args.sender,
+          s.args.message,
+          s.args.messageNonce,
+        ]),
+      })
+    }
+
+    fs.writeFileSync(
+      './data/evm-messages.json',
+      JSON.stringify(messages, null, 2)
+    )
+  })
+
+program
+  .command('sent-slots')
+  .description('queries storage slots in the message passer')
+  .option('--rpc <rpc>', 'rpc url to use')
+  .action(async (options) => {
+    const provider = new ethers.providers.JsonRpcProvider(options.rpc)
+
+    let nextKey = '0x'
+    let slots: any[] = []
+    while (nextKey) {
+      const latestBlock = await provider.getBlock('latest')
+      const ret = await provider.send('debug_storageRangeAt', [
+        latestBlock.hash,
+        0,
+        '0x4200000000000000000000000000000000000000',
+        nextKey,
+        10000,
+      ])
+
+      slots = slots.concat(
+        Object.values(ret.storage).map((s: any) => {
+          return s.key
+        })
+      )
+
+      // Update next key and potentially try again
+      nextKey = ret.nextKey
+    }
+
+    fs.writeFileSync('./data/slots.json', JSON.stringify(slots, null, 2))
+  })
+
+program
+  .command('accounting')
+  .description('verifies that we have sufficient slot data')
+  .action(async () => {
+    const parseMessageFile = (
+      path: string
+    ): Array<{
+      message: string
+      slot: string
+    }> => {
+      const messages: any[] = JSON.parse(fs.readFileSync(path, 'utf8'))
+      return messages.map((message) => {
+        return {
+          message,
+          slot: ethers.utils.keccak256(
+            ethers.utils.hexConcat([
+              ethers.utils.keccak256(
+                ethers.utils.hexConcat([message.msg, message.who])
+              ),
+              ethers.constants.HashZero,
+            ])
+          ),
+        }
+      })
+    }
+
+    const ovmMessages = parseMessageFile('./data/ovm-messages.json')
+    const evmMessages = parseMessageFile('./data/evm-messages.json')
+    const slotList: string[] = JSON.parse(
+      fs.readFileSync('./data/slots.json', 'utf8')
+    )
+
+    const unaccounted = slotList.filter((slot) => {
+      return (
+        !ovmMessages.some((m) => m.slot === slot) &&
+        !evmMessages.some((m) => m.slot === slot)
+      )
+    })
+
+    console.log(`Total slots: ${slotList.length}`)
+    console.log(`Unaccounted slots: ${unaccounted.length}`)
+  })
+
+program.parse(process.argv)

packages/migration-data/package.json

+{
+  "private": true,
+  "name": "@eth-optimism/migration-data",
+  "version": "0.0.1",
+  "description": "[Optimism] Data collection scripts for Bedrock migration",
+  "main": "dist/index",
+  "types": "dist/index",
+  "files": [
+    "dist/*"
+  ],
+  "scripts": {
+    "start": "ts-node ./src/service.ts",
+    "test:coverage": "echo 'No tests defined.'",
+    "build": "tsc -p ./tsconfig.json",
+    "clean": "rimraf  dist/ ./tsconfig.tsbuildinfo",
+    "lint": "yarn lint:fix && yarn lint:check",
+    "pre-commit": "lint-staged",
+    "lint:fix": "yarn lint:check --fix",
+    "lint:check": "eslint . --max-warnings=0"
+  },
+  "keywords": [
+    "optimism",
+    "ethereum",
+    "migration",
+    "bedrock"
+  ],
+  "homepage": "https://github.com/ethereum-optimism/optimism/tree/develop/packages/migration-data#readme",
+  "license": "MIT",
+  "author": "Optimism PBC",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ethereum-optimism/optimism.git"
+  },
+  "devDependencies": {
+    "commander": "^9.0.0",
+    "@eth-optimism/contracts": "0.5.37",
+    "@eth-optimism/old-contracts": "npm:@eth-optimism/contracts@0.4.10",
+    "ethers": "^5.7.0",
+    "ts-node": "^10.0.0"
+  }
+}

packages/migration-data/src/advanced-query.ts

+import { ethers } from 'ethers'
+
+/**
+ * Helper function for querying all events for a given contract/filter. Improves on the standard
+ * event querying functionality by decreasing the block range by half when a query errors out. If
+ * the query succeeds, event range will return back to the default size, and so on. Also allows
+ * more advanced filtering during the querying process to avoid OOM issues.
+ *
+ * @param contract Contract to query events for.
+ * @param options Options for the query.
+ * @returns Array of events.
+ */
+export const advancedQueryFilter = async (
+  contract: ethers.Contract,
+  options: {
+    queryFilter: ethers.EventFilter
+    filter?: (event: ethers.Event) => boolean
+    startBlock?: number
+    endBlock?: number
+  }
+): Promise<ethers.Event[]> => {
+  const defaultStep = 500000
+  const end = options.endBlock ?? (await contract.provider.getBlockNumber())
+  let step = defaultStep
+  let i = options.startBlock ?? 0
+  let events: ethers.Event[] = []
+
+  while (i < end) {
+    try {
+      const allEvents = await contract.queryFilter(
+        options.queryFilter,
+        i,
+        i + step
+      )
+      const matching = options.filter
+        ? allEvents.filter(options.filter)
+        : allEvents
+      events = events.concat(matching)
+      i += step
+      step = step * 2
+    } catch (err) {
+      step = Math.floor(step / 2)
+      if (step < 1) {
+        throw err
+      }
+    }
+  }
+
+  return events
+}

packages/migration-data/tsconfig.json

+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist"
+  },
+  "include": [
+    "src/**/*"
+  ]
+}

yarn.lock

@@ -652,6 +652,27 @@
     minimatch "^3.1.2"
     strip-json-comments "^3.1.1"
 
+"@eth-optimism/core-utils@^0.5.2":
+  version "0.5.5"
+  resolved "https://registry.yarnpkg.com/@eth-optimism/core-utils/-/core-utils-0.5.5.tgz#0e2bb95b23965fb51adfb8ba6841c3afd26a6411"
+  integrity sha512-N/uyZjHltnvnQyBOE498EGlqeYvWRUQTW6BpXhexKljEXZpnria4J4MFO9s1lJOpogLXTaS+lhM1Ic8zUNj8Pg==
+  dependencies:
+    "@ethersproject/abstract-provider" "^5.4.1"
+    ethers "^5.4.5"
+    lodash "^4.17.21"
+
+"@eth-optimism/old-contracts@npm:@eth-optimism/contracts@0.4.10":
+  version "0.4.10"
+  resolved "https://registry.yarnpkg.com/@eth-optimism/contracts/-/contracts-0.4.10.tgz#536055ae8ad5c74ea3490245a217d1029ba51699"
+  integrity sha512-DjLFp7y4HAHfn4OYGKKVx7v+zpncVeg6Iro9tPevIRK9YT+8iplfrQ1JjUqxX/XG22EglqvlQiX04l5qg4ycbg==
+  dependencies:
+    "@eth-optimism/core-utils" "^0.5.2"
+    "@ethersproject/abstract-provider" "^5.4.1"
+    "@ethersproject/abstract-signer" "^5.4.1"
+    "@ethersproject/contracts" "^5.4.1"
+    "@ethersproject/hardware-wallets" "^5.4.0"
+    glob "^7.1.6"
+
 "@ethereum-waffle/chai@^3.4.0":
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/@ethereum-waffle/chai/-/chai-3.4.0.tgz#2477877410a96bf370edd64df905b04fb9aba9d5"
@@ -1007,7 +1028,7 @@
     "@ethersproject/transactions" "^5.6.2"
     "@ethersproject/web" "^5.6.1"
 
-"@ethersproject/abstract-provider@5.7.0", "@ethersproject/abstract-provider@^5.7.0":
+"@ethersproject/abstract-provider@5.7.0", "@ethersproject/abstract-provider@^5.4.1", "@ethersproject/abstract-provider@^5.7.0":
   version "5.7.0"
   resolved "https://registry.yarnpkg.com/@ethersproject/abstract-provider/-/abstract-provider-5.7.0.tgz#b0a8550f88b6bf9d51f90e4795d48294630cb9ef"
   integrity sha512-R41c9UkchKCpAqStMYUpdunjo3pkEvZC3FAwZn5S5MGbXoMQOHIdHItezTETxAO5bevtMApSyEhn9+CHcDsWBw==
@@ -1042,7 +1063,7 @@
     "@ethersproject/logger" "^5.6.0"
     "@ethersproject/properties" "^5.6.0"
 
-"@ethersproject/abstract-signer@5.7.0", "@ethersproject/abstract-signer@^5.7.0":
+"@ethersproject/abstract-signer@5.7.0", "@ethersproject/abstract-signer@^5.4.1", "@ethersproject/abstract-signer@^5.7.0":
   version "5.7.0"
   resolved "https://registry.yarnpkg.com/@ethersproject/abstract-signer/-/abstract-signer-5.7.0.tgz#13f4f32117868452191a4649723cb086d2b596b2"
   integrity sha512-a16V8bq1/Cz+TGCkE2OPMTOUDLS3grCpdjoJCYNnVBbdYEMSgKrU0+B90s8b6H+ByYTBZN7a3g76jdIJi7UfKQ==
@@ -1232,7 +1253,7 @@
     "@ethersproject/properties" "^5.6.0"
     "@ethersproject/transactions" "^5.6.2"
 
-"@ethersproject/contracts@5.7.0", "@ethersproject/contracts@^5.7.0":
+"@ethersproject/contracts@5.7.0", "@ethersproject/contracts@^5.4.1", "@ethersproject/contracts@^5.7.0":
   version "5.7.0"
   resolved "https://registry.yarnpkg.com/@ethersproject/contracts/-/contracts-5.7.0.tgz#c305e775abd07e48aa590e1a877ed5c316f8bd1e"
   integrity sha512-5GJbzEU3X+d33CdfPhcyS+z8MzsTrBGk/sc+G+59+tPa9yFkl6HQ9D6L0QMgNTA9q8dT0XKxxkyp883XsQvbbg==
@@ -1248,7 +1269,7 @@
     "@ethersproject/properties" "^5.7.0"
     "@ethersproject/transactions" "^5.7.0"
 
-"@ethersproject/hardware-wallets@^5.7.0":
+"@ethersproject/hardware-wallets@^5.4.0", "@ethersproject/hardware-wallets@^5.7.0":
   version "5.7.0"
   resolved "https://registry.yarnpkg.com/@ethersproject/hardware-wallets/-/hardware-wallets-5.7.0.tgz#1c902fc255e2f108af44d4c1dc46ec2c34cb669c"
   integrity sha512-DjMMXIisRc8xFvEoLoYz1w7JDOYmaz/a0X9sp7Zu668RR8U1zCAyj5ow25HLRW+TCzEC5XiFetTXqS5kXonFCQ==
@@ -8821,6 +8842,42 @@ ethers@^5.0.0, ethers@^5.0.1, ethers@^5.0.2:
     "@ethersproject/web" "5.4.0"
     "@ethersproject/wordlists" "5.4.0"
 
+ethers@^5.4.5, ethers@^5.7.0:
+  version "5.7.1"
+  resolved "https://registry.yarnpkg.com/ethers/-/ethers-5.7.1.tgz#48c83a44900b5f006eb2f65d3ba6277047fd4f33"
+  integrity sha512-5krze4dRLITX7FpU8J4WscXqADiKmyeNlylmmDLbS95DaZpBhDe2YSwRQwKXWNyXcox7a3gBgm/MkGXV1O1S/Q==
+  dependencies:
+    "@ethersproject/abi" "5.7.0"
+    "@ethersproject/abstract-provider" "5.7.0"
+    "@ethersproject/abstract-signer" "5.7.0"
+    "@ethersproject/address" "5.7.0"
+    "@ethersproject/base64" "5.7.0"
+    "@ethersproject/basex" "5.7.0"
+    "@ethersproject/bignumber" "5.7.0"
+    "@ethersproject/bytes" "5.7.0"
+    "@ethersproject/constants" "5.7.0"
+    "@ethersproject/contracts" "5.7.0"
+    "@ethersproject/hash" "5.7.0"
+    "@ethersproject/hdnode" "5.7.0"
+    "@ethersproject/json-wallets" "5.7.0"
+    "@ethersproject/keccak256" "5.7.0"
+    "@ethersproject/logger" "5.7.0"
+    "@ethersproject/networks" "5.7.1"
+    "@ethersproject/pbkdf2" "5.7.0"
+    "@ethersproject/properties" "5.7.0"
+    "@ethersproject/providers" "5.7.1"
+    "@ethersproject/random" "5.7.0"
+    "@ethersproject/rlp" "5.7.0"
+    "@ethersproject/sha2" "5.7.0"
+    "@ethersproject/signing-key" "5.7.0"
+    "@ethersproject/solidity" "5.7.0"
+    "@ethersproject/strings" "5.7.0"
+    "@ethersproject/transactions" "5.7.0"
+    "@ethersproject/units" "5.7.0"
+    "@ethersproject/wallet" "5.7.0"
+    "@ethersproject/web" "5.7.1"
+    "@ethersproject/wordlists" "5.7.0"
+
 ethers@^5.5.2, ethers@^5.5.3:
   version "5.6.8"
   resolved "https://registry.yarnpkg.com/ethers/-/ethers-5.6.8.tgz#d36b816b4896341a80a8bbd2a44e8cb6e9b98dd4"
@@ -8857,42 +8914,6 @@ ethers@^5.5.2, ethers@^5.5.3:
     "@ethersproject/web" "5.6.1"
     "@ethersproject/wordlists" "5.6.1"
 
-ethers@^5.7.0:
-  version "5.7.1"
-  resolved "https://registry.yarnpkg.com/ethers/-/ethers-5.7.1.tgz#48c83a44900b5f006eb2f65d3ba6277047fd4f33"
-  integrity sha512-5krze4dRLITX7FpU8J4WscXqADiKmyeNlylmmDLbS95DaZpBhDe2YSwRQwKXWNyXcox7a3gBgm/MkGXV1O1S/Q==
-  dependencies:
-    "@ethersproject/abi" "5.7.0"
-    "@ethersproject/abstract-provider" "5.7.0"
-    "@ethersproject/abstract-signer" "5.7.0"
-    "@ethersproject/address" "5.7.0"
-    "@ethersproject/base64" "5.7.0"
-    "@ethersproject/basex" "5.7.0"
-    "@ethersproject/bignumber" "5.7.0"
-    "@ethersproject/bytes" "5.7.0"
-    "@ethersproject/constants" "5.7.0"
-    "@ethersproject/contracts" "5.7.0"
-    "@ethersproject/hash" "5.7.0"
-    "@ethersproject/hdnode" "5.7.0"
-    "@ethersproject/json-wallets" "5.7.0"
-    "@ethersproject/keccak256" "5.7.0"
-    "@ethersproject/logger" "5.7.0"
-    "@ethersproject/networks" "5.7.1"
-    "@ethersproject/pbkdf2" "5.7.0"
-    "@ethersproject/properties" "5.7.0"
-    "@ethersproject/providers" "5.7.1"
-    "@ethersproject/random" "5.7.0"
-    "@ethersproject/rlp" "5.7.0"
-    "@ethersproject/sha2" "5.7.0"
-    "@ethersproject/signing-key" "5.7.0"
-    "@ethersproject/solidity" "5.7.0"
-    "@ethersproject/strings" "5.7.0"
-    "@ethersproject/transactions" "5.7.0"
-    "@ethersproject/units" "5.7.0"
-    "@ethersproject/wallet" "5.7.0"
-    "@ethersproject/web" "5.7.1"
-    "@ethersproject/wordlists" "5.7.0"
-
 ethjs-unit@0.1.6:
   version "0.1.6"
   resolved "https://registry.yarnpkg.com/ethjs-unit/-/ethjs-unit-0.1.6.tgz#c665921e476e87bce2a9d588a6fe0405b2c41699"