feat: import the message relayer service (#36)

* feat: import the message relayer service

https://github.com/ethereum-optimism/optimism-ts-services/blob/1c6f8ad07afadc37ace76c9c63635d9b5132f6bd/src/services/message-relayer.service.ts

* chore: clean up unused types

* fix: do not try to run relayer unit tests

* ops(message-relayer): add docker image and compose settings

* test: add L1 <> L2 communication

* ci: parallelize building steps

* ci: try to resolve hardhat race condition

.github/workflows/integration.yml

@@ -15,18 +15,6 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Build the base layer dependencies
-        working-directory: ./ops
-        run: docker-compose build --parallel -- builder l2geth l1_chain
-
-      - name: Build the other services
-        working-directory: ./ops
-        run: docker-compose build --parallel -- deployer dtl batch_submitter
-
-      - name: Bring the stack up and wait for the sequencer to be ready
-        working-directory: ./ops
-        run: docker-compose up -d && ./scripts/wait-for-sequencer.sh
-
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "::set-output name=dir::$(yarn cache dir)"
@@ -39,14 +27,17 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-yarn-
 
-      - name: Install integration test dependencies
-        working-directory: ./integration-tests
-        # if: steps.yarn-cache.outputs.cache-hit != 'true'
-        run: yarn install
+      - name: Build the services
+        working-directory: ./ops
+        run: ./scripts/build-ci.sh
 
-      - name: Build deps for the integration tests
-        run: yarn build
+      - name: Bring the stack up and wait for the sequencer to be ready
+        working-directory: ./ops
+        run: docker-compose up -d && ./scripts/wait-for-sequencer.sh
 
       - name: Run the integration tests
         working-directory: ./integration-tests
-        run: yarn test:integration
+        run: |
+            yarn build:contracts
+            yarn build:contracts:ovm
+            yarn test:integration

integration-tests/contracts/SimpleStorage.sol

+pragma solidity >=0.7.0;
+
+contract ICrossDomainMessenger {
+    address public xDomainMessageSender;
+}
+
+contract SimpleStorage {
+    address public msgSender;
+    address public xDomainSender;
+    bytes32 public value;
+    uint256 public totalCount;
+
+    function setValue(bytes32 newValue) public {
+        msgSender = msg.sender;
+        xDomainSender = ICrossDomainMessenger(msg.sender)
+            .xDomainMessageSender();
+        value = newValue;
+        totalCount++;
+    }
+
+    function dumbSetValue(bytes32 newValue) public {
+        value = newValue;
+    }
+}

integration-tests/package.json

 {
-  "name": "@eth-optimism/sequencer-interactions",
+  "name": "@eth-optimism/integration-tests",
   "version": "0.0.1",
-  "description": "[Optimism] Integration Tests: Sequencer Interactions",
+  "description": "[Optimism] Integration Tests",
   "author": "Optimism PBC",
   "license": "MIT",
   "scripts": {
     "lint": "yarn lint:fix && yarn lint:check",
     "lint:check": "tslint --format stylish --project .",
     "lint:fix": "prettier --config ./prettier-config.json --write 'test/**/*.ts'",
-    "test:integration": "hardhat --network optimism test"
+    "build:integration": "./scripts/build.sh",
+    "build:contracts": "hardhat compile",
+    "build:contracts:ovm": "hardhat compile --network optimism",
+    "test:integration": "hardhat --network optimism test",
+    "clean": "rimraf cache artifacts artifacts-ovm cache-ovm"
   },
   "devDependencies": {
     "@eth-optimism/contracts": "^0.1.11",
@@ -21,6 +25,7 @@
     "ethers": "^5.0.32",
     "hardhat": "^2.1.2",
     "lodash": "^4.17.21",
-    "mocha": "^8.3.1"
+    "mocha": "^8.3.1",
+    "rimraf": "^3.0.2"
   }
 }

integration-tests/scripts/build.sh

+yarn build:contracts &
+yarn build:contracts:ovm &
+wait

integration-tests/test/basic-l1-l2-communication.spec.ts

+import { expect } from 'chai'
+
+/* Imports: External */
+import { Contract, ContractFactory, Wallet, providers } from 'ethers'
+import { Watcher } from '@eth-optimism/core-utils'
+import { getContractFactory } from '@eth-optimism/contracts'
+
+/* Imports: Internal */
+import l1SimpleStorageJson from '../artifacts/contracts/SimpleStorage.sol/SimpleStorage.json'
+import l2SimpleStorageJson from '../artifacts-ovm/contracts/SimpleStorage.sol/SimpleStorage.json'
+
+describe('Basic L1<>L2 Communication', async () => {
+  let l1Wallet: Wallet
+  let l2Wallet: Wallet
+  let l1Provider: providers.JsonRpcProvider
+  let l2Provider: providers.JsonRpcProvider
+  let AddressManager: Contract
+
+  let Factory__L1SimpleStorage: ContractFactory
+  let Factory__L2SimpleStorage: ContractFactory
+  let L1CrossDomainMessenger: Contract
+  let L2CrossDomainMessenger: Contract
+
+  let watcher: Watcher
+
+  let L1SimpleStorage: Contract
+  let L2SimpleStorage: Contract
+
+  before(async () => {
+    const httpPort = 8545
+    const l1HttpPort = 9545
+    l1Provider = new providers.JsonRpcProvider(`http://localhost:${l1HttpPort}`)
+    l2Provider = new providers.JsonRpcProvider(`http://localhost:${httpPort}`)
+    l1Wallet = new Wallet(
+      '0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80',
+      l1Provider
+    )
+    l2Wallet = Wallet.createRandom().connect(l2Provider)
+
+    const addressManagerAddress = '0x5FbDB2315678afecb367f032d93F642f64180aa3'
+    AddressManager = getContractFactory('Lib_AddressManager')
+      .connect(l1Wallet)
+      .attach(addressManagerAddress)
+
+    Factory__L1SimpleStorage = new ContractFactory(
+      l1SimpleStorageJson.abi,
+      l1SimpleStorageJson.bytecode,
+      l1Wallet
+    )
+    Factory__L2SimpleStorage = new ContractFactory(
+      l2SimpleStorageJson.abi,
+      l2SimpleStorageJson.bytecode,
+      l2Wallet
+    )
+
+    const l1MessengerAddress = await AddressManager.getAddress(
+      'Proxy__OVM_L1CrossDomainMessenger'
+    )
+    const l2MessengerAddress = await AddressManager.getAddress(
+      'OVM_L2CrossDomainMessenger'
+    )
+
+    L1CrossDomainMessenger = getContractFactory('iOVM_L1CrossDomainMessenger')
+      .connect(l1Wallet)
+      .attach(l1MessengerAddress)
+
+    L2CrossDomainMessenger = getContractFactory('iOVM_L2CrossDomainMessenger')
+      .connect(l2Wallet)
+      .attach(l2MessengerAddress)
+
+    watcher = new Watcher({
+      l1: {
+        provider: l1Provider,
+        messengerAddress: L1CrossDomainMessenger.address,
+      },
+      l2: {
+        provider: l2Provider,
+        messengerAddress: L2CrossDomainMessenger.address,
+      },
+    })
+  })
+
+  beforeEach(async () => {
+    L1SimpleStorage = await Factory__L1SimpleStorage.deploy()
+    await L1SimpleStorage.deployTransaction.wait()
+    L2SimpleStorage = await Factory__L2SimpleStorage.deploy()
+    await L2SimpleStorage.deployTransaction.wait()
+  })
+
+  it('should withdraw from L2 -> L1', async () => {
+    const value = `0x${'77'.repeat(32)}`
+
+    // Send L2 -> L1 message.
+    const transaction = await L2CrossDomainMessenger.sendMessage(
+      L1SimpleStorage.address,
+      L1SimpleStorage.interface.encodeFunctionData('setValue', [value]),
+      5000000,
+      { gasLimit: 7000000 }
+    )
+
+    // Wait for the L2 transaction to be mined.
+    await transaction.wait()
+
+    // Wait for the transaction to be relayed on L1.
+    const messageHashes = await watcher.getMessageHashesFromL2Tx(
+      transaction.hash
+    )
+    await watcher.getL1TransactionReceipt(messageHashes[0])
+
+    expect(await L1SimpleStorage.msgSender()).to.equal(
+      L1CrossDomainMessenger.address
+    )
+    expect(await L1SimpleStorage.xDomainSender()).to.equal(l2Wallet.address)
+    expect(await L1SimpleStorage.value()).to.equal(value)
+    expect((await L1SimpleStorage.totalCount()).toNumber()).to.equal(1)
+  })
+
+  it('should deposit from L1 -> L2', async () => {
+    const value = `0x${'42'.repeat(32)}`
+
+    // Send L1 -> L2 message.
+    const transaction = await L1CrossDomainMessenger.sendMessage(
+      L2SimpleStorage.address,
+      L2SimpleStorage.interface.encodeFunctionData('setValue', [value]),
+      5000000,
+      { gasLimit: 7000000 }
+    )
+
+    // Wait for the L1 transaction to be mined.
+    await transaction.wait()
+
+    // Wait for the transaction to be included on L2.
+    const messageHashes = await watcher.getMessageHashesFromL1Tx(
+      transaction.hash
+    )
+    await watcher.getL2TransactionReceipt(messageHashes[0])
+
+    expect(await L2SimpleStorage.msgSender()).to.equal(
+      L2CrossDomainMessenger.address
+    )
+    expect(await L2SimpleStorage.xDomainSender()).to.equal(l1Wallet.address)
+    expect(await L2SimpleStorage.value()).to.equal(value)
+    expect((await L2SimpleStorage.totalCount()).toNumber()).to.equal(1)
+  })
+})

ops/docker-compose.yml

@@ -92,3 +92,17 @@ services:
     ports:
       - ${L2GETH_HTTP_PORT:-8545}:8545
       - ${L2GETH_WS_PORT:-8546}:8546
+
+  relayer:
+    image: ethereumoptimism:relayer
+    build:
+      context: ..
+      dockerfile: ./ops/docker/Dockerfile.relayer
+    entrypoint: ./relayer.sh
+    environment:
+        L1_NODE_WEB3_URL: http://l1_chain:8545
+        L2_NODE_WEB3_URL: http://l2geth:8545
+        URL: http://deployer:8081/addresses.json
+        # a funded hardhat account
+        L1_WALLET_KEY: "0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97"
+        RETRIES: 60

ops/docker/Dockerfile.monorepo

@@ -30,6 +30,7 @@ COPY packages/smock/package.json ./packages/smock/package.json
 COPY packages/contracts/package.json ./packages/contracts/package.json
 COPY packages/data-transport-layer/package.json ./packages/data-transport-layer/package.json
 COPY packages/batch-submitter/package.json ./packages/batch-submitter/package.json
+COPY packages/message-relayer/package.json ./packages/message-relayer/package.json
 
 RUN yarn install --frozen-lockfile
 

ops/docker/Dockerfile.relayer

+FROM ethereumoptimism:builder AS builder
+
+FROM node:14-alpine
+
+RUN apk add --no-cache curl bash jq
+
+WORKDIR /opt/optimism
+
+# copy top level files
+COPY --from=builder /optimism/*.json ./
+COPY --from=builder /optimism/yarn.lock .
+COPY --from=builder /optimism/node_modules ./node_modules
+
+# copy deps (would have been nice if docker followed the symlinks required)
+COPY --from=builder /optimism/packages/core-utils/package.json ./packages/core-utils/package.json
+COPY --from=builder /optimism/packages/core-utils/dist ./packages/core-utils/dist
+
+COPY --from=builder /optimism/packages/contracts/package.json ./packages/contracts/package.json
+COPY --from=builder /optimism/packages/contracts/dist ./packages/contracts/dist
+COPY --from=builder /optimism/packages/contracts/artifacts ./packages/contracts/artifacts
+COPY --from=builder /optimism/packages/contracts/artifacts-ovm ./packages/contracts/artifacts-ovm
+
+# copy the service
+WORKDIR /opt/optimism/packages/message-relayer
+COPY --from=builder /optimism/packages/message-relayer/dist ./dist
+COPY --from=builder /optimism/packages/message-relayer/package.json .
+COPY --from=builder /optimism/packages/message-relayer/exec ./exec
+COPY --from=builder /optimism/packages/message-relayer/node_modules ./node_modules
+
+# copy this over in case you want to run alongside other services
+COPY ./ops/scripts/relayer.sh .
+ENTRYPOINT ["node", "exec/run-message-relayer.js"]

ops/envs/batches.env

@@ -5,6 +5,7 @@ DEBUG=info*,error*,warn*,debug*
 MAX_L1_TX_SIZE=90000
 MIN_L1_TX_SIZE=0
 MAX_TX_BATCH_COUNT=50
+MAX_STATE_BATCH_COUNT=50
 
 POLL_INTERVAL=15000
 NUM_CONFIRMATIONS=0

ops/scripts/build-ci.sh

+# build in 2 steps
+function build_images() {
+    docker-compose build --parallel -- builder l2geth l1_chain
+    docker-compose build --parallel -- deployer dtl batch_submitter relayer
+}
+
+function build_dependencies() {
+    yarn
+    yarn build
+}
+
+build_images &
+build_dependencies &
+
+wait

ops/scripts/relayer.sh

+#!/bin/bash
+RETRIES=${RETRIES:-20}
+
+# get the addrs from the URL provided
+ADDRESSES=$(curl --retry-connrefused --retry $RETRIES --retry-delay 3 $URL)
+# set the env
+export ADDRESS_MANAGER_ADDRESS=$(echo $ADDRESSES | jq -r '.AddressManager')
+
+# waits for l2geth to be up
+curl --retry-connrefused --retry $RETRIES --retry-delay 1 $L2_NODE_WEB3_URL
+
+# go
+node ./exec/run-message-relayer.js

packages/contracts/src/contract-defs.ts

 import * as path from 'path'
 import * as glob from 'glob'
-import { ethers, ContractFactory, Signer } from 'ethers'
+import {
+  ethers,
+  ContractFactory,
+  Signer,
+  providers,
+  Contract,
+  constants,
+} from 'ethers'
 import { Interface } from 'ethers/lib/utils'
 
 export const getContractDefinition = (name: string, ovm?: boolean): any => {
@@ -33,3 +40,29 @@ export const getContractFactory = (
   const contractInterface = getContractInterface(name, ovm)
   return new ContractFactory(contractInterface, definition.bytecode, signer)
 }
+
+export const loadContract = (
+  name: string,
+  address: string,
+  provider: providers.JsonRpcProvider
+): Contract => {
+  return new Contract(address, getContractInterface(name) as any, provider)
+}
+
+export const loadContractFromManager = async (args: {
+  name: string
+  proxy?: string
+  Lib_AddressManager: Contract
+  provider: providers.JsonRpcProvider
+}): Promise<Contract> => {
+  const { name, proxy, Lib_AddressManager, provider } = args
+  const address = await Lib_AddressManager.getAddress(proxy ? proxy : name)
+
+  if (address === constants.AddressZero) {
+    throw new Error(
+      `Lib_AddressManager does not have a record for a contract named: ${name}`
+    )
+  }
+
+  return loadContract(name, address, provider)
+}

packages/message-relayer/exec/run-message-relayer.js

+#!/usr/bin/env node
+
+const main = require("../dist/exec/run").default
+
+;(async () => {
+  await main()
+})().catch((err) => {
+  console.log(err)
+  process.exit(1)
+})

packages/message-relayer/package.json

+{
+  "name": "@eth-optimism/message-relayer",
+  "version": "0.0.1",
+  "description": "[Optimism] Cross Domain Message Relayer service",
+  "main": "dist/index",
+  "types": "dist/index",
+  "files": [
+    "dist/index"
+  ],
+  "scripts": {
+    "start": "node ./exec/run-message-relayer.js",
+    "build": "tsc -p ./tsconfig.build.json",
+    "clean": "rimraf  dist/ ./tsconfig.build.tsbuildinfo",
+    "lint": "yarn lint:fix && yarn lint:check",
+    "lint:check": "tslint --format stylish --project .",
+    "lint:fix": "prettier --config prettier-config.json --write \"{src,exec,test}/**/*.ts\""
+  },
+  "keywords": [
+    "optimism",
+    "ethereum",
+    "relayer"
+  ],
+  "homepage": "https://github.com/ethereum-optimism/optimism-monorepo/tree/master/packages/batch-submitter#readme",
+  "license": "MIT",
+  "author": "Optimism",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ethereum-optimism/optimism.git"
+  },
+  "dependencies": {
+    "@eth-optimism/core-utils": "^0.1.10",
+    "@eth-optimism/contracts": "^0.1.11",
+    "dotenv": "^8.2.0",
+    "ethers": "^5.1.0",
+    "google-spreadsheet": "^3.1.15",
+    "merkletreejs": "^0.2.18",
+    "rlp": "^2.2.6"
+  }
+}

packages/message-relayer/prettier-config.json

+../../prettier-config.json
\ No newline at end of file

packages/message-relayer/src/exec/run.ts

+import { Wallet, providers } from 'ethers'
+import { MessageRelayerService } from '../service'
+import SpreadSheet from '../spreadsheet'
+import { config } from 'dotenv'
+config()
+
+const env = process.env
+const L2_NODE_WEB3_URL = env.L2_NODE_WEB3_URL
+const L1_NODE_WEB3_URL = env.L1_NODE_WEB3_URL
+const ADDRESS_MANAGER_ADDRESS = env.ADDRESS_MANAGER_ADDRESS
+const L1_WALLET_KEY = env.L1_WALLET_KEY
+const MNEMONIC = env.MNEMONIC
+const HD_PATH = env.HD_PATH
+const RELAY_GAS_LIMIT = env.RELAY_GAS_LIMIT || '4000000'
+const POLLING_INTERVAL = env.POLLING_INTERVAL || '5000'
+const GET_LOGS_INTERVAL = env.GET_LOGS_INTERVAL || '2000'
+const L2_BLOCK_OFFSET = env.L2_BLOCK_OFFSET || '1'
+const L1_START_OFFSET = env.L1_BLOCK_OFFSET || '1'
+const FROM_L2_TRANSACTION_INDEX = env.FROM_L2_TRANSACTION_INDEX || '0'
+
+// Spreadsheet configuration
+const SPREADSHEET_MODE = env.SPREADSHEET_MODE || ''
+const SHEET_ID = env.SHEET_ID || ''
+const CLIENT_EMAIL = env.CLIENT_EMAIL || ''
+const CLIENT_PRIVATE_KEY = env.CLIENT_PRIVATE_KEY || ''
+
+const main = async () => {
+  if (!ADDRESS_MANAGER_ADDRESS) {
+    throw new Error('Must pass ADDRESS_MANAGER_ADDRESS')
+  }
+  if (!L1_NODE_WEB3_URL) {
+    throw new Error('Must pass L1_NODE_WEB3_URL')
+  }
+  if (!L2_NODE_WEB3_URL) {
+    throw new Error('Must pass L2_NODE_WEB3_URL')
+  }
+
+  const l2Provider = new providers.JsonRpcProvider(L2_NODE_WEB3_URL)
+  const l1Provider = new providers.JsonRpcProvider(L1_NODE_WEB3_URL)
+
+  let wallet: Wallet
+  if (L1_WALLET_KEY) {
+    wallet = new Wallet(L1_WALLET_KEY, l1Provider)
+  } else if (MNEMONIC) {
+    wallet = Wallet.fromMnemonic(MNEMONIC, HD_PATH)
+    wallet = wallet.connect(l1Provider)
+  } else {
+    throw new Error('Must pass one of L1_WALLET_KEY or MNEMONIC')
+  }
+
+  let spreadsheet = null
+  if (SPREADSHEET_MODE) {
+    if (!SHEET_ID) {
+      throw new Error('Must pass SHEET_ID')
+    }
+    if (!CLIENT_EMAIL) {
+      throw new Error('Must pass CLIENT_EMAIL')
+    }
+    if (!CLIENT_PRIVATE_KEY) {
+      throw new Error('Must pass CLIENT_PRIVATE_KEY')
+    }
+    const privateKey = CLIENT_PRIVATE_KEY.replace(/\\n/g, '\n')
+    spreadsheet = new SpreadSheet(SHEET_ID)
+    await spreadsheet.init(CLIENT_EMAIL, privateKey)
+  }
+
+  const service = new MessageRelayerService({
+    l1RpcProvider: l1Provider,
+    l2RpcProvider: l2Provider,
+    addressManagerAddress: ADDRESS_MANAGER_ADDRESS,
+    l1Wallet: wallet,
+    relayGasLimit: parseInt(RELAY_GAS_LIMIT, 10),
+    fromL2TransactionIndex: parseInt(FROM_L2_TRANSACTION_INDEX, 10),
+    pollingInterval: parseInt(POLLING_INTERVAL, 10),
+    l2BlockOffset: parseInt(L2_BLOCK_OFFSET, 10),
+    l1StartOffset: parseInt(L1_START_OFFSET, 10),
+    getLogsInterval: parseInt(GET_LOGS_INTERVAL, 10),
+    spreadsheetMode: !!SPREADSHEET_MODE,
+    spreadsheet,
+  })
+
+  await service.start()
+}
+export default main

packages/message-relayer/src/spreadsheet.ts

+import { GoogleSpreadsheet } from 'google-spreadsheet'
+
+export default class SpreadSheet {
+  public doc
+  public sheet
+
+  constructor(id) {
+    this.doc = new GoogleSpreadsheet(id)
+    this.sheet = null
+  }
+
+  async init(email, privateKey) {
+    await this.doc.useServiceAccountAuth({
+      client_email: email,
+      private_key: privateKey,
+    })
+
+    await this.doc.loadInfo()
+    this.sheet = this.doc.sheetsByIndex[0]
+  }
+
+  async addRow(row) {
+    return this.sheet.addRow(row)
+  }
+}

packages/message-relayer/src/types.ts

+import { BigNumber } from 'ethers'
+
+export interface StateRootBatchHeader {
+  batchIndex: BigNumber
+  batchRoot: string
+  batchSize: BigNumber
+  prevTotalElements: BigNumber
+  extraData: string
+}
+
+export interface SentMessage {
+  target: string
+  sender: string
+  message: string
+  messageNonce: number
+  encodedMessage: string
+  encodedMessageHash: string
+  parentTransactionIndex: number
+  parentTransactionHash: string
+}
+
+export interface SentMessageProof {
+  stateRoot: string
+  stateRootBatchHeader: StateRootBatchHeader
+  stateRootProof: StateRootProof
+  stateTrieWitness: string | Buffer
+  storageTrieWitness: string | Buffer
+}
+
+export interface StateRootProof {
+  index: number
+  siblings: string[]
+}

packages/message-relayer/tsconfig.build.json

+{
+  "extends": "../../tsconfig.build.json",
+
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist"
+  },
+
+  "include": [
+    "src/**/*"
+  ]
+}

packages/message-relayer/tsconfig.json

+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+      "resolveJsonModule": true
+  }
+}

packages/message-relayer/tslint.json

+{
+    "extends": "../../tslint.base.json"
+}