pkg: Add batch submitter (#12)

* pkg: Add batch submitter

* chore(batch-submitter): fix configs

* chore(batch-submitter): use latest smock

* fix: use latest typescript

* chore: disable prettier in incompat lines

Long term fixed by switching to eslint

* perf: only test packages that have changed since master

https://github.com/lerna/lerna/tree/main/core/filter-options\#--since-refCo-authored-by: Georgios Konstantopoulos <me@gakonst.com>

.github/workflows/ci.yml

@@ -20,6 +20,9 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: Fetch history
+        run: git fetch
+
       - name: Setup node ${{ matrix.node }}
         uses: actions/setup-node@v1
         with:

package.json

@@ -10,10 +10,10 @@
     "lerna": "^4.0.0"
   },
   "scripts": {
-      "clean": "yarn workspaces run clean",
-      "build": "yarn workspaces run build",
-      "test": "yarn lerna run test --parallel",
-      "lint": "yarn lerna run lint  --parallel",
-      "lint:fix": "yarn workspaces run lint:fix"
+      "clean": "yarn lerna run clean",
+      "build": "yarn lerna run build",
+      "test": "yarn lerna run test --parallel --since origin/master",
+      "lint": "yarn lerna run lint  --parallel --since origin/master",
+      "lint:fix": "yarn lerna run lint:fix"
   }
 }

packages/batch-submitter/.env.example

+# Logging
+DEBUG=info*,error*,warn*,debug*
+
+L1_NODE_WEB3_URL=http://localhost:9545
+L2_NODE_WEB3_URL=http://localhost:8545
+
+MAX_TX_SIZE=90000
+MIN_TX_SIZE=0
+MAX_BATCH_SIZE=50
+MAX_BATCH_SUBMISSION_TIME=0
+POLL_INTERVAL=15000
+NUM_CONFIRMATIONS=0
+RESUBMISSION_TIMEOUT=300 # in seconds
+FINALITY_CONFIRMATIONS=0
+RUN_TX_BATCH_SUBMITTER=true
+RUN_STATE_BATCH_SUBMITTER=true
+SAFE_MINIMUM_ETHER_BALANCE=0
+CLEAR_PENDING_TXS=false
+ADDRESS_MANAGER_ADDRESS=
+
+# Optional gas settings
+MAX_GAS_PRICE_IN_GWEI=200
+GAS_RETRY_INCREMENT=5
+GAS_THRESHOLD_IN_GWEI=100
+
+SEQUENCER_PRIVATE_KEY=0xd2ab07f7c10ac88d5f86f1b4c1035d5195e81f27dbe62ad65e59cbf88205629b

packages/batch-submitter/.gitignore

+build
+dist
+node_modules
+.env
+cache/*
+
+# vim
+*.swp

packages/batch-submitter/CHANGELOG.md

+# Changelog
+
+## v0.1.3
+
+- Add tx resubmission logic
+- Log when the batch submitter runs low on ETH
+
+## v0.1.2
+
+Adds mnemonic config parsing
+
+## v0.1.1
+
+Final fixes before minnet release.
+
+- Add batch submission timeout
+- Log sequencer address
+- remove ssh
+
+## v0.1.0
+
+The inital release

packages/batch-submitter/README.md

+# Batch Submitter
+
+Contains an executable batch submitter service which watches L1 and a local L2 node and submits batches to the
+`CanonicalTransactionChain` & `StateCommitmentChain` based on its local information.
+
+## Configuration
+All configuration is done via environment variables. See all variables at [.env.example](.env.example); copy into a `.env` file before running.
+
+## Building & Running
+1. Make sure dependencies are installed just run `yarn` in the base directory
+2. Build `yarn build`
+3. Run `yarn start`
+
+## Controlling log output verbosity
+Before running, set the `DEBUG` environment variable to specify the verbosity level. It must be made up of comma-separated values of patterns to match in debug logs. Here's a few common options:
+* `debug*` - Will match all debug statements -- very verbose
+* `info*` - Will match all info statements -- less verbose, useful in most cases
+* `warn*` - Will match all warnings -- recommended at a minimum
+* `error*` - Will match all errors -- would not omit this
+
+Examples:
+* Everything but debug: `export DEBUG=info*,error*,warn*`
+* Most verbose: `export DEBUG=info*,error*,warn*,debug*`
+
+## Testing & linting
+
+### Local
+
+- Run unit tests with `yarn test`
+- See lint errors with `yarn lint`; auto-fix with `yarn lint --fix`
+
+### Submission
+
+You may test a submission locally against a local Hardhat fork. 
+
+1. Follow the instructions [here](https://github.com/ethereum-optimism/hardhat) to run a Hardhat node. 
+2. Change the Batch Submitter `.env` field `L1_NODE_WEB3_URL` to the local Hardhat url. Depending on which network you are using, update `ADDRESS_MANAGER_ADDRESS` according to the [Regenesis repo](https://github.com/ethereum-optimism/regenesis).
+3. Also check `L2_NODE_WEB3_URL` is correctly set and has transactions to submit.
+3. Run `yarn build` to build your changes.
+4. Start Batch Submitter with `yarn start`. It will automatically start submitting pending transactions from L2.

packages/batch-submitter/exec/run-batch-submitter.js

+#!/usr/bin/env node
+
+const batchSubmitter = require("../build/src/exec/run-batch-submitter")
+
+batchSubmitter.run()
\ No newline at end of file

packages/batch-submitter/hardhat.config.ts

+import '@nomiclabs/hardhat-waffle'
+import { HardhatUserConfig } from 'hardhat/config'
+
+import {
+  DEFAULT_ACCOUNTS_HARDHAT,
+  RUN_OVM_TEST_GAS,
+} from './test/helpers/constants'
+
+const config: HardhatUserConfig = {
+  networks: {
+    hardhat: {
+      accounts: DEFAULT_ACCOUNTS_HARDHAT,
+      blockGasLimit: RUN_OVM_TEST_GAS * 2,
+    },
+  },
+  mocha: {
+    timeout: 50000,
+  },
+  solidity: {
+    version: '0.7.0',
+    settings: {
+      optimizer: {
+        enabled: true,
+        runs: 200,
+      },
+      outputSelection: {
+        '*': {
+          '*': ['storageLayout'],
+        },
+      },
+    },
+  },
+}
+
+export default config

packages/batch-submitter/index.ts

+const rootPath = __dirname
+
+export { rootPath }

packages/batch-submitter/package.json

+{
+  "name": "@eth-optimism/batch-submitter",
+  "version": "0.1.5",
+  "description": "[Optimism] Batch submission for sequencer & aggregators",
+  "main": "dist/index",
+  "types": "dist/index",
+  "files": [
+    "dist/index"
+  ],
+  "scripts": {
+    "start": "node ./exec/run-batch-submitter.js",
+    "build": "tsc -p ./tsconfig.build.json",
+    "clean": "rimraf dist/ ./tsconfig.build.tsbuildinfo",
+    "lint": "yarn lint:fix && yarn lint:check",
+    "lint:check": "tslint --format stylish --project .",
+    "lint:fix": "prettier --config prettier-config.json --write \"hardhat.config.ts\" \"{src,exec,test}/**/*.ts\"",
+    "test": "hardhat test --show-stack-traces"
+  },
+  "keywords": [
+    "optimism",
+    "ethereum",
+    "sequencer",
+    "aggregator"
+  ],
+  "homepage": "https://github.com/ethereum-optimism/optimism-monorepo/tree/master/packages/batch-submitter#readme",
+  "license": "MIT",
+  "author": "Optimism",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ethereum-optimism/optimism-monorepo.git"
+  },
+  "dependencies": {
+    "@eth-optimism/contracts": "^0.0.2-alpha.7",
+    "@eth-optimism/core-utils": "^0.1.10",
+    "@eth-optimism/provider": "^0.0.1-alpha.13",
+    "@eth-optimism/ynatm": "^0.2.2",
+    "@ethersproject/abstract-provider": "^5.0.5",
+    "@ethersproject/providers": "^5.0.14",
+    "bluebird": "^3.7.2",
+    "dotenv": "^8.2.0",
+    "ethers": "5.0.0",
+    "new-contracts": "npm:@eth-optimism/contracts@0.1.4"
+  },
+  "devDependencies": {
+    "@eth-optimism/smock": "1.0.0-alpha.3",
+    "@nomiclabs/hardhat-ethers": "^2.0.2",
+    "@nomiclabs/hardhat-waffle": "^2.0.1",
+    "@types/chai": "^4.1.7",
+    "@types/chai-as-promised": "^7.1.0",
+    "@types/mocha": "^5.2.6",
+    "@types/node": "^11.11.3",
+    "@types/sinon": "^9.0.10",
+    "chai": "^4.2.0",
+    "chai-as-promised": "^7.1.1",
+    "ethereum-waffle": "3.0.0",
+    "ganache-core": "^2.13.2",
+    "hardhat": "^2.1.1",
+    "mocha": "^6.1.4",
+    "prettier": "^1.16.4",
+    "rimraf": "^2.6.3",
+    "sinon": "^9.2.4",
+    "sinon-chai": "^3.5.0",
+    "ts-node": "^8.2.0",
+    "typescript": "^4.2.3"
+  },
+  "resolutions": {
+    "ganache-core": "^2.13.2"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

packages/batch-submitter/prettier-config.json

+../../prettier-config.json
\ No newline at end of file

packages/batch-submitter/src/batch-submitter/batch-submitter.ts

+/* External Imports */
+import { Contract, Signer, utils } from 'ethers'
+import { TransactionReceipt } from '@ethersproject/abstract-provider'
+import * as ynatm from '@eth-optimism/ynatm'
+import { Address, Bytes32, Logger } from '@eth-optimism/core-utils'
+import { OptimismProvider } from '@eth-optimism/provider'
+import { getContractFactory } from '@eth-optimism/contracts'
+
+export interface RollupInfo {
+  mode: 'sequencer' | 'verifier'
+  syncing: boolean
+  ethContext: {
+    blockNumber: number
+    timestamp: number
+  }
+  rollupContext: {
+    index: number
+    queueIndex: number
+  }
+}
+export interface Range {
+  start: number
+  end: number
+}
+export interface ResubmissionConfig {
+  resubmissionTimeout: number
+  minGasPriceInGwei: number
+  maxGasPriceInGwei: number
+  gasRetryIncrement: number
+}
+
+export abstract class BatchSubmitter {
+  protected rollupInfo: RollupInfo
+  protected chainContract: Contract
+  protected l2ChainId: number
+  protected syncing: boolean
+  protected lastBatchSubmissionTimestamp: number = 0
+
+  constructor(
+    readonly signer: Signer,
+    readonly l2Provider: OptimismProvider,
+    readonly minTxSize: number,
+    readonly maxTxSize: number,
+    readonly maxBatchSize: number,
+    readonly maxBatchSubmissionTime: number,
+    readonly numConfirmations: number,
+    readonly resubmissionTimeout: number,
+    readonly finalityConfirmations: number,
+    readonly addressManagerAddress: string,
+    readonly minBalanceEther: number,
+    readonly minGasPriceInGwei: number,
+    readonly maxGasPriceInGwei: number,
+    readonly gasRetryIncrement: number,
+    readonly gasThresholdInGwei: number,
+    readonly log: Logger
+  ) {}
+
+  public abstract _submitBatch(
+    startBlock: number,
+    endBlock: number
+  ): Promise<TransactionReceipt>
+  public abstract _onSync(): Promise<TransactionReceipt>
+  public abstract _getBatchStartAndEnd(): Promise<Range>
+  public abstract _updateChainInfo(): Promise<void>
+
+  public async submitNextBatch(): Promise<TransactionReceipt> {
+    if (typeof this.l2ChainId === 'undefined') {
+      this.l2ChainId = await this._getL2ChainId()
+    }
+    await this._updateChainInfo()
+    await this._checkBalance()
+
+    if (this.syncing === true) {
+      this.log.info(
+        'Syncing mode enabled! Skipping batch submission and clearing queue...'
+      )
+      return this._onSync()
+    }
+    const range = await this._getBatchStartAndEnd()
+    if (!range) {
+      return
+    }
+
+    return this._submitBatch(range.start, range.end)
+  }
+
+  protected async _checkBalance(): Promise<void> {
+    const address = await this.signer.getAddress()
+    const balance = await this.signer.getBalance()
+    const ether = utils.formatEther(balance)
+    const num = parseFloat(ether)
+
+    this.log.info('Checked balance', {
+      address,
+      ether,
+    })
+
+    if (num < this.minBalanceEther) {
+      this.log.warn('Current balance lower than min safe balance', {
+        current: num,
+        safeBalance: this.minBalanceEther,
+      })
+    }
+  }
+
+  protected async _getRollupInfo(): Promise<RollupInfo> {
+    return this.l2Provider.send('rollup_getInfo', [])
+  }
+
+  protected async _getL2ChainId(): Promise<number> {
+    return this.l2Provider.send('eth_chainId', [])
+  }
+
+  protected async _getChainAddresses(): Promise<{
+    ctcAddress: string
+    sccAddress: string
+  }> {
+    const addressManager = (
+      await getContractFactory('Lib_AddressManager', this.signer)
+    ).attach(this.addressManagerAddress)
+    const sccAddress = await addressManager.getAddress(
+      'OVM_StateCommitmentChain'
+    )
+    const ctcAddress = await addressManager.getAddress(
+      'OVM_CanonicalTransactionChain'
+    )
+    return {
+      ctcAddress,
+      sccAddress,
+    }
+  }
+
+  protected _shouldSubmitBatch(batchSizeInBytes: number): boolean {
+    const isTimeoutReached =
+      this.lastBatchSubmissionTimestamp + this.maxBatchSubmissionTime <=
+      Date.now()
+    if (batchSizeInBytes < this.minTxSize) {
+      if (!isTimeoutReached) {
+        this.log.info(
+          'Skipping batch submission. Batch too small & max submission timeout not reached.',
+          {
+            batchSizeInBytes,
+            minTxSize: this.minTxSize,
+          }
+        )
+        return false
+      }
+      this.log.info('Timeout reached.')
+    }
+    return true
+  }
+
+  public static async getReceiptWithResubmission(
+    txFunc: (gasPrice) => Promise<TransactionReceipt>,
+    resubmissionConfig: ResubmissionConfig,
+    log: Logger
+  ): Promise<TransactionReceipt> {
+    const {
+      resubmissionTimeout,
+      minGasPriceInGwei,
+      maxGasPriceInGwei,
+      gasRetryIncrement,
+    } = resubmissionConfig
+
+    const receipt = await ynatm.send({
+      sendTransactionFunction: txFunc,
+      minGasPrice: ynatm.toGwei(minGasPriceInGwei),
+      maxGasPrice: ynatm.toGwei(maxGasPriceInGwei),
+      gasPriceScalingFunction: ynatm.LINEAR(gasRetryIncrement),
+      delay: resubmissionTimeout,
+    })
+
+    log.debug('Resubmission tx receipt', { receipt })
+
+    return receipt
+  }
+
+  private async _getMinGasPriceInGwei(): Promise<number> {
+    if (this.minGasPriceInGwei !== 0) {
+      return this.minGasPriceInGwei
+    }
+    let minGasPriceInGwei = parseInt(
+      utils.formatUnits(await this.signer.getGasPrice(), 'gwei'),
+      10
+    )
+    if (minGasPriceInGwei > this.maxGasPriceInGwei) {
+      this.log.warn(
+        'Minimum gas price is higher than max! Ethereum must be congested...'
+      )
+      minGasPriceInGwei = this.maxGasPriceInGwei
+    }
+    return minGasPriceInGwei
+  }
+
+  protected async _submitAndLogTx(
+    txFunc: (gasPrice) => Promise<TransactionReceipt>,
+    successMessage: string
+  ): Promise<TransactionReceipt> {
+    this.lastBatchSubmissionTimestamp = Date.now()
+    this.log.debug('Waiting for receipt...')
+
+    const resubmissionConfig: ResubmissionConfig = {
+      resubmissionTimeout: this.resubmissionTimeout,
+      minGasPriceInGwei: await this._getMinGasPriceInGwei(),
+      maxGasPriceInGwei: this.maxGasPriceInGwei,
+      gasRetryIncrement: this.gasRetryIncrement,
+    }
+
+    const receipt = await BatchSubmitter.getReceiptWithResubmission(
+      txFunc,
+      resubmissionConfig,
+      this.log
+    )
+
+    this.log.debug('Transaction receipt:', { receipt })
+    this.log.info(successMessage)
+    return receipt
+  }
+}

packages/batch-submitter/src/batch-submitter/index.ts

+export * from './batch-submitter'
+export * from './tx-batch-submitter'
+export * from './state-batch-submitter'
+
+export const TX_BATCH_SUBMITTER_LOG_TAG = 'oe:batch-submitter:tx-chain'
+export const STATE_BATCH_SUBMITTER_LOG_TAG = 'oe:batch-submitter:state-chain'
+
+// BLOCK_OFFSET is the number of L2 blocks we need to skip for the
+// batch submitter.
+export const BLOCK_OFFSET = 1 // TODO: Update testnet / mainnet to make this zero.

packages/batch-submitter/src/batch-submitter/state-batch-submitter.ts

+/* External Imports */
+import { Contract, Signer } from 'ethers'
+import { TransactionReceipt } from '@ethersproject/abstract-provider'
+import { getContractFactory } from '@eth-optimism/contracts'
+import { Logger, Bytes32 } from '@eth-optimism/core-utils'
+import { OptimismProvider } from '@eth-optimism/provider'
+
+/* Internal Imports */
+import { L2Block } from '..'
+import { RollupInfo, Range, BatchSubmitter, BLOCK_OFFSET } from '.'
+
+export class StateBatchSubmitter extends BatchSubmitter {
+  // TODO: Change this so that we calculate start = scc.totalElements() and end = ctc.totalElements()!
+  // Not based on the length of the L2 chain -- that is only used in the batch submitter
+  // Note this means we've got to change the state / end calc logic
+
+  protected l2ChainId: number
+  protected syncing: boolean
+  protected ctcContract: Contract
+  private fraudSubmissionAddress: string
+
+  constructor(
+    signer: Signer,
+    l2Provider: OptimismProvider,
+    minTxSize: number,
+    maxTxSize: number,
+    maxBatchSize: number,
+    maxBatchSubmissionTime: number,
+    numConfirmations: number,
+    resubmissionTimeout: number,
+    finalityConfirmations: number,
+    addressManagerAddress: string,
+    minBalanceEther: number,
+    minGasPriceInGwei: number,
+    maxGasPriceInGwei: number,
+    gasRetryIncrement: number,
+    gasThresholdInGwei: number,
+    log: Logger,
+    fraudSubmissionAddress: string
+  ) {
+    super(
+      signer,
+      l2Provider,
+      minTxSize,
+      maxTxSize,
+      maxBatchSize,
+      maxBatchSubmissionTime,
+      numConfirmations,
+      resubmissionTimeout,
+      finalityConfirmations,
+      addressManagerAddress,
+      minBalanceEther,
+      minGasPriceInGwei,
+      maxGasPriceInGwei,
+      gasRetryIncrement,
+      gasThresholdInGwei,
+      log
+    )
+    this.fraudSubmissionAddress = fraudSubmissionAddress
+  }
+
+  /*****************************
+   * Batch Submitter Overrides *
+   ****************************/
+
+  public async _updateChainInfo(): Promise<void> {
+    const info: RollupInfo = await this._getRollupInfo()
+    if (info.mode === 'verifier') {
+      this.log.error(
+        'Verifier mode enabled! Batch submitter only compatible with sequencer mode'
+      )
+      process.exit(1)
+    }
+    this.syncing = info.syncing
+    const addrs = await this._getChainAddresses()
+    const sccAddress = addrs.sccAddress
+    const ctcAddress = addrs.ctcAddress
+
+    if (
+      typeof this.chainContract !== 'undefined' &&
+      sccAddress === this.chainContract.address &&
+      ctcAddress === this.ctcContract.address
+    ) {
+      return
+    }
+
+    this.chainContract = (
+      await getContractFactory('OVM_StateCommitmentChain', this.signer)
+    ).attach(sccAddress)
+    this.ctcContract = (
+      await getContractFactory('OVM_CanonicalTransactionChain', this.signer)
+    ).attach(ctcAddress)
+
+    this.log.info('Connected Optimism contracts', {
+      stateCommitmentChain: this.chainContract.address,
+      canonicalTransactionChain: this.ctcContract.address,
+    })
+    return
+  }
+
+  public async _onSync(): Promise<TransactionReceipt> {
+    this.log.info('Syncing mode enabled! Skipping state batch submission...')
+    return
+  }
+
+  public async _getBatchStartAndEnd(): Promise<Range> {
+    // TODO: Remove BLOCK_OFFSET by adding a tx to Geth's genesis
+    const startBlock: number =
+      (await this.chainContract.getTotalElements()).toNumber() + BLOCK_OFFSET
+    // We will submit state roots for txs which have been in the tx chain for a while.
+    const callBlockNumber: number =
+      (await this.signer.provider.getBlockNumber()) - this.finalityConfirmations
+    const totalElements: number =
+      (await this.ctcContract.getTotalElements()).toNumber() + BLOCK_OFFSET
+    const endBlock: number = Math.min(
+      startBlock + this.maxBatchSize,
+      totalElements
+    )
+    if (startBlock >= endBlock) {
+      if (startBlock > endBlock) {
+        this.log.error(
+          'State commitment chain is larger than transaction chain. This should never happen!'
+        )
+      }
+      this.log.info(
+        'No state commitments to submit. Skipping batch submission...'
+      )
+      return
+    }
+    return {
+      start: startBlock,
+      end: endBlock,
+    }
+  }
+
+  public async _submitBatch(
+    startBlock: number,
+    endBlock: number
+  ): Promise<TransactionReceipt> {
+    const batch = await this._generateStateCommitmentBatch(startBlock, endBlock)
+    const tx = this.chainContract.interface.encodeFunctionData(
+      'appendStateBatch',
+      [batch, startBlock]
+    )
+    if (!this._shouldSubmitBatch(tx.length * 2)) {
+      return
+    }
+
+    const offsetStartsAtIndex = startBlock - BLOCK_OFFSET // TODO: Remove BLOCK_OFFSET by adding a tx to Geth's genesis
+    this.log.debug('Submitting batch.', { tx })
+
+    const nonce = await this.signer.getTransactionCount()
+    const contractFunction = async (gasPrice): Promise<TransactionReceipt> => {
+      const contractTx = await this.chainContract.appendStateBatch(
+        batch,
+        offsetStartsAtIndex,
+        { nonce, gasPrice }
+      )
+      return this.signer.provider.waitForTransaction(
+        contractTx.hash,
+        this.numConfirmations
+      )
+    }
+    return this._submitAndLogTx(contractFunction, 'Submitted state root batch!')
+  }
+
+  /*********************
+   * Private Functions *
+   ********************/
+
+  private async _generateStateCommitmentBatch(
+    startBlock: number,
+    endBlock: number
+  ): Promise<Bytes32[]> {
+    const batch: Bytes32[] = []
+
+    for (let i = startBlock; i < endBlock; i++) {
+      const block = (await this.l2Provider.getBlockWithTransactions(
+        i
+      )) as L2Block
+      if (block.transactions[0].from === this.fraudSubmissionAddress) {
+        batch.push(
+          '0xbad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1'
+        )
+        this.fraudSubmissionAddress = 'no fraud'
+      } else {
+        batch.push(block.stateRoot)
+      }
+    }
+
+    let tx = this.chainContract.interface.encodeFunctionData(
+      'appendStateBatch',
+      [batch, startBlock]
+    )
+    while (tx.length > this.maxTxSize) {
+      batch.splice(Math.ceil((batch.length * 2) / 3)) // Delete 1/3rd of all of the batch elements
+      tx = this.chainContract.interface.encodeFunctionData('appendStateBatch', [
+        batch,
+        startBlock,
+      ])
+    }
+    return batch
+  }
+}

packages/batch-submitter/src/exec/run-batch-submitter.ts

+/* External Imports */
+import { Logger } from '@eth-optimism/core-utils'
+import { exit } from 'process'
+import { Signer, Wallet } from 'ethers'
+import {
+  Provider,
+  JsonRpcProvider,
+  TransactionReceipt,
+} from '@ethersproject/providers'
+import { OptimismProvider } from '@eth-optimism/provider'
+import { config } from 'dotenv'
+config()
+
+/* Internal Imports */
+import {
+  TransactionBatchSubmitter,
+  AutoFixBatchOptions,
+  BatchSubmitter,
+  StateBatchSubmitter,
+  STATE_BATCH_SUBMITTER_LOG_TAG,
+  TX_BATCH_SUBMITTER_LOG_TAG,
+} from '..'
+
+/* Logger */
+const log = new Logger({ name: 'oe:batch-submitter:init' })
+
+interface RequiredEnvVars {
+  // The HTTP provider URL for L1.
+  L1_NODE_WEB3_URL: 'L1_NODE_WEB3_URL'
+  // The HTTP provider URL for L2.
+  L2_NODE_WEB3_URL: 'L2_NODE_WEB3_URL'
+  // The layer one address manager address
+  ADDRESS_MANAGER_ADDRESS: 'ADDRESS_MANAGER_ADDRESS'
+  // The minimum size in bytes of any L1 transactions generated by the batch submitter.
+  MIN_TX_SIZE: 'MIN_TX_SIZE'
+  // The maximum size in bytes of any L1 transactions generated by the batch submitter.
+  MAX_TX_SIZE: 'MAX_TX_SIZE'
+  // The maximum number of L2 transactions that can ever be in a batch.
+  MAX_BATCH_SIZE: 'MAX_BATCH_SIZE'
+  // The maximum amount of time (seconds) that we will wait before submitting an under-sized batch.
+  MAX_BATCH_SUBMISSION_TIME: 'MAX_BATCH_SUBMISSION_TIME'
+  // The delay in milliseconds between querying L2 for more transactions / to create a new batch.
+  POLL_INTERVAL: 'POLL_INTERVAL'
+  // The number of confirmations which we will wait after appending new batches.
+  NUM_CONFIRMATIONS: 'NUM_CONFIRMATIONS'
+  // The number of seconds to wait before resubmitting a transaction.
+  RESUBMISSION_TIMEOUT: 'RESUBMISSION_TIMEOUT'
+  // The number of confirmations that we should wait before submitting state roots for CTC elements.
+  FINALITY_CONFIRMATIONS: 'FINALITY_CONFIRMATIONS'
+  // Whether or not to run the tx batch submitter.
+  RUN_TX_BATCH_SUBMITTER: 'true' | 'false' | 'RUN_TX_BATCH_SUBMITTER'
+  // Whether or not to run the state batch submitter.
+  RUN_STATE_BATCH_SUBMITTER: 'true' | 'false' | 'RUN_STATE_BATCH_SUBMITTER'
+  // The safe minimum amount of ether the batch submitter key should
+  // hold before it starts to log errors.
+  SAFE_MINIMUM_ETHER_BALANCE: 'SAFE_MINIMUM_ETHER_BALANCE'
+  // A boolean to clear the pending transactions in the mempool
+  // on start up.
+  CLEAR_PENDING_TXS: 'true' | 'false' | 'CLEAR_PENDING_TXS'
+}
+const requiredEnvVars: RequiredEnvVars = {
+  L1_NODE_WEB3_URL: 'L1_NODE_WEB3_URL',
+  L2_NODE_WEB3_URL: 'L2_NODE_WEB3_URL',
+  ADDRESS_MANAGER_ADDRESS: 'ADDRESS_MANAGER_ADDRESS',
+  MIN_TX_SIZE: 'MIN_TX_SIZE',
+  MAX_TX_SIZE: 'MAX_TX_SIZE',
+  MAX_BATCH_SIZE: 'MAX_BATCH_SIZE',
+  MAX_BATCH_SUBMISSION_TIME: 'MAX_BATCH_SUBMISSION_TIME',
+  POLL_INTERVAL: 'POLL_INTERVAL',
+  NUM_CONFIRMATIONS: 'NUM_CONFIRMATIONS',
+  RESUBMISSION_TIMEOUT: 'RESUBMISSION_TIMEOUT',
+  FINALITY_CONFIRMATIONS: 'FINALITY_CONFIRMATIONS',
+  RUN_TX_BATCH_SUBMITTER: 'RUN_TX_BATCH_SUBMITTER',
+  RUN_STATE_BATCH_SUBMITTER: 'RUN_STATE_BATCH_SUBMITTER',
+  SAFE_MINIMUM_ETHER_BALANCE: 'SAFE_MINIMUM_ETHER_BALANCE',
+  CLEAR_PENDING_TXS: 'CLEAR_PENDING_TXS',
+}
+
+/* Optional Env Vars
+ * FRAUD_SUBMISSION_ADDRESS
+ * DISABLE_QUEUE_BATCH_APPEND
+ * SEQUENCER_PRIVATE_KEY
+ * MNEMONIC
+ */
+const env = process.env
+const FRAUD_SUBMISSION_ADDRESS = env.FRAUD_SUBMISSION_ADDRESS || 'no fraud'
+const DISABLE_QUEUE_BATCH_APPEND = !!env.DISABLE_QUEUE_BATCH_APPEND
+const MIN_GAS_PRICE_IN_GWEI = parseInt(env.MIN_GAS_PRICE_IN_GWEI, 10) || 0
+const MAX_GAS_PRICE_IN_GWEI = parseInt(env.MAX_GAS_PRICE_IN_GWEI, 10) || 70
+const GAS_RETRY_INCREMENT = parseInt(env.GAS_RETRY_INCREMENT, 10) || 5
+const GAS_THRESHOLD_IN_GWEI = parseInt(env.GAS_THRESHOLD_IN_GWEI, 10) || 100
+// The private key that will be used to submit tx and state batches.
+const SEQUENCER_PRIVATE_KEY = env.SEQUENCER_PRIVATE_KEY
+const MNEMONIC = env.MNEMONIC
+const HD_PATH = env.HD_PATH
+// Auto fix batch options -- TODO: Remove this very hacky config
+const AUTO_FIX_BATCH_OPTIONS_CONF = env.AUTO_FIX_BATCH_OPTIONS_CONF
+const autoFixBatchOptions: AutoFixBatchOptions = {
+  fixDoublePlayedDeposits: AUTO_FIX_BATCH_OPTIONS_CONF
+    ? AUTO_FIX_BATCH_OPTIONS_CONF.includes('fixDoublePlayedDeposits')
+    : false,
+  fixMonotonicity: AUTO_FIX_BATCH_OPTIONS_CONF
+    ? AUTO_FIX_BATCH_OPTIONS_CONF.includes('fixMonotonicity')
+    : false,
+}
+
+export const run = async () => {
+  log.info('Starting batch submitter...')
+
+  for (const [i, val] of Object.entries(requiredEnvVars)) {
+    if (!process.env[val]) {
+      log.warn('Missing environment variable', {
+        varName: val,
+      })
+      exit(1)
+    }
+    requiredEnvVars[val] = process.env[val]
+  }
+
+  const clearPendingTxs = requiredEnvVars.CLEAR_PENDING_TXS === 'true'
+
+  const l1Provider: Provider = new JsonRpcProvider(
+    requiredEnvVars.L1_NODE_WEB3_URL
+  )
+  const l2Provider: OptimismProvider = new OptimismProvider(
+    requiredEnvVars.L2_NODE_WEB3_URL
+  )
+
+  let sequencerSigner: Signer
+  if (SEQUENCER_PRIVATE_KEY) {
+    sequencerSigner = new Wallet(SEQUENCER_PRIVATE_KEY, l1Provider)
+  } else if (MNEMONIC) {
+    sequencerSigner = Wallet.fromMnemonic(MNEMONIC, HD_PATH).connect(l1Provider)
+  } else {
+    throw new Error('Must pass one of SEQUENCER_PRIVATE_KEY or MNEMONIC')
+  }
+
+  const address = await sequencerSigner.getAddress()
+  log.info('Configured batch submitter addresses', {
+    batchSubmitterAddress: address,
+    addressManagerAddress: requiredEnvVars.ADDRESS_MANAGER_ADDRESS,
+  })
+
+  const txBatchSubmitter = new TransactionBatchSubmitter(
+    sequencerSigner,
+    l2Provider,
+    parseInt(requiredEnvVars.MIN_TX_SIZE, 10),
+    parseInt(requiredEnvVars.MAX_TX_SIZE, 10),
+    parseInt(requiredEnvVars.MAX_BATCH_SIZE, 10),
+    parseInt(requiredEnvVars.MAX_BATCH_SUBMISSION_TIME, 10) * 1_000,
+    parseInt(requiredEnvVars.NUM_CONFIRMATIONS, 10),
+    parseInt(requiredEnvVars.RESUBMISSION_TIMEOUT, 10) * 1_000,
+    requiredEnvVars.ADDRESS_MANAGER_ADDRESS,
+    parseFloat(requiredEnvVars.SAFE_MINIMUM_ETHER_BALANCE),
+    MIN_GAS_PRICE_IN_GWEI,
+    MAX_GAS_PRICE_IN_GWEI,
+    GAS_RETRY_INCREMENT,
+    GAS_THRESHOLD_IN_GWEI,
+    new Logger({ name: TX_BATCH_SUBMITTER_LOG_TAG }),
+    DISABLE_QUEUE_BATCH_APPEND,
+    autoFixBatchOptions
+  )
+
+  const stateBatchSubmitter = new StateBatchSubmitter(
+    sequencerSigner,
+    l2Provider,
+    parseInt(requiredEnvVars.MIN_TX_SIZE, 10),
+    parseInt(requiredEnvVars.MAX_TX_SIZE, 10),
+    parseInt(requiredEnvVars.MAX_BATCH_SIZE, 10),
+    parseInt(requiredEnvVars.MAX_BATCH_SUBMISSION_TIME, 10) * 1_000,
+    parseInt(requiredEnvVars.NUM_CONFIRMATIONS, 10),
+    parseInt(requiredEnvVars.RESUBMISSION_TIMEOUT, 10) * 1_000,
+    parseInt(requiredEnvVars.FINALITY_CONFIRMATIONS, 10),
+    requiredEnvVars.ADDRESS_MANAGER_ADDRESS,
+    parseFloat(requiredEnvVars.SAFE_MINIMUM_ETHER_BALANCE),
+    MIN_GAS_PRICE_IN_GWEI,
+    MAX_GAS_PRICE_IN_GWEI,
+    GAS_RETRY_INCREMENT,
+    GAS_THRESHOLD_IN_GWEI,
+    new Logger({ name: STATE_BATCH_SUBMITTER_LOG_TAG }),
+    FRAUD_SUBMISSION_ADDRESS
+  )
+
+  // Loops infinitely!
+  const loop = async (
+    func: () => Promise<TransactionReceipt>
+  ): Promise<void> => {
+    // Clear all pending transactions
+    if (clearPendingTxs) {
+      try {
+        const pendingTxs = await sequencerSigner.getTransactionCount('pending')
+        const latestTxs = await sequencerSigner.getTransactionCount('latest')
+        if (pendingTxs > latestTxs) {
+          log.info('Detected pending transactions. Clearing all transactions!')
+          for (let i = latestTxs; i < pendingTxs; i++) {
+            const response = await sequencerSigner.sendTransaction({
+              to: await sequencerSigner.getAddress(),
+              value: 0,
+              nonce: i,
+            })
+            log.info('Submitting empty transaction', {
+              nonce: i,
+              txHash: response.hash,
+            })
+            await sequencerSigner.provider.waitForTransaction(
+              response.hash,
+              parseInt(requiredEnvVars.NUM_CONFIRMATIONS, 10)
+            )
+          }
+        }
+      } catch (err) {
+        log.error('Cannot clear transactions', err)
+        process.exit(1)
+      }
+    }
+
+    while (true) {
+      try {
+        await func()
+      } catch (err) {
+        log.error('Error submitting batch', err)
+        log.info('Retrying...')
+      }
+      // Sleep
+      await new Promise((r) =>
+        setTimeout(r, parseInt(requiredEnvVars.POLL_INTERVAL, 10))
+      )
+    }
+  }
+
+  // Run batch submitters in two seperate infinite loops!
+  if (requiredEnvVars.RUN_TX_BATCH_SUBMITTER === 'true') {
+    loop(() => txBatchSubmitter.submitNextBatch())
+  }
+  if (requiredEnvVars.RUN_STATE_BATCH_SUBMITTER === 'true') {
+    loop(() => stateBatchSubmitter.submitNextBatch())
+  }
+}

packages/batch-submitter/src/index.ts

+export * from './batch-submitter'
+export * from './utils'
+export * from './transaction-chain-contract'
+export * from './types'

packages/batch-submitter/src/transaction-chain-contract.ts

+/* External Imports */
+import { Contract, BigNumber } from 'ethers'
+import {
+  TransactionResponse,
+  TransactionRequest,
+} from '@ethersproject/abstract-provider'
+import { keccak256 } from 'ethers/lib/utils'
+import { remove0x, encodeHex } from './utils'
+import {
+  AppendSequencerBatchParams,
+  BatchContext,
+  encodeAppendSequencerBatch,
+} from '@eth-optimism/core-utils'
+
+export { encodeAppendSequencerBatch, BatchContext, AppendSequencerBatchParams }
+
+/*
+ * OVM_CanonicalTransactionChainContract is a wrapper around a normal Ethers contract
+ * where the `appendSequencerBatch(...)` function uses a specialized encoding for improved efficiency.
+ */
+export class CanonicalTransactionChainContract extends Contract {
+  public async appendSequencerBatch(
+    batch: AppendSequencerBatchParams,
+    options?: TransactionRequest
+  ): Promise<TransactionResponse> {
+    return appendSequencerBatch(this, batch, options)
+  }
+}
+
+/**********************
+ * Internal Functions *
+ *********************/
+
+const APPEND_SEQUENCER_BATCH_METHOD_ID = 'appendSequencerBatch()'
+
+const appendSequencerBatch = async (
+  OVM_CanonicalTransactionChain: Contract,
+  batch: AppendSequencerBatchParams,
+  options?: TransactionRequest
+): Promise<TransactionResponse> => {
+  const methodId = keccak256(
+    Buffer.from(APPEND_SEQUENCER_BATCH_METHOD_ID)
+  ).slice(2, 10)
+  const calldata = encodeAppendSequencerBatch(batch)
+  return OVM_CanonicalTransactionChain.signer.sendTransaction({
+    to: OVM_CanonicalTransactionChain.address,
+    data: '0x' + methodId + calldata,
+    ...options,
+  })
+}
+
+const encodeBatchContext = (context: BatchContext): string => {
+  return (
+    encodeHex(context.numSequencedTransactions, 6) +
+    encodeHex(context.numSubsequentQueueTransactions, 6) +
+    encodeHex(context.timestamp, 10) +
+    encodeHex(context.blockNumber, 10)
+  )
+}

packages/batch-submitter/src/types.ts

+/* External Imports */
+import {
+  BlockWithTransactions,
+  Provider,
+  TransactionResponse,
+} from '@ethersproject/abstract-provider'
+
+/* Internal Imports */
+import { EIP155TxData, TxType } from '@eth-optimism/core-utils'
+
+export enum QueueOrigin {
+  Sequencer = 0,
+  L1ToL2 = 1,
+}
+
+export const queueOriginPlainText = {
+  0: QueueOrigin.Sequencer,
+  1: QueueOrigin.L1ToL2,
+  sequencer: QueueOrigin.Sequencer,
+  l1ToL2: QueueOrigin.L1ToL2,
+}
+
+/**
+ * Transaction & Blocks. These are the true data-types we expect
+ * from running a batch submitter.
+ */
+export interface L2Transaction extends TransactionResponse {
+  l1BlockNumber: number
+  l1TxOrigin: string
+  txType: number
+  queueOrigin: number
+}
+
+export interface L2Block extends BlockWithTransactions {
+  stateRoot: string
+  transactions: [L2Transaction]
+}
+
+/**
+ * BatchElement & Batch. These are the data-types of the compressed / batched
+ * block data we submit to L1.
+ */
+export interface BatchElement {
+  stateRoot: string
+  isSequencerTx: boolean
+  sequencerTxType: undefined | TxType
+  txData: undefined | EIP155TxData
+  timestamp: number
+  blockNumber: number
+}
+
+export type Batch = BatchElement[]

packages/batch-submitter/src/utils.ts

+/* External Imports */
+import { BigNumber } from 'ethers'
+
+export const getLen = (pos: { start; end }) => (pos.end - pos.start) * 2
+
+export const encodeHex = (val: any, len: number) =>
+  remove0x(BigNumber.from(val).toHexString()).padStart(len, '0')
+
+export const toVerifiedBytes = (val: string, len: number) => {
+  val = remove0x(val)
+  if (val.length !== len) {
+    throw new Error('Invalid length!')
+  }
+  return val
+}
+
+export const remove0x = (str: string): string => {
+  if (str.startsWith('0x')) {
+    return str.slice(2)
+  } else {
+    return str
+  }
+}

packages/batch-submitter/test/helpers/constants.ts

+/* External Imports */
+import { ethers } from 'ethers'
+import { defaultAccounts } from 'ethereum-waffle'
+
+export const FORCE_INCLUSION_PERIOD_SECONDS = 600
+export const DEFAULT_ACCOUNTS = defaultAccounts
+export const DEFAULT_ACCOUNTS_HARDHAT = defaultAccounts.map((account) => {
+  return {
+    balance: ethers.BigNumber.from(account.balance).toHexString(),
+    privateKey: account.secretKey,
+  }
+})
+
+export const OVM_TX_GAS_LIMIT = 10_000_000
+export const RUN_OVM_TEST_GAS = 20_000_000

packages/batch-submitter/test/helpers/dummy/bytecode.ts

+/* External Imports */
+import { keccak256 } from 'ethers/lib/utils'
+
+export const DUMMY_BYTECODE = '0x123412341234'
+export const DUMMY_BYTECODE_BYTELEN = 6
+export const DUMMY_BYTECODE_HASH = keccak256(DUMMY_BYTECODE)

packages/batch-submitter/test/helpers/dummy/bytes32.ts

+/* External Imports */
+import { ethers } from 'ethers'
+
+export const DUMMY_BYTES32: string[] = Array.from(
+  {
+    length: 10,
+  },
+  (_, i) => {
+    return ethers.utils.keccak256(`0x0${i}`)
+  }
+)

packages/batch-submitter/test/helpers/dummy/index.ts

+export * from './bytes32'
+export * from './bytecode'

packages/batch-submitter/test/helpers/index.ts

+export * from './dummy'
+export * from './constants'
+export * from './resolver'
+export * from './utils'

packages/batch-submitter/test/helpers/resolver/address-manager.ts

+/* External Imports */
+import { ethers } from 'hardhat'
+import { Contract } from 'ethers'
+import { getContractFactory as ctFactory } from '@eth-optimism/contracts'
+
+export const getContractFactory = async (contract: string) =>
+  ctFactory(contract, (await ethers.getSigners())[0])
+
+export const setProxyTarget = async (
+  AddressManager: Contract,
+  name: string,
+  target: Contract
+): Promise<void> => {
+  const SimpleProxy: Contract = await (
+    await getContractFactory('Helper_SimpleProxy')
+  ).deploy()
+
+  await SimpleProxy.setTarget(target.address)
+  await AddressManager.setAddress(name, SimpleProxy.address)
+}
+
+export const makeAddressManager = async (): Promise<Contract> => {
+  return (await getContractFactory('Lib_AddressManager')).deploy()
+}

packages/batch-submitter/test/helpers/resolver/index.ts

+export * from './address-manager'

packages/batch-submitter/test/helpers/utils/buffer-utils.ts

+/* External Imports */
+import { BigNumber } from 'ethers'
+
+/**
+ * Converts a string or buffer to a '0x'-prefixed hex string.
+ * @param buf String or buffer to convert.
+ * @returns '0x'-prefixed string.
+ */
+export const toHexString = (buf: Buffer | string): string => {
+  return '0x' + fromHexString(buf).toString('hex')
+}
+
+/**
+ * Converts a '0x'-prefixed string to a buffer.
+ * @param str '0x'-prefixed string to convert.
+ * @returns Hex buffer.
+ */
+export const fromHexString = (str: string | Buffer): Buffer => {
+  if (typeof str === 'string' && str.startsWith('0x')) {
+    return Buffer.from(str.slice(2), 'hex')
+  }
+
+  return Buffer.from(str)
+}
+
+export const toHexString32 = (
+  input: Buffer | string | number,
+  padRight = false
+): string => {
+  if (typeof input === 'number') {
+    input = BigNumber.from(input).toHexString()
+  }
+
+  input = toHexString(input).slice(2)
+  return '0x' + (padRight ? input.padEnd(64, '0') : input.padStart(64, '0'))
+}
+
+export const getHexSlice = (
+  input: Buffer | string,
+  start: number,
+  length: number
+): string => {
+  return toHexString(fromHexString(input).slice(start, start + length))
+}

packages/batch-submitter/test/helpers/utils/byte-utils.ts

+/**
+ * Generates a hex string of repeated bytes.
+ * @param byte Byte to repeat.
+ * @param len Number of times to repeat the byte.
+ * @return '0x'-prefixed hex string filled with the provided byte.
+ */
+export const makeHexString = (byte: string, len: number): string => {
+  return '0x' + byte.repeat(len)
+}
+
+/**
+ * Genereates an address with a repeated byte.
+ * @param byte Byte to repeat in the address.
+ * @return Address filled with the repeated byte.
+ */
+export const makeAddress = (byte: string): string => {
+  return makeHexString(byte, 20)
+}
+
+/**
+ * Removes '0x' from a hex string.
+ * @param str Hex string to remove '0x' from.
+ * @returns String without the '0x' prefix.
+ */
+export const remove0x = (str: string): string => {
+  if (str.startsWith('0x')) {
+    return str.slice(2)
+  } else {
+    return str
+  }
+}

packages/batch-submitter/test/helpers/utils/eth-time.ts

+export const getEthTime = async (provider: any): Promise<number> => {
+  return (await provider.getBlock('latest')).timestamp
+}
+
+export const setEthTime = async (
+  provider: any,
+  time: number
+): Promise<void> => {
+  await provider.send('evm_setNextBlockTimestamp', [time])
+}
+
+export const increaseEthTime = async (
+  provider: any,
+  amount: number
+): Promise<void> => {
+  await setEthTime(provider, (await getEthTime(provider)) + amount)
+  await provider.send('evm_mine', [])
+}
+
+export const getBlockTime = async (
+  provider: any,
+  block: number
+): Promise<number> => {
+  await provider.send('evm_mine', [])
+  return (await provider.getBlock(block)).timestamp
+}
+
+export const getNextBlockNumber = async (provider: any): Promise<number> => {
+  return (await provider.getBlock('latest')).number + 1
+}

packages/batch-submitter/test/helpers/utils/index.ts

+export * from './buffer-utils'
+export * from './byte-utils'
+export * from './eth-time'

packages/batch-submitter/test/setup.ts

+/* External Imports */
+import chai = require('chai')
+import sinonChai from 'sinon-chai'
+import Mocha from 'mocha'
+
+const should = chai.should()
+const expect = chai.expect
+chai.use(sinonChai)
+
+export { should, expect, chai, Mocha }

packages/batch-submitter/tsconfig.build.json

+{
+  "extends": "../../tsconfig.build.json",
+
+  "compilerOptions": {
+    "outDir": "./dist"
+  },
+
+  "include": [
+    "src/**/*"
+  ]
+}

packages/batch-submitter/tsconfig.json

+{
+  "extends": "../../tsconfig.json"
+}

packages/batch-submitter/tslint.json

+{
+    "extends": "../../tslint.base.json"
+}

packages/data-transport-layer/src/db/simple-db.ts

@@ -65,6 +65,7 @@ export class SimpleDB {
   }
 
   private _makeKey(key: string, index: number): string {
+    // prettier-ignore
     return `${key}:${BigNumber.from(index).toString().padStart(32, '0')}`
   }
 }

packages/data-transport-layer/src/services/main/service.ts

@@ -40,6 +40,7 @@ const optionSettings = {
   },
 }
 
+// prettier-ignore
 export class L1DataTransportService extends BaseService<L1DataTransportServiceOptions> {
   constructor(options: L1DataTransportServiceOptions) {
     super('L1 Data Transport Service', options, optionSettings)

packages/data-transport-layer/src/utils/common.ts

@@ -24,6 +24,7 @@ export const toRpcHexString = (n: number): string => {
   if (n === 0) {
     return '0x0'
   } else {
+    // prettier-ignore
     return '0x' + toHexString(n).slice(2).replace(/^0+/, '')
   }
 }

packages/smock/package.json

 {
   "name": "@eth-optimism/smock",
   "version": "1.0.0-alpha.3",
-  "main": "dist/index",
-  "types": "dist/index",
+  "main": "dist/src/index",
+  "types": "dist/src/index",
   "files": [
-    "dist/index"
+    "dist/src/index"
   ],
   "license": "MIT",
   "scripts": {

tslint.base.json

@@ -26,6 +26,7 @@
         "semicolon": false,
         "variable-name": false,
         "no-focused-test": true,
+        "array-type": false,
         "prettier": [true, "./prettier-config.json"]
     },
     "linterOptions": {