Merge pull request #2552 from ethereum-optimism/develop

Develop -> Master

.changeset/brown-spies-unite.md

+---
+'@eth-optimism/teleportr': patch
+---
+
+Add disburser balance to status

.changeset/green-kangaroos-beam.md

+---
+'@eth-optimism/fault-detector': minor
+---
+
+Releases the first public version of the fault detector

.changeset/lucky-buckets-crash.md

+---
+'@eth-optimism/teleportr': patch
+---
+
+Fix teleportr FailedDatabaseOperations method for upsert_disbursement

.changeset/shiny-bugs-scream.md

+---
+'@eth-optimism/teleportr': patch
+---
+
+Expose metrics server

.changeset/swift-flies-care.md

+---
+'@eth-optimism/contracts': patch
+---
+
+Remove unused network name parameter in contract deploy configs

.circleci/config.yml

@@ -86,6 +86,7 @@ jobs:
             - packages/contracts/node_modules
             - packages/core-utils/node_modules
             - packages/data-transport-layer/node_modules
+            - packages/fault-detector/node_modules
             - packages/message-relayer/node_modules
             - packages/replica-healthcheck/node_modules
             - packages/sdk/node_modules
@@ -351,6 +352,11 @@ workflows:
           package_name: sdk
           requires:
             - yarn-monorepo
+      - js-lint-test:
+          name: fault-detector-tests
+          package_name: fault-detector
+          requires:
+            - yarn-monorepo
       - js-lint-test:
           name: message-relayer-tests
           package_name: message-relayer
@@ -433,6 +439,14 @@ workflows:
           docker_context: .
           context:
             - optimism
+      - docker-publish:
+          name: fault-detector-release
+          docker_file: ops/docker/Dockerfile.packages
+          docker_tags: ethereumoptimism/fault-detector:nightly
+          docker_context: .
+          target: fault-detector
+          context:
+            - optimism
       - docker-publish:
           name: message-relayer-release
           docker_file: ops/docker/Dockerfile.packages
@@ -450,9 +464,9 @@ workflows:
           context:
             - optimism
       - docker-publish:
-          name: deployer-release
+          name: integration-tests-release
           docker_file: ops/docker/Dockerfile.packages
-          docker_tags: ethereumoptimism/deployer:nightly
+          docker_tags: ethereumoptimism/integration-tests:nightly
           docker_context: .
           target: integration-tests
           context:

.github/labeler.yml

@@ -7,6 +7,7 @@
   - 'packages/contracts/**/*'
   - 'packages/data-transport-layer/**/*'
   - 'packages/message-relayer/**/*'
+  - 'packages/fault-detector/**/*'
   - 'patches/**/*'
 
 M-ci:

.github/workflows/publish-canary.yml

@@ -17,6 +17,7 @@ jobs:
     outputs:
       l2geth: ${{ steps.packages.outputs.l2geth }}
       message-relayer: ${{ steps.packages.outputs.message-relayer }}
+      fault-detector: ${{ steps.packages.outputs.fault-detector }}
       data-transport-layer: ${{ steps.packages.outputs.data-transport-layer }}
       contracts: ${{ steps.packages.outputs.contracts }}
       gas-oracle: ${{ steps.packages.outputs.gas-oracle }}
@@ -196,6 +197,33 @@ jobs:
           push: true
           tags: ethereumoptimism/message-relayer:${{ needs.canary-publish.outputs.canary-docker-tag }}
 
+  fault-detector:
+    name: Publish Fault Detector Version ${{ needs.canary-publish.outputs.canary-docker-tag }}
+    needs: canary-publish
+    if: needs.canary-publish.outputs.fault-detector != ''
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./ops/docker/Dockerfile.packages
+          target: relayer
+          push: true
+          tags: ethereumoptimism/fault-detector:${{ needs.canary-publish.outputs.canary-docker-tag }}
+
   data-transport-layer:
     name: Publish Data Transport Layer Version ${{ needs.canary-publish.outputs.canary-docker-tag }}
     needs: canary-publish

.github/workflows/release.yml

@@ -13,6 +13,7 @@ jobs:
     outputs:
       l2geth: ${{ steps.packages.outputs.l2geth }}
       message-relayer: ${{ steps.packages.outputs.message-relayer }}
+      fault-detector: ${{ steps.packages.outputs.fault-detector }}
       data-transport-layer: ${{ steps.packages.outputs.data-transport-layer }}
       contracts: ${{ steps.packages.outputs.contracts }}
       gas-oracle: ${{ steps.packages.outputs.gas-oracle }}
@@ -339,6 +340,33 @@ jobs:
           push: true
           tags: ethereumoptimism/message-relayer:${{ needs.release.outputs.message-relayer }},ethereumoptimism/message-relayer:latest
 
+  fault-detector:
+    name: Publish Fault Detector Version ${{ needs.release.outputs.fault-detector }}
+    needs: release
+    if: needs.release.outputs.fault-detector != ''
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./ops/docker/Dockerfile.packages
+          target: fault-detector
+          push: true
+          tags: ethereumoptimism/fault-detector:${{ needs.release.outputs.fault-detector }},ethereumoptimism/fault-detector:latest
+
   data-transport-layer:
     name: Publish Data Transport Layer Version ${{ needs.release.outputs.data-transport-layer }}
     needs: release

.vscode/settings.json

@@ -10,6 +10,7 @@
     {"directory": "packages/data-transport-layer", "changeProcessCWD": true },
     {"directory": "packages/batch-submitter", "changeProcessCWD": true },
     {"directory": "packages/message-relayer", "changeProcessCWD": true },
+    {"directory": "packages/fault-detector", "changeProcessCWD": true },
   ],
   "eslint.nodePath": "./node_modules/eslint/bin/",
   "eslint.format.enable": true,

README.md

@@ -35,6 +35,7 @@ root
 │   ├── <a href="./packages/core-utils">core-utils</a>: Low-level utilities that make building Optimism easier
 │   ├── <a href="./packages/common-ts">common-ts</a>: Common tools for building apps in TypeScript
 │   ├── <a href="./packages/data-transport-layer">data-transport-layer</a>: Service for indexing Optimism-related L1 data
+│   ├── <a href="./packages/fault-detector">fault-detector</a>: Service for detecting faulty L2 output proposals
 │   ├── <a href="./packages/message-relayer">message-relayer</a>: Tool for automatically relaying L1<>L2 messages in development
 │   └── <a href="./packages/replica-healthcheck">replica-healthcheck</a>: Service for monitoring the health of a replica node
 ├── <a href="./go">go</a>

ops/docker-compose.yml

@@ -117,6 +117,23 @@ services:
       MESSAGE_RELAYER__L1_WALLET: '0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97'
       RETRIES: 60
 
+  fault_detector:
+    depends_on:
+      - l1_chain
+      - l2geth
+    deploy:
+      replicas: 0
+    build:
+      context: ..
+      dockerfile: ./ops/docker/Dockerfile.packages
+      target: fault-detector
+    image: ethereumoptimism/fault-detector:${DOCKER_TAG_FAULT_DETECTOR:-latest}
+    entrypoint: ./detector.sh
+    environment:
+      FAULT_DETECTOR__L1_RPC_PROVIDER: http://l1_chain:8545
+      FAULT_DETECTOR__L2_RPC_PROVIDER: http://l2geth:8545
+      RETRIES: 60
+
   verifier:
     depends_on:
       - l1_chain

ops/docker/Dockerfile.packages

@@ -17,6 +17,7 @@ COPY packages/common-ts/package.json ./packages/common-ts/package.json
 COPY packages/contracts/package.json ./packages/contracts/package.json
 COPY packages/data-transport-layer/package.json ./packages/data-transport-layer/package.json
 COPY packages/message-relayer/package.json ./packages/message-relayer/package.json
+COPY packages/fault-detector/package.json ./packages/fault-detector/package.json
 COPY packages/replica-healthcheck/package.json ./packages/replica-healthcheck/package.json
 COPY integration-tests/package.json ./integration-tests/package.json
 
@@ -53,6 +54,12 @@ COPY ./ops/scripts/relayer.sh .
 CMD ["npm", "run", "start"]
 
 
+FROM base as fault-detector
+WORKDIR /opt/optimism/packages/fault-detector
+COPY ./ops/scripts/detector.sh .
+CMD ["npm", "run", "start"]
+
+
 FROM base as replica-healthcheck
 WORKDIR /opt/optimism/packages/replica-healthcheck
 ENTRYPOINT ["npm", "run", "start"]

ops/scripts/build-ci.sh

@@ -23,6 +23,7 @@ wait
 build deployer "ethereumoptimism/deployer:latest" "./ops/docker/Dockerfile.deployer" .
 build dtl "ethereumoptimism/data-transport-layer:latest" "./ops/docker/Dockerfile.data-transport-layer" .
 build relayer "ethereumoptimism/message-relayer:latest" "./ops/docker/Dockerfile.message-relayer" .
+build relayer "ethereumoptimism/fault-detector:latest" "./ops/docker/Dockerfile.fault-detector" .
 build integration-tests "ethereumoptimism/integration-tests:latest" "./ops/docker/Dockerfile.integration-tests" .
 
 wait

ops/scripts/detector.sh

+#!/bin/bash
+
+set -e
+
+RETRIES=${RETRIES:-60}
+
+# waits for l2geth to be up
+curl \
+    --fail \
+    --show-error \
+    --silent \
+    --output /dev/null \
+    --retry-connrefused \
+    --retry $RETRIES \
+    --retry-delay 1 \
+    $FAULT_DETECTOR__L2_RPC_PROVIDER
+
+# go
+exec yarn start

packages/contracts/deploy-config/goerli-nightly.ts

 import { DeployConfig } from '../src/deploy-config'
 
 const config: DeployConfig = {
-  network: 'goerli-nightly',
   l1BlockTimeSeconds: 15,
   l2BlockGasLimit: 15_000_000,
   l2ChainId: 421,

packages/contracts/deploy-config/goerli.ts

 import { DeployConfig } from '../src/deploy-config'
 
 const config: DeployConfig = {
-  network: 'goerli',
   l1BlockTimeSeconds: 15,
   l2BlockGasLimit: 15_000_000,
   l2ChainId: 420,

packages/contracts/deploy-config/kovan.ts

 import { DeployConfig } from '../src/deploy-config'
 
 const config: DeployConfig = {
-  network: 'kovan',
   numDeployConfirmations: 1,
   gasPrice: 5_000_000_000,
   l1BlockTimeSeconds: 15,

packages/contracts/deploy-config/local.ts

 import { DeployConfig } from '../src/deploy-config'
 
 const config: DeployConfig = {
-  network: 'local',
   l1BlockTimeSeconds: 15,
   l2BlockGasLimit: 15_000_000,
   l2ChainId: 17,

packages/contracts/deploy-config/mainnet.ts

 import { DeployConfig } from '../src/deploy-config'
 
 const config: DeployConfig = {
-  network: 'mainnet',
   numDeployConfirmations: 4,
   gasPrice: 150_000_000_000,
   l1BlockTimeSeconds: 15,

packages/contracts/src/deploy-config.ts

@@ -4,12 +4,6 @@ import { ethers } from 'ethers'
  * Defines the configuration for a deployment.
  */
 export interface DeployConfig {
-  /**
-   * Name of the network to deploy to. Must be the name of one of the networks listed in
-   * hardhat.config.ts.
-   */
-  network: string
-
   /**
    * Whether or not this network is a forked network.
    */
@@ -135,9 +129,6 @@ const configSpec: {
     default?: any
   }
 } = {
-  network: {
-    type: 'string',
-  },
   isForkedNetwork: {
     type: 'boolean',
     default: false,

packages/data-transport-layer/openapi.yml

@@ -10,9 +10,11 @@ components:
       type: object
       properties:
         blockNumber:
-          type: uint64
+          type: integer
+          format: "int64"
         timestamp:
-          type: uint64
+          type: integer
+          format: "int64"
         blockHash:
           type: string
     EnqueueEntry:
@@ -131,22 +133,22 @@ components:
     StateRootEntry:
       type: object
       properties:
-      index:
-        type: number
-      batchIndex:
-        type: number
-      value:
-        type: string
+        index:
+          type: number
+        batchIndex:
+          type: number
+        value:
+          type: string
 
     StateRootResponse:
       type: object
       properties:
-      batch:
-        type: object
-        $ref: '#/components/schemas/BatchEntry'
-      stateRoot:
-        type: object
-        $ref: '#/components/schemas/StateRootEntry'
+        batch:
+          type: object
+          $ref: '#/components/schemas/BatchEntry'
+        stateRoot:
+          type: object
+          $ref: '#/components/schemas/StateRootEntry'
 
     StateRootBatchResponse:
       type: object
@@ -178,9 +180,11 @@ paths:
                   syncing:
                     type: boolean
                   currentTransactionIndex:
-                    type: uint64
+                    type: integer
+                    format: "int64"
                   highestKnownTransactionIndex:
-                    type: uint64
+                    type: integer
+                    format: "int64"
 
   /eth/context/latest:
     get:
@@ -198,6 +202,13 @@ paths:
 
   /eth/context/blocknumber/{number}:
     get:
+      parameters:
+        - in: path
+          name: number
+          schema:
+            type: integer
+            format: "int64"
+          required: true
       summary: Returns the Ethereum Layer one context at a specific height
       description: |
         This returns the L1 blocknumber, block hash and timestamp corresponding to a specific
@@ -225,6 +236,13 @@ paths:
 
   /enqueue/index/{index}:
     get:
+      parameters:
+        - in: path
+          name: index
+          schema:
+            type: integer
+            format: "int64"
+          required: true
       summary: Returns the enqueued transaction by index
       description: |
         This returns the Canonical Transaction Chain `enqueue()` by index
@@ -252,6 +270,13 @@ paths:
 
   /transaction/index/{index}:
     get:
+      parameters:
+        - in: path
+          name: index
+          schema:
+            type: integer
+            format: "int64"
+          required: true
       summary: Returns a Canonical Transaction Chain transaction by index
       description: |
         This returns a transaction that has been appended to the Canonical Transaction
@@ -280,6 +305,13 @@ paths:
 
   /batch/transaction/index/{index}:
     get:
+      parameters:
+        - in: path
+          name: index
+          schema:
+            type: integer
+            format: "int64"
+          required: true
       summary: Returns the Batch to be appended to the Canonical Transaction Chain by index
       description: |
         This returns a batch that has been appended to the Canonical Transaction
@@ -307,6 +339,13 @@ paths:
 
   /stateroot/index/{index}:
     get:
+      parameters:
+        - in: path
+          name: index
+          schema:
+            type: integer
+            format: "int64"
+          required: true
       summary: Returns the state root by index
       description: |
         This returns a state root appended to the State Commitment Chain by index
@@ -334,6 +373,13 @@ paths:
   /batch/stateroot/index/{index}:
     get:
       summary: Returns the state root batch by index
+      parameters:
+        - in: path
+          name: index
+          schema:
+            type: integer
+            format: "int64"
+          required: true
       description: |
         This returns a state root batch appended to the State Commitment Chain by index
       responses:

packages/fault-detector/.env.example

+# URL for an L1 RPC provider, used to query L2 output proposals
+FAULT_DETECTOR__L1_RPC_PROVIDER=
+
+# URL for an L2 RPC provider, used to query canonical L2 state
+FAULT_DETECTOR__L2_RPC_PROVIDER=

packages/fault-detector/.eslintrc.js

+module.exports = {
+  extends: '../../.eslintrc.js',
+}

packages/fault-detector/.gitignore

+node_modules/
+build/

packages/fault-detector/.lintstagedrc.yml

+"*.{ts,js}":
+  - eslint

packages/fault-detector/.prettierrc.js

+module.exports = {
+  ...require('../../.prettierrc.js'),
+};

packages/fault-detector/LICENSE

+(The MIT License)
+
+Copyright 2020-2021 Optimism
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

packages/fault-detector/README.md

+# @eth-optimism/fault-detector
+
+The `fault-detector` is a simple service for detecting discrepancies between your local view of the Optimism network and the L2 output proposals published to Ethereum.
+
+## Installation
+
+Clone, install, and build the Optimism monorepo:
+
+```
+git clone https://github.com/ethereum-optimism/optimism.git
+yarn install
+yarn build
+```
+
+## Running the service
+
+Copy `.env.example` into a new file named `.env`, then set the environment variables listed there.
+Once your environment variables have been set, run the service via:
+
+```
+yarn start
+```

packages/fault-detector/package.json

+{
+  "private": true,
+  "name": "@eth-optimism/fault-detector",
+  "version": "0.0.1",
+  "description": "[Optimism] Service for detecting faulty L2 output proposals",
+  "main": "dist/index",
+  "types": "dist/index",
+  "files": [
+    "dist/*"
+  ],
+  "scripts": {
+    "start": "ts-node ./src/service.ts",
+    "test:coverage": "echo 'No tests defined.'",
+    "build": "tsc -p tsconfig.build.json",
+    "clean": "rimraf dist/ ./tsconfig.build.tsbuildinfo",
+    "lint": "yarn lint:fix && yarn lint:check",
+    "pre-commit": "lint-staged",
+    "lint:fix": "yarn lint:check --fix",
+    "lint:check": "eslint . --max-warnings=0"
+  },
+  "keywords": [
+    "optimism",
+    "ethereum",
+    "fault",
+    "detector"
+  ],
+  "homepage": "https://github.com/ethereum-optimism/optimism/tree/develop/packages/fault-detector#readme",
+  "license": "MIT",
+  "author": "Optimism PBC",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ethereum-optimism/optimism.git"
+  },
+  "devDependencies": {
+    "@types/dateformat": "^5.0.0",
+    "@types/node": "^17.0.23",
+    "dateformat": "^4.5.1",
+    "ethers": "^5.6.2",
+    "ts-node": "^10.7.0",
+    "typescript": "^4.6.3"
+  },
+  "dependencies": {
+    "@eth-optimism/common-ts": "^0.2.4",
+    "@eth-optimism/core-utils": "^0.8.1",
+    "@eth-optimism/sdk": "^1.0.2",
+    "@ethersproject/abstract-provider": "^5.6.0"
+  }
+}

packages/fault-detector/src/index.ts

+export * from './service'

packages/fault-detector/src/service.ts

+import { BaseServiceV2, Gauge, validators } from '@eth-optimism/common-ts'
+import { sleep, toRpcHexString } from '@eth-optimism/core-utils'
+import { CrossChainMessenger } from '@eth-optimism/sdk'
+import { Provider } from '@ethersproject/abstract-provider'
+import { ethers } from 'ethers'
+import dateformat from 'dateformat'
+
+type Options = {
+  l1RpcProvider: Provider
+  l2RpcProvider: Provider
+  startBatchIndex: number
+}
+
+type Metrics = {
+  highestCheckedBatchIndex: Gauge
+  highestKnownBatchIndex: Gauge
+  isCurrentlyMismatched: Gauge
+  inUnexpectedErrorState: Gauge
+}
+
+type State = {
+  messenger: CrossChainMessenger
+  highestCheckedBatchIndex: number
+}
+
+export class FaultDetector extends BaseServiceV2<Options, Metrics, State> {
+  constructor(options?: Partial<Options>) {
+    super({
+      name: 'fault-detector',
+      loop: true,
+      loopIntervalMs: 1000,
+      options,
+      optionsSpec: {
+        l1RpcProvider: {
+          validator: validators.provider,
+          desc: 'Provider for interacting with L1',
+        },
+        l2RpcProvider: {
+          validator: validators.provider,
+          desc: 'Provider for interacting with L2',
+        },
+        startBatchIndex: {
+          validator: validators.num,
+          default: 0,
+          desc: 'Batch index to start checking from',
+        },
+      },
+      metricsSpec: {
+        highestCheckedBatchIndex: {
+          type: Gauge,
+          desc: 'Highest good batch index',
+        },
+        highestKnownBatchIndex: {
+          type: Gauge,
+          desc: 'Highest known batch index',
+        },
+        isCurrentlyMismatched: {
+          type: Gauge,
+          desc: '0 if state is ok, 1 if state is mismatched',
+        },
+        inUnexpectedErrorState: {
+          type: Gauge,
+          desc: '0 if service is ok, 1 service is in unexpected error state',
+        },
+      },
+    })
+  }
+
+  async init(): Promise<void> {
+    const network = await this.options.l1RpcProvider.getNetwork()
+    this.state.messenger = new CrossChainMessenger({
+      l1SignerOrProvider: this.options.l1RpcProvider,
+      l2SignerOrProvider: this.options.l2RpcProvider,
+      l1ChainId: network.chainId,
+    })
+
+    this.state.highestCheckedBatchIndex = this.options.startBatchIndex
+  }
+
+  async main(): Promise<void> {
+    const latestBatchIndex =
+      await this.state.messenger.contracts.l1.StateCommitmentChain.getTotalBatches()
+    if (this.state.highestCheckedBatchIndex >= latestBatchIndex.toNumber()) {
+      await sleep(15000)
+      return
+    }
+
+    this.metrics.highestKnownBatchIndex.set(latestBatchIndex.toNumber())
+
+    this.logger.info(`checking batch`, {
+      batchIndex: this.state.highestCheckedBatchIndex,
+    })
+
+    const targetEvents =
+      await this.state.messenger.contracts.l1.StateCommitmentChain.queryFilter(
+        this.state.messenger.contracts.l1.StateCommitmentChain.filters.StateBatchAppended(
+          this.state.highestCheckedBatchIndex
+        )
+      )
+
+    if (targetEvents.length === 0) {
+      this.logger.error(`unable to find event for batch`, {
+        batchIndex: this.state.highestCheckedBatchIndex,
+      })
+      this.metrics.inUnexpectedErrorState.set(1)
+      return
+    }
+
+    if (targetEvents.length > 1) {
+      this.logger.error(`found too many events for batch`, {
+        batchIndex: this.state.highestCheckedBatchIndex,
+      })
+      this.metrics.inUnexpectedErrorState.set(1)
+      return
+    }
+
+    const targetEvent = targetEvents[0]
+    const batchTransaction = await targetEvent.getTransaction()
+    const [stateRoots] =
+      this.state.messenger.contracts.l1.StateCommitmentChain.interface.decodeFunctionData(
+        'appendStateBatch',
+        batchTransaction.data
+      )
+
+    const batchStart = targetEvent.args._prevTotalElements.toNumber() + 1
+    const batchSize = targetEvent.args._batchSize.toNumber()
+
+    // `getBlockRange` has a limit of 1000 blocks, so we have to break this request out into
+    // multiple requests of maximum 1000 blocks in the case that batchSize > 1000.
+    let blocks: any[] = []
+    for (let i = 0; i < batchSize; i += 1000) {
+      const provider = this.options
+        .l2RpcProvider as ethers.providers.JsonRpcProvider
+      blocks = blocks.concat(
+        await provider.send('eth_getBlockRange', [
+          toRpcHexString(batchStart + i),
+          toRpcHexString(batchStart + i + Math.min(batchSize - i, 1000) - 1),
+          false,
+        ])
+      )
+    }
+
+    for (const [i, stateRoot] of stateRoots.entries()) {
+      if (blocks[i].stateRoot !== stateRoot) {
+        this.metrics.isCurrentlyMismatched.set(1)
+        const fpw =
+          await this.state.messenger.contracts.l1.StateCommitmentChain.FRAUD_PROOF_WINDOW()
+        this.logger.error(`state root mismatch`, {
+          blockNumber: blocks[i].number,
+          expectedStateRoot: blocks[i].stateRoot,
+          actualStateRoot: stateRoot,
+          finalizationTime: dateformat(
+            new Date(
+              (ethers.BigNumber.from(blocks[i].timestamp).toNumber() +
+                fpw.toNumber()) *
+                1000
+            ),
+            'mmmm dS, yyyy, h:MM:ss TT'
+          ),
+        })
+        return
+      }
+    }
+
+    this.metrics.highestCheckedBatchIndex.set(
+      this.state.highestCheckedBatchIndex
+    )
+    this.state.highestCheckedBatchIndex++
+
+    // If we got through the above without throwing an error, we should be fine to reset.
+    this.metrics.isCurrentlyMismatched.set(0)
+    this.metrics.inUnexpectedErrorState.set(0)
+  }
+}
+
+if (require.main === module) {
+  const service = new FaultDetector()
+  service.run()
+}

packages/fault-detector/tsconfig.build.json

+{
+  "extends": "../../tsconfig.build.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist"
+  },
+  "include": [
+    "src/**/*"
+  ]
+}

packages/fault-detector/tsconfig.json

+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "resolveJsonModule": true
+  }
+}

teleportr/api/server.go

@@ -18,6 +18,7 @@ import (
 	bsscore "github.com/ethereum-optimism/optimism/bss-core"
 	"github.com/ethereum-optimism/optimism/bss-core/dial"
 	"github.com/ethereum-optimism/optimism/bss-core/drivers"
+	"github.com/ethereum-optimism/optimism/bss-core/metrics"
 	"github.com/ethereum-optimism/optimism/bss-core/txmgr"
 	"github.com/ethereum-optimism/optimism/teleportr/bindings/deposit"
 	"github.com/ethereum-optimism/optimism/teleportr/db"
@@ -55,6 +56,11 @@ func Main(gitVersion string) func(*cli.Context) error {
 			return err
 		}
 
+		disburserWalletAddr, err := bsscore.ParseAddress(cfg.DisburserWalletAddress)
+		if err != nil {
+			return err
+		}
+
 		l1Client, err := dial.L1EthClientWithTimeout(
 			ctx, cfg.L1EthRpc, cfg.DisableHTTP2,
 		)
@@ -63,6 +69,14 @@ func Main(gitVersion string) func(*cli.Context) error {
 		}
 		defer l1Client.Close()
 
+		l2Client, err := dial.L1EthClientWithTimeout(
+			ctx, cfg.L2EthRpc, cfg.DisableHTTP2,
+		)
+		if err != nil {
+			return err
+		}
+		defer l2Client.Close()
+
 		depositContract, err := deposit.NewTeleportrDeposit(
 			depositAddr, l1Client,
 		)
@@ -84,11 +98,17 @@ func Main(gitVersion string) func(*cli.Context) error {
 		}
 		defer database.Close()
 
+		if cfg.MetricsServerEnable {
+			go metrics.RunServer(cfg.MetricsHostname, cfg.MetricsPort)
+		}
+
 		server := NewServer(
 			ctx,
 			l1Client,
+			l2Client,
 			database,
 			depositAddr,
+			disburserWalletAddr,
 			depositContract,
 			cfg.NumConfirmations,
 		)
@@ -124,33 +144,44 @@ func Main(gitVersion string) func(*cli.Context) error {
 }
 
 type Config struct {
-	Hostname          string
-	Port              uint16
-	L1EthRpc          string
-	DepositAddress    string
-	NumConfirmations  uint64
-	PostgresHost      string
-	PostgresPort      uint16
-	PostgresUser      string
-	PostgresPassword  string
-	PostgresDBName    string
-	PostgresEnableSSL bool
-	DisableHTTP2      bool
+	Hostname               string
+	Port                   uint16
+	L1EthRpc               string
+	L2EthRpc               string
+	DepositAddress         string
+	NumConfirmations       uint64
+	DisburserWalletAddress string
+	PostgresHost           string
+	PostgresPort           uint16
+	PostgresUser           string
+	PostgresPassword       string
+	PostgresDBName         string
+	PostgresEnableSSL      bool
+	MetricsServerEnable    bool
+	MetricsHostname        string
+	MetricsPort            uint64
+	DisableHTTP2           bool
 }
 
 func NewConfig(ctx *cli.Context) (Config, error) {
 	return Config{
-		Hostname:          ctx.GlobalString(flags.APIHostnameFlag.Name),
-		Port:              uint16(ctx.GlobalUint64(flags.APIPortFlag.Name)),
-		L1EthRpc:          ctx.GlobalString(flags.L1EthRpcFlag.Name),
-		DepositAddress:    ctx.GlobalString(flags.DepositAddressFlag.Name),
-		NumConfirmations:  ctx.GlobalUint64(flags.NumDepositConfirmationsFlag.Name),
-		PostgresHost:      ctx.GlobalString(flags.PostgresHostFlag.Name),
-		PostgresPort:      uint16(ctx.GlobalUint64(flags.PostgresPortFlag.Name)),
-		PostgresUser:      ctx.GlobalString(flags.PostgresUserFlag.Name),
-		PostgresPassword:  ctx.GlobalString(flags.PostgresPasswordFlag.Name),
-		PostgresDBName:    ctx.GlobalString(flags.PostgresDBNameFlag.Name),
-		PostgresEnableSSL: ctx.GlobalBool(flags.PostgresEnableSSLFlag.Name),
+		Hostname:               ctx.GlobalString(flags.APIHostnameFlag.Name),
+		Port:                   uint16(ctx.GlobalUint64(flags.APIPortFlag.Name)),
+		L1EthRpc:               ctx.GlobalString(flags.L1EthRpcFlag.Name),
+		L2EthRpc:               ctx.GlobalString(flags.L2EthRpcFlag.Name),
+		DepositAddress:         ctx.GlobalString(flags.DepositAddressFlag.Name),
+		NumConfirmations:       ctx.GlobalUint64(flags.NumDepositConfirmationsFlag.Name),
+		DisburserWalletAddress: ctx.GlobalString(flags.DisburserWalletAddressFlag.Name),
+		PostgresHost:           ctx.GlobalString(flags.PostgresHostFlag.Name),
+		PostgresPort:           uint16(ctx.GlobalUint64(flags.PostgresPortFlag.Name)),
+		PostgresUser:           ctx.GlobalString(flags.PostgresUserFlag.Name),
+		PostgresPassword:       ctx.GlobalString(flags.PostgresPasswordFlag.Name),
+		PostgresDBName:         ctx.GlobalString(flags.PostgresDBNameFlag.Name),
+		PostgresEnableSSL:      ctx.GlobalBool(flags.PostgresEnableSSLFlag.Name),
+		MetricsServerEnable:    ctx.GlobalBool(flags.MetricsServerEnableFlag.Name),
+		MetricsHostname:        ctx.GlobalString(flags.MetricsHostnameFlag.Name),
+		MetricsPort:            ctx.GlobalUint64(flags.MetricsPortFlag.Name),
+		DisableHTTP2:           ctx.GlobalBool(flags.HTTP2DisableFlag.Name),
 	}, nil
 }
 
@@ -162,12 +193,14 @@ const (
 )
 
 type Server struct {
-	ctx              context.Context
-	l1Client         *ethclient.Client
-	database         *db.Database
-	depositAddr      common.Address
-	depositContract  *deposit.TeleportrDeposit
-	numConfirmations uint64
+	ctx                 context.Context
+	l1Client            *ethclient.Client
+	l2Client            *ethclient.Client
+	database            *db.Database
+	depositAddr         common.Address
+	disburserWalletAddr common.Address
+	depositContract     *deposit.TeleportrDeposit
+	numConfirmations    uint64
 
 	httpServer *http.Server
 }
@@ -175,23 +208,26 @@ type Server struct {
 func NewServer(
 	ctx context.Context,
 	l1Client *ethclient.Client,
+	l2Client *ethclient.Client,
 	database *db.Database,
 	depositAddr common.Address,
+	disburserWalletAddr common.Address,
 	depositContract *deposit.TeleportrDeposit,
 	numConfirmations uint64,
 ) *Server {
-
 	if numConfirmations == 0 {
 		panic("NumConfirmations cannot be zero")
 	}
 
 	return &Server{
-		ctx:              ctx,
-		l1Client:         l1Client,
-		database:         database,
-		depositAddr:      depositAddr,
-		depositContract:  depositContract,
-		numConfirmations: numConfirmations,
+		ctx:                 ctx,
+		l1Client:            l1Client,
+		l2Client:            l2Client,
+		database:            database,
+		depositAddr:         depositAddr,
+		disburserWalletAddr: disburserWalletAddr,
+		depositContract:     depositContract,
+		numConfirmations:    numConfirmations,
 	}
 }
 
@@ -234,11 +270,12 @@ func HandleHealthz(w http.ResponseWriter, r *http.Request) {
 }
 
 type StatusResponse struct {
-	CurrentBalanceWei   string `json:"current_balance_wei"`
-	MaximumBalanceWei   string `json:"maximum_balance_wei"`
-	MinDepositAmountWei string `json:"min_deposit_amount_wei"`
-	MaxDepositAmountWei string `json:"max_deposit_amount_wei"`
-	IsAvailable         bool   `json:"is_available"`
+	DisburserWalletBalanceWei string `json:"disburser_wallet_balance_wei"`
+	DepositContractBalanceWei string `json:"deposit_contract_balance_wei"`
+	MaximumBalanceWei         string `json:"maximum_balance_wei"`
+	MinDepositAmountWei       string `json:"min_deposit_amount_wei"`
+	MaxDepositAmountWei       string `json:"max_deposit_amount_wei"`
+	IsAvailable               bool   `json:"is_available"`
 }
 
 func (s *Server) HandleStatus(
@@ -273,21 +310,28 @@ func (s *Server) HandleStatus(
 
 	curBalance, err := s.l1Client.BalanceAt(ctx, s.depositAddr, nil)
 	if err != nil {
-		rpcErrorsTotal.WithLabelValues("balance_at").Inc()
+		rpcErrorsTotal.WithLabelValues("deposit_balance_at").Inc()
+		return err
+	}
+
+	disburserWalletBal, err := s.l2Client.BalanceAt(ctx, s.disburserWalletAddr, nil)
+	if err != nil {
+		rpcErrorsTotal.WithLabelValues("disburser_wallet_balance_at").Inc()
 		return err
 	}
 
 	balanceAfterMaxDeposit := new(big.Int).Add(
 		curBalance, maxDepositAmount,
 	)
-	isAvailable := maxBalance.Cmp(balanceAfterMaxDeposit) >= 0
+	isAvailable := maxBalance.Cmp(balanceAfterMaxDeposit) >= 0 && disburserWalletBal.Cmp(maxDepositAmount) > 0
 
 	resp := StatusResponse{
-		CurrentBalanceWei:   curBalance.String(),
-		MaximumBalanceWei:   maxBalance.String(),
-		MinDepositAmountWei: minDepositAmount.String(),
-		MaxDepositAmountWei: maxDepositAmount.String(),
-		IsAvailable:         isAvailable,
+		DisburserWalletBalanceWei: disburserWalletBal.String(),
+		DepositContractBalanceWei: curBalance.String(),
+		MaximumBalanceWei:         maxBalance.String(),
+		MinDepositAmountWei:       minDepositAmount.String(),
+		MaxDepositAmountWei:       maxDepositAmount.String(),
+		IsAvailable:               isAvailable,
 	}
 
 	jsonResp, err := json.Marshal(resp)

teleportr/drivers/disburser/driver.go

@@ -436,7 +436,7 @@ func (d *Driver) processPendingTxs(ctx context.Context) error {
 	d.metrics.SuccessfulDisbursements.Add(float64(successfulDisbursements))
 	d.metrics.FailedDisbursements.Add(float64(failedDisbursements))
 	d.metrics.FailedDatabaseMethods.With(DBMethodUpsertDisbursement).
-		Inc()
+		Add(float64(failedUpserts))
 
 	// We have completed our post-processing once all of the disbursements are
 	// written without failures.

teleportr/flags/api_flags.go

@@ -24,12 +24,20 @@ var (
 		Required: true,
 		EnvVar:   prefixAPIEnvVar("PORT"),
 	}
+	DisburserWalletAddressFlag = cli.StringFlag{
+		Name:     "disburser-wallet-address",
+		Usage:    "The address of the disburser wallet",
+		Required: true,
+		EnvVar:   prefixAPIEnvVar("DISBURSER_WALLET_ADDRESS"),
+	}
 )
 
 var APIFlags = []cli.Flag{
 	APIHostnameFlag,
 	APIPortFlag,
+	DisburserWalletAddressFlag,
 	L1EthRpcFlag,
+	L2EthRpcFlag,
 	DepositAddressFlag,
 	NumDepositConfirmationsFlag,
 	PostgresHostFlag,
@@ -38,4 +46,8 @@ var APIFlags = []cli.Flag{
 	PostgresPasswordFlag,
 	PostgresDBNameFlag,
 	PostgresEnableSSLFlag,
+	MetricsServerEnableFlag,
+	MetricsHostnameFlag,
+	MetricsPortFlag,
+	HTTP2DisableFlag,
 }