feat: introduce fault-detector package (#2549)

Introduces the fault-detector package. This is basically a copy/paste of
the old state-root-alarm package, slightly cleaned up and renamed to a
more appropriate name.
Co-authored-by: Mark Tyneway <mark.tyneway@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

.changeset/green-kangaroos-beam.md

+---
+'@eth-optimism/fault-detector': minor
+---
+
+Releases the first public version of the fault detector

.circleci/config.yml

@@ -86,6 +86,7 @@ jobs:
             - packages/contracts/node_modules
             - packages/core-utils/node_modules
             - packages/data-transport-layer/node_modules
+            - packages/fault-detector/node_modules
             - packages/message-relayer/node_modules
             - packages/replica-healthcheck/node_modules
             - packages/sdk/node_modules
@@ -351,6 +352,11 @@ workflows:
           package_name: sdk
           requires:
             - yarn-monorepo
+      - js-lint-test:
+          name: fault-detector-tests
+          package_name: fault-detector
+          requires:
+            - yarn-monorepo
       - js-lint-test:
           name: message-relayer-tests
           package_name: message-relayer
@@ -433,6 +439,14 @@ workflows:
           docker_context: .
           context:
             - optimism
+      - docker-publish:
+          name: fault-detector-release
+          docker_file: ops/docker/Dockerfile.packages
+          docker_tags: ethereumoptimism/fault-detector:nightly
+          docker_context: .
+          target: fault-detector
+          context:
+            - optimism
       - docker-publish:
           name: message-relayer-release
           docker_file: ops/docker/Dockerfile.packages

.github/labeler.yml

@@ -7,6 +7,7 @@
   - 'packages/contracts/**/*'
   - 'packages/data-transport-layer/**/*'
   - 'packages/message-relayer/**/*'
+  - 'packages/fault-detector/**/*'
   - 'patches/**/*'
 
 M-ci:

.github/workflows/publish-canary.yml

@@ -17,6 +17,7 @@ jobs:
     outputs:
       l2geth: ${{ steps.packages.outputs.l2geth }}
       message-relayer: ${{ steps.packages.outputs.message-relayer }}
+      fault-detector: ${{ steps.packages.outputs.fault-detector }}
       data-transport-layer: ${{ steps.packages.outputs.data-transport-layer }}
       contracts: ${{ steps.packages.outputs.contracts }}
       gas-oracle: ${{ steps.packages.outputs.gas-oracle }}
@@ -196,6 +197,33 @@ jobs:
           push: true
           tags: ethereumoptimism/message-relayer:${{ needs.canary-publish.outputs.canary-docker-tag }}
 
+  fault-detector:
+    name: Publish Fault Detector Version ${{ needs.canary-publish.outputs.canary-docker-tag }}
+    needs: canary-publish
+    if: needs.canary-publish.outputs.fault-detector != ''
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./ops/docker/Dockerfile.packages
+          target: relayer
+          push: true
+          tags: ethereumoptimism/fault-detector:${{ needs.canary-publish.outputs.canary-docker-tag }}
+
   data-transport-layer:
     name: Publish Data Transport Layer Version ${{ needs.canary-publish.outputs.canary-docker-tag }}
     needs: canary-publish

.github/workflows/release.yml

@@ -13,6 +13,7 @@ jobs:
     outputs:
       l2geth: ${{ steps.packages.outputs.l2geth }}
       message-relayer: ${{ steps.packages.outputs.message-relayer }}
+      fault-detector: ${{ steps.packages.outputs.fault-detector }}
       data-transport-layer: ${{ steps.packages.outputs.data-transport-layer }}
       contracts: ${{ steps.packages.outputs.contracts }}
       gas-oracle: ${{ steps.packages.outputs.gas-oracle }}
@@ -339,6 +340,33 @@ jobs:
           push: true
           tags: ethereumoptimism/message-relayer:${{ needs.release.outputs.message-relayer }},ethereumoptimism/message-relayer:latest
 
+  fault-detector:
+    name: Publish Fault Detector Version ${{ needs.release.outputs.fault-detector }}
+    needs: release
+    if: needs.release.outputs.fault-detector != ''
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./ops/docker/Dockerfile.packages
+          target: fault-detector
+          push: true
+          tags: ethereumoptimism/fault-detector:${{ needs.release.outputs.fault-detector }},ethereumoptimism/fault-detector:latest
+
   data-transport-layer:
     name: Publish Data Transport Layer Version ${{ needs.release.outputs.data-transport-layer }}
     needs: release

.vscode/settings.json

@@ -10,6 +10,7 @@
     {"directory": "packages/data-transport-layer", "changeProcessCWD": true },
     {"directory": "packages/batch-submitter", "changeProcessCWD": true },
     {"directory": "packages/message-relayer", "changeProcessCWD": true },
+    {"directory": "packages/fault-detector", "changeProcessCWD": true },
   ],
   "eslint.nodePath": "./node_modules/eslint/bin/",
   "eslint.format.enable": true,

README.md

@@ -35,6 +35,7 @@ root
 │   ├── <a href="./packages/core-utils">core-utils</a>: Low-level utilities that make building Optimism easier
 │   ├── <a href="./packages/common-ts">common-ts</a>: Common tools for building apps in TypeScript
 │   ├── <a href="./packages/data-transport-layer">data-transport-layer</a>: Service for indexing Optimism-related L1 data
+│   ├── <a href="./packages/fault-detector">fault-detector</a>: Service for detecting faulty L2 output proposals
 │   ├── <a href="./packages/message-relayer">message-relayer</a>: Tool for automatically relaying L1<>L2 messages in development
 │   └── <a href="./packages/replica-healthcheck">replica-healthcheck</a>: Service for monitoring the health of a replica node
 ├── <a href="./go">go</a>

ops/docker-compose.yml

@@ -117,6 +117,23 @@ services:
       MESSAGE_RELAYER__L1_WALLET: '0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97'
       RETRIES: 60
 
+  fault_detector:
+    depends_on:
+      - l1_chain
+      - l2geth
+    deploy:
+      replicas: 0
+    build:
+      context: ..
+      dockerfile: ./ops/docker/Dockerfile.packages
+      target: fault-detector
+    image: ethereumoptimism/fault-detector:${DOCKER_TAG_FAULT_DETECTOR:-latest}
+    entrypoint: ./detector.sh
+    environment:
+      FAULT_DETECTOR__L1_RPC_PROVIDER: http://l1_chain:8545
+      FAULT_DETECTOR__L2_RPC_PROVIDER: http://l2geth:8545
+      RETRIES: 60
+
   verifier:
     depends_on:
       - l1_chain

ops/docker/Dockerfile.packages

@@ -17,6 +17,7 @@ COPY packages/common-ts/package.json ./packages/common-ts/package.json
 COPY packages/contracts/package.json ./packages/contracts/package.json
 COPY packages/data-transport-layer/package.json ./packages/data-transport-layer/package.json
 COPY packages/message-relayer/package.json ./packages/message-relayer/package.json
+COPY packages/fault-detector/package.json ./packages/fault-detector/package.json
 COPY packages/replica-healthcheck/package.json ./packages/replica-healthcheck/package.json
 COPY integration-tests/package.json ./integration-tests/package.json
 
@@ -53,6 +54,12 @@ COPY ./ops/scripts/relayer.sh .
 CMD ["npm", "run", "start"]
 
 
+FROM base as fault-detector
+WORKDIR /opt/optimism/packages/fault-detector
+COPY ./ops/scripts/detector.sh .
+CMD ["npm", "run", "start"]
+
+
 FROM base as replica-healthcheck
 WORKDIR /opt/optimism/packages/replica-healthcheck
 ENTRYPOINT ["npm", "run", "start"]

ops/scripts/build-ci.sh

@@ -23,6 +23,7 @@ wait
 build deployer "ethereumoptimism/deployer:latest" "./ops/docker/Dockerfile.deployer" .
 build dtl "ethereumoptimism/data-transport-layer:latest" "./ops/docker/Dockerfile.data-transport-layer" .
 build relayer "ethereumoptimism/message-relayer:latest" "./ops/docker/Dockerfile.message-relayer" .
+build relayer "ethereumoptimism/fault-detector:latest" "./ops/docker/Dockerfile.fault-detector" .
 build integration-tests "ethereumoptimism/integration-tests:latest" "./ops/docker/Dockerfile.integration-tests" .
 
 wait

ops/scripts/detector.sh

+#!/bin/bash
+
+set -e
+
+RETRIES=${RETRIES:-60}
+
+# waits for l2geth to be up
+curl \
+    --fail \
+    --show-error \
+    --silent \
+    --output /dev/null \
+    --retry-connrefused \
+    --retry $RETRIES \
+    --retry-delay 1 \
+    $FAULT_DETECTOR__L2_RPC_PROVIDER
+
+# go
+exec yarn start

packages/fault-detector/.env.example

+# URL for an L1 RPC provider, used to query L2 output proposals
+FAULT_DETECTOR__L1_RPC_PROVIDER=
+
+# URL for an L2 RPC provider, used to query canonical L2 state
+FAULT_DETECTOR__L2_RPC_PROVIDER=

packages/fault-detector/.eslintrc.js

+module.exports = {
+  extends: '../../.eslintrc.js',
+}

packages/fault-detector/.gitignore

+node_modules/
+build/

packages/fault-detector/.lintstagedrc.yml

+"*.{ts,js}":
+  - eslint

packages/fault-detector/.prettierrc.js

+module.exports = {
+  ...require('../../.prettierrc.js'),
+};

packages/fault-detector/LICENSE

+(The MIT License)
+
+Copyright 2020-2021 Optimism
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

packages/fault-detector/README.md

+# @eth-optimism/fault-detector
+
+The `fault-detector` is a simple service for detecting discrepancies between your local view of the Optimism network and the L2 output proposals published to Ethereum.
+
+## Installation
+
+Clone, install, and build the Optimism monorepo:
+
+```
+git clone https://github.com/ethereum-optimism/optimism.git
+yarn install
+yarn build
+```
+
+## Running the service
+
+Copy `.env.example` into a new file named `.env`, then set the environment variables listed there.
+Once your environment variables have been set, run the service via:
+
+```
+yarn start
+```

packages/fault-detector/package.json

+{
+  "private": true,
+  "name": "@eth-optimism/fault-detector",
+  "version": "0.0.1",
+  "description": "[Optimism] Service for detecting faulty L2 output proposals",
+  "main": "dist/index",
+  "types": "dist/index",
+  "files": [
+    "dist/*"
+  ],
+  "scripts": {
+    "start": "ts-node ./src/service.ts",
+    "test:coverage": "echo 'No tests defined.'",
+    "build": "tsc -p tsconfig.build.json",
+    "clean": "rimraf dist/ ./tsconfig.build.tsbuildinfo",
+    "lint": "yarn lint:fix && yarn lint:check",
+    "pre-commit": "lint-staged",
+    "lint:fix": "yarn lint:check --fix",
+    "lint:check": "eslint . --max-warnings=0"
+  },
+  "keywords": [
+    "optimism",
+    "ethereum",
+    "fault",
+    "detector"
+  ],
+  "homepage": "https://github.com/ethereum-optimism/optimism/tree/develop/packages/fault-detector#readme",
+  "license": "MIT",
+  "author": "Optimism PBC",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ethereum-optimism/optimism.git"
+  },
+  "devDependencies": {
+    "@types/dateformat": "^5.0.0",
+    "@types/node": "^17.0.23",
+    "dateformat": "^4.5.1",
+    "ethers": "^5.6.2",
+    "ts-node": "^10.7.0",
+    "typescript": "^4.6.3"
+  },
+  "dependencies": {
+    "@eth-optimism/common-ts": "^0.2.4",
+    "@eth-optimism/core-utils": "^0.8.1",
+    "@eth-optimism/sdk": "^1.0.2",
+    "@ethersproject/abstract-provider": "^5.6.0"
+  }
+}

packages/fault-detector/src/index.ts

+export * from './service'

packages/fault-detector/src/service.ts

+import { BaseServiceV2, Gauge, validators } from '@eth-optimism/common-ts'
+import { sleep, toRpcHexString } from '@eth-optimism/core-utils'
+import { CrossChainMessenger } from '@eth-optimism/sdk'
+import { Provider } from '@ethersproject/abstract-provider'
+import { ethers } from 'ethers'
+import dateformat from 'dateformat'
+
+type Options = {
+  l1RpcProvider: Provider
+  l2RpcProvider: Provider
+  startBatchIndex: number
+}
+
+type Metrics = {
+  highestCheckedBatchIndex: Gauge
+  highestKnownBatchIndex: Gauge
+  isCurrentlyMismatched: Gauge
+  inUnexpectedErrorState: Gauge
+}
+
+type State = {
+  messenger: CrossChainMessenger
+  highestCheckedBatchIndex: number
+}
+
+export class FaultDetector extends BaseServiceV2<Options, Metrics, State> {
+  constructor(options?: Partial<Options>) {
+    super({
+      name: 'fault-detector',
+      loop: true,
+      loopIntervalMs: 1000,
+      options,
+      optionsSpec: {
+        l1RpcProvider: {
+          validator: validators.provider,
+          desc: 'Provider for interacting with L1',
+        },
+        l2RpcProvider: {
+          validator: validators.provider,
+          desc: 'Provider for interacting with L2',
+        },
+        startBatchIndex: {
+          validator: validators.num,
+          default: 0,
+          desc: 'Batch index to start checking from',
+        },
+      },
+      metricsSpec: {
+        highestCheckedBatchIndex: {
+          type: Gauge,
+          desc: 'Highest good batch index',
+        },
+        highestKnownBatchIndex: {
+          type: Gauge,
+          desc: 'Highest known batch index',
+        },
+        isCurrentlyMismatched: {
+          type: Gauge,
+          desc: '0 if state is ok, 1 if state is mismatched',
+        },
+        inUnexpectedErrorState: {
+          type: Gauge,
+          desc: '0 if service is ok, 1 service is in unexpected error state',
+        },
+      },
+    })
+  }
+
+  async init(): Promise<void> {
+    const network = await this.options.l1RpcProvider.getNetwork()
+    this.state.messenger = new CrossChainMessenger({
+      l1SignerOrProvider: this.options.l1RpcProvider,
+      l2SignerOrProvider: this.options.l2RpcProvider,
+      l1ChainId: network.chainId,
+    })
+
+    this.state.highestCheckedBatchIndex = this.options.startBatchIndex
+  }
+
+  async main(): Promise<void> {
+    const latestBatchIndex =
+      await this.state.messenger.contracts.l1.StateCommitmentChain.getTotalBatches()
+    if (this.state.highestCheckedBatchIndex >= latestBatchIndex.toNumber()) {
+      await sleep(15000)
+      return
+    }
+
+    this.metrics.highestKnownBatchIndex.set(latestBatchIndex.toNumber())
+
+    this.logger.info(`checking batch`, {
+      batchIndex: this.state.highestCheckedBatchIndex,
+    })
+
+    const targetEvents =
+      await this.state.messenger.contracts.l1.StateCommitmentChain.queryFilter(
+        this.state.messenger.contracts.l1.StateCommitmentChain.filters.StateBatchAppended(
+          this.state.highestCheckedBatchIndex
+        )
+      )
+
+    if (targetEvents.length === 0) {
+      this.logger.error(`unable to find event for batch`, {
+        batchIndex: this.state.highestCheckedBatchIndex,
+      })
+      this.metrics.inUnexpectedErrorState.set(1)
+      return
+    }
+
+    if (targetEvents.length > 1) {
+      this.logger.error(`found too many events for batch`, {
+        batchIndex: this.state.highestCheckedBatchIndex,
+      })
+      this.metrics.inUnexpectedErrorState.set(1)
+      return
+    }
+
+    const targetEvent = targetEvents[0]
+    const batchTransaction = await targetEvent.getTransaction()
+    const [stateRoots] =
+      this.state.messenger.contracts.l1.StateCommitmentChain.interface.decodeFunctionData(
+        'appendStateBatch',
+        batchTransaction.data
+      )
+
+    const batchStart = targetEvent.args._prevTotalElements.toNumber() + 1
+    const batchSize = targetEvent.args._batchSize.toNumber()
+
+    // `getBlockRange` has a limit of 1000 blocks, so we have to break this request out into
+    // multiple requests of maximum 1000 blocks in the case that batchSize > 1000.
+    let blocks: any[] = []
+    for (let i = 0; i < batchSize; i += 1000) {
+      const provider = this.options
+        .l2RpcProvider as ethers.providers.JsonRpcProvider
+      blocks = blocks.concat(
+        await provider.send('eth_getBlockRange', [
+          toRpcHexString(batchStart + i),
+          toRpcHexString(batchStart + i + Math.min(batchSize - i, 1000) - 1),
+          false,
+        ])
+      )
+    }
+
+    for (const [i, stateRoot] of stateRoots.entries()) {
+      if (blocks[i].stateRoot !== stateRoot) {
+        this.metrics.isCurrentlyMismatched.set(1)
+        const fpw =
+          await this.state.messenger.contracts.l1.StateCommitmentChain.FRAUD_PROOF_WINDOW()
+        this.logger.error(`state root mismatch`, {
+          blockNumber: blocks[i].number,
+          expectedStateRoot: blocks[i].stateRoot,
+          actualStateRoot: stateRoot,
+          finalizationTime: dateformat(
+            new Date(
+              (ethers.BigNumber.from(blocks[i].timestamp).toNumber() +
+                fpw.toNumber()) *
+                1000
+            ),
+            'mmmm dS, yyyy, h:MM:ss TT'
+          ),
+        })
+        return
+      }
+    }
+
+    this.metrics.highestCheckedBatchIndex.set(
+      this.state.highestCheckedBatchIndex
+    )
+    this.state.highestCheckedBatchIndex++
+
+    // If we got through the above without throwing an error, we should be fine to reset.
+    this.metrics.isCurrentlyMismatched.set(0)
+    this.metrics.inUnexpectedErrorState.set(0)
+  }
+}
+
+if (require.main === module) {
+  const service = new FaultDetector()
+  service.run()
+}

packages/fault-detector/tsconfig.build.json

+{
+  "extends": "../../tsconfig.build.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist"
+  },
+  "include": [
+    "src/**/*"
+  ]
+}

packages/fault-detector/tsconfig.json

+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "resolveJsonModule": true
+  }
+}