Merge pull request #1365 from ethereum-optimism/annie/healthcheck-write-latency

replica-healthcheck feat: add tx write latency check

.changeset/pink-owls-hope.md

+---
+'@eth-optimism/replica-healthcheck': patch
+---
+
+Add tx write latency cron check

packages/replica-healthcheck/.env.example

@@ -2,3 +2,6 @@ REPLICA_HEALTHCHECK__ETH_NETWORK=mainnet
 REPLICA_HEALTHCHECK__ETH_NETWORK_RPC_PROVIDER=https://mainnet.optimism.io
 REPLICA_HEALTHCHECK__ETH_REPLICA_RPC_PROVIDER=http://localhost:9991
 REPLICA_HEALTHCHECK__L2GETH_IMAGE_TAG=0.4.7
+REPLICA_HEALTHCHECK__CHECK_TX_WRITE_LATENCY=false
+REPLICA_HEALTHCHECK__WALLET1_PRIVATE_KEY=
+REPLICA_HEALTHCHECK__WALLET2_PRIVATE_KEY=

packages/replica-healthcheck/README.md

@@ -29,9 +29,12 @@ We're using `dotenv` for our configuration.
 To configure the project, clone this repository and copy the `env.example` file to `.env`.
 Here's a list of environment variables:
 
-| Variable                                        | Purpose                                                  | Default                                                                                  |
-| ----------------------------------------------- | -------------------------------------------------------- | ---------------------------------------------------------------------------------------- |
-| REPLICA_HEALTHCHECK\_\_ETH_NETWORK              | Ethereum Layer1 and Layer2 network (mainnet,kovan)       | mainnet (change to `kovan` for the test network)                                         |
-| REPLICA_HEALTHCHECK\_\_ETH_NETWORK_RPC_PROVIDER | Layer2 source of truth endpoint, used for the sync check | https://mainnet.optimism.io (change to `https://kovan.optimism.io` for the test network) |
-| REPLICA_HEALTHCHECK\_\_ETH_REPLICA_RPC_PROVIDER | Layer2 local replica endpoint, used for the sync check   | http://localhost:9991                                                                    |
-| REPLICA_HEALTHCHECK\_\_L2GETH_IMAGE_TAG         | L2geth version                                           | 0.4.                                                                                     |
+| Variable                                        | Purpose                                                                                          | Default                                                                                  |
+| ----------------------------------------------- | ------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------- |
+| REPLICA_HEALTHCHECK\_\_ETH_NETWORK              | Ethereum Layer1 and Layer2 network (mainnet,kovan)                                               | mainnet (change to `kovan` for the test network)                                         |
+| REPLICA_HEALTHCHECK\_\_ETH_NETWORK_RPC_PROVIDER | Layer2 source of truth endpoint, used for the sync check                                         | https://mainnet.optimism.io (change to `https://kovan.optimism.io` for the test network) |
+| REPLICA_HEALTHCHECK\_\_ETH_REPLICA_RPC_PROVIDER | Layer2 local replica endpoint, used for the sync check                                           | http://localhost:9991                                                                    |
+| REPLICA_HEALTHCHECK\_\_L2GETH_IMAGE_TAG         | L2geth version                                                                                   | 0.4.9                                                                                    |
+| REPLICA_HEALTHCHECK\_\_CHECK_TX_WRITE_LATENCY   | Boolean for whether to perform the transaction latency check. Recommend to only use for testnets | false                                                                                    |
+| REPLICA_HEALTHCHECK\_\_WALLET1_PRIVATE_KEY      | Private key to one wallet for checking write latency                                             | -                                                                                        |
+| REPLICA_HEALTHCHECK\_\_WALLET2_PRIVATE_KEY      | Private key to the other wallet for checking write latency                                       | -                                                                                        |

packages/replica-healthcheck/package.json

@@ -22,7 +22,7 @@
   "devDependencies": {
     "@types/express": "^4.17.12",
     "@types/node": "^15.12.2",
-    "dotenv": "^10.0.0",
+    "@types/node-cron": "^2.0.4",
     "supertest": "^6.1.4",
     "ts-mocha": "^8.0.0",
     "ts-node": "^10.0.0",
@@ -32,8 +32,10 @@
     "@eth-optimism/common-ts": "0.1.5",
     "@eth-optimism/core-utils": "^0.5.1",
     "ethers": "^5.4.5",
+    "dotenv": "^10.0.0",
     "express": "^4.17.1",
     "express-prom-bundle": "^6.3.6",
+    "node-cron": "^3.0.0",
     "prom-client": "^13.1.0"
   }
 }

packages/replica-healthcheck/src/healthcheck-server.ts

 import express from 'express'
 import { Server } from 'net'
 import promBundle from 'express-prom-bundle'
-import { Gauge } from 'prom-client'
-import { providers } from 'ethers'
+import { Gauge, Histogram } from 'prom-client'
+import cron from 'node-cron'
+import { providers, Wallet } from 'ethers'
 import { Metrics, Logger } from '@eth-optimism/common-ts'
 import { injectL2Context, sleep } from '@eth-optimism/core-utils'
 
@@ -13,13 +14,21 @@ export interface HealthcheckServerOptions {
   gethRelease: string
   sequencerRpcProvider: string
   replicaRpcProvider: string
+  checkTxWriteLatency: boolean
+  txWriteOptions?: TxWriteOptions
   logger: Logger
 }
 
+export interface TxWriteOptions {
+  wallet1PrivateKey: string
+  wallet2PrivateKey: string
+}
+
 export interface ReplicaMetrics {
   lastMatchingStateRootHeight: Gauge<string>
   replicaHeight: Gauge<string>
   sequencerHeight: Gauge<string>
+  txWriteLatencyMs: Histogram<string>
 }
 
 export class HealthcheckServer {
@@ -27,6 +36,7 @@ export class HealthcheckServer {
   protected app: express.Express
   protected logger: Logger
   protected metrics: ReplicaMetrics
+  protected replicaProvider: providers.StaticJsonRpcProvider
   server: Server
 
   constructor(options: HealthcheckServerOptions) {
@@ -38,6 +48,12 @@ export class HealthcheckServer {
   init = () => {
     this.metrics = this.initMetrics()
     this.server = this.initServer()
+    this.replicaProvider = injectL2Context(
+      new providers.StaticJsonRpcProvider(this.options.replicaRpcProvider)
+    )
+    if (this.options.checkTxWriteLatency) {
+      this.initTxLatencyCheck()
+    }
   }
 
   initMetrics = (): ReplicaMetrics => {
@@ -69,6 +85,11 @@ export class HealthcheckServer {
         help: 'Block number of the latest block from the sequencer',
         registers: [metrics.registry],
       }),
+      txWriteLatencyMs: new metrics.client.Histogram({
+        name: 'tx_write_latency_in_ms',
+        help: 'The latency of sending a write transaction through a replica in ms',
+        registers: [metrics.registry],
+      }),
     }
   }
 
@@ -91,17 +112,77 @@ export class HealthcheckServer {
     return server
   }
 
+  initTxLatencyCheck = () => {
+    // Check latency for every Monday
+    cron.schedule('0 0 * * 1', this.runTxLatencyCheck)
+  }
+
+  runTxLatencyCheck = async () => {
+    const wallet1 = new Wallet(
+      this.options.txWriteOptions.wallet1PrivateKey,
+      this.replicaProvider
+    )
+    const wallet2 = new Wallet(
+      this.options.txWriteOptions.wallet2PrivateKey,
+      this.replicaProvider
+    )
+
+    // Send funds between the 2 addresses
+    try {
+      const res1 = await this.getLatencyForSend(wallet1, wallet2)
+      this.logger.info('Sent transaction from wallet1 to wallet2', {
+        latencyMs: res1.latencyMs,
+        status: res1.status,
+      })
+
+      const res2 = await this.getLatencyForSend(wallet2, wallet2)
+      this.logger.info('Sent transaction from wallet2 to wallet1', {
+        latencyMs: res2.latencyMs,
+        status: res2.status,
+      })
+    } catch (err) {
+      this.logger.error('Failed to get tx write latency', {
+        message: err.toString(),
+        stack: err.stack,
+        code: err.code,
+        wallet1: wallet1.address,
+        wallet2: wallet2.address,
+      })
+    }
+  }
+
+  getLatencyForSend = async (
+    from: Wallet,
+    to: Wallet
+  ): Promise<{
+    latencyMs: number
+    status: number
+  }> => {
+    const fromBal = await from.getBalance()
+    if (fromBal.isZero()) {
+      throw new Error('Wallet balance is zero, cannot make test transaction')
+    }
+
+    const startTime = new Date()
+    const tx = await from.sendTransaction({
+      to: to.address,
+      value: fromBal.div(2), // send half
+    })
+    const { status } = await tx.wait()
+    const endTime = new Date()
+    const latencyMs = endTime.getTime() - startTime.getTime()
+    this.metrics.txWriteLatencyMs.observe(latencyMs)
+    return { latencyMs, status }
+  }
+
   runSyncCheck = async () => {
     const sequencerProvider = injectL2Context(
-      new providers.JsonRpcProvider(this.options.sequencerRpcProvider)
-    )
-    const replicaProvider = injectL2Context(
-      new providers.JsonRpcBatchProvider(this.options.replicaRpcProvider)
+      new providers.StaticJsonRpcProvider(this.options.sequencerRpcProvider)
     )
 
     // Continuously loop while replica runs
     while (true) {
-      let replicaLatest = (await replicaProvider.getBlock('latest')) as any
+      let replicaLatest = (await this.replicaProvider.getBlock('latest')) as any
       const sequencerCorresponding = (await sequencerProvider.getBlock(
         replicaLatest.number
       )) as any
@@ -112,7 +193,7 @@ export class HealthcheckServer {
         )
         const firstMismatch = await binarySearchForMismatch(
           sequencerProvider,
-          replicaProvider,
+          this.replicaProvider,
           replicaLatest.number,
           this.logger
         )
@@ -129,7 +210,7 @@ export class HealthcheckServer {
       })
       this.metrics.lastMatchingStateRootHeight.set(replicaLatest.number)
 
-      replicaLatest = await replicaProvider.getBlock('latest')
+      replicaLatest = await this.replicaProvider.getBlock('latest')
       const sequencerLatest = await sequencerProvider.getBlock('latest')
       this.logger.info('Syncing from sequencer', {
         sequencerHeight: sequencerLatest.number,
@@ -145,7 +226,7 @@ export class HealthcheckServer {
           'Replica caught up with sequencer, waiting for next block'
         )
         await sleep(1_000)
-        replicaLatest = await replicaProvider.getBlock('latest')
+        replicaLatest = await this.replicaProvider.getBlock('latest')
       }
     }
   }

packages/replica-healthcheck/src/helpers.ts

@@ -30,6 +30,19 @@ export const readConfig = (): HealthcheckServerOptions => {
     process.exit(1)
   }
 
+  const checkTxWriteLatency =
+    process.env['REPLICA_HEALTHCHECK__CHECK_TX_WRITE_LATENCY'] === 'true'
+  let txWriteOptions
+  if (checkTxWriteLatency) {
+    const wallet1PrivateKey = readEnvOrQuitProcess(
+      'REPLICA_HEALTHCHECK__WALLET1_PRIVATE_KEY'
+    )
+    const wallet2PrivateKey = readEnvOrQuitProcess(
+      'REPLICA_HEALTHCHECK__WALLET2_PRIVATE_KEY'
+    )
+    txWriteOptions = { wallet1PrivateKey, wallet2PrivateKey }
+  }
+
   const logger = new Logger({ name: 'replica-healthcheck' })
 
   return {
@@ -37,6 +50,8 @@ export const readConfig = (): HealthcheckServerOptions => {
     gethRelease,
     sequencerRpcProvider,
     replicaRpcProvider,
+    checkTxWriteLatency,
+    txWriteOptions,
     logger,
   }
 }

yarn.lock

@@ -2635,6 +2635,13 @@
   resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-8.2.3.tgz#bbeb55fbc73f28ea6de601fbfa4613f58d785323"
   integrity sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw==
 
+"@types/node-cron@^2.0.4":
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/@types/node-cron/-/node-cron-2.0.4.tgz#6d467440762e7d3539890d477b33670c020c458f"
+  integrity sha512-vXzgDRWCZpuut5wJVZtluEnkNhzGojYlyMch2c4kMj7H74L8xTLytVlgQzj+/17wfcjs49aJDFBDglFSGt7GeA==
+  dependencies:
+    "@types/tz-offset" "*"
+
 "@types/node-fetch@^2.5.10", "@types/node-fetch@^2.5.5", "@types/node-fetch@^2.5.8":
   version "2.5.12"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.12.tgz#8a6f779b1d4e60b7a57fb6fd48d84fb545b9cc66"
@@ -2816,6 +2823,11 @@
   resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.3.tgz#79df6f358ae8f79e628fe35a63608a0ea8e7cf08"
   integrity sha512-E1dU4fzC9wN2QK2Cr1MLCfyHM8BoNnRFvuf45LYMPNDA+WqbNzC45S4UzPxvp1fFJ1rvSGU0bPvdd35VLmXG8g==
 
+"@types/tz-offset@*":
+  version "0.0.0"
+  resolved "https://registry.yarnpkg.com/@types/tz-offset/-/tz-offset-0.0.0.tgz#d58f1cebd794148d245420f8f0660305d320e565"
+  integrity sha512-XLD/llTSB6EBe3thkN+/I0L+yCTB6sjrcVovQdx2Cnl6N6bTzHmwe/J8mWnsXFgxLrj/emzdv8IR4evKYG2qxQ==
+
 "@types/underscore@*":
   version "1.11.3"
   resolved "https://registry.yarnpkg.com/@types/underscore/-/underscore-1.11.3.tgz#d6734f3741ce41b2630018c6b61c6745f6188c07"
@@ -10451,6 +10463,18 @@ modify-values@^1.0.0:
   resolved "https://registry.yarnpkg.com/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022"
   integrity sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw==
 
+moment-timezone@^0.5.31:
+  version "0.5.33"
+  resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.33.tgz#b252fd6bb57f341c9b59a5ab61a8e51a73bbd22c"
+  integrity sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==
+  dependencies:
+    moment ">= 2.9.0"
+
+"moment@>= 2.9.0":
+  version "2.29.1"
+  resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3"
+  integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==
+
 mri@1.1.4:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/mri/-/mri-1.1.4.tgz#7cb1dd1b9b40905f1fac053abe25b6720f44744a"
@@ -10651,6 +10675,13 @@ node-addon-api@^3.0.2:
   resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161"
   integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==
 
+node-cron@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/node-cron/-/node-cron-3.0.0.tgz#b33252803e430f9cd8590cf85738efa1497a9522"
+  integrity sha512-DDwIvvuCwrNiaU7HEivFDULcaQualDv7KoNlB/UU1wPW0n1tDEmBJKhEIE6DlF2FuoOHcNbLJ8ITL2Iv/3AWmA==
+  dependencies:
+    moment-timezone "^0.5.31"
+
 node-emoji@^1.10.0, node-emoji@^1.4.1:
   version "1.11.0"
   resolved "https://registry.yarnpkg.com/node-emoji/-/node-emoji-1.11.0.tgz#69a0150e6946e2f115e9d7ea4df7971e2628301c"