Skip to content

N
nebula
Commit 4a5833f7 authored by Annie Ke's avatar Annie Ke Committed by GitHub
Browse files
Options

Merge pull request #1365 from ethereum-optimism/annie/healthcheck-write-latency 

replica-healthcheck feat: add tx write latency check
parents e23b2643 4262ea2c
Show whitespace changes
Inline Side-by-side
Showing with 157 additions and 17 deletions
.changeset/pink-owls-hope.md 0 → 100644
View file @ 4a5833f7
---
'@eth-optimism/replica-healthcheck': patch
---
Add tx write latency cron check
packages/replica-healthcheck/.env.example
View file @ 4a5833f7
......@@ -2,3 +2,6 @@ REPLICA_HEALTHCHECK__ETH_NETWORK=mainnet
REPLICA_HEALTHCHECK__ETH_NETWORK_RPC_PROVIDER=https://mainnet.optimism.io
REPLICA_HEALTHCHECK__ETH_REPLICA_RPC_PROVIDER=http://localhost:9991
REPLICA_HEALTHCHECK__L2GETH_IMAGE_TAG=0.4.7
REPLICA_HEALTHCHECK__CHECK_TX_WRITE_LATENCY=false
REPLICA_HEALTHCHECK__WALLET1_PRIVATE_KEY=
REPLICA_HEALTHCHECK__WALLET2_PRIVATE_KEY=
packages/replica-healthcheck/README.md
View file @ 4a5833f7
......@@ -30,8 +30,11 @@ To configure the project, clone this repository and copy the `env.example` file
Here's a list of environment variables:
| Variable | Purpose | Default |
| ----------------------------------------------- | -------------------------------------------------------- | ---------------------------------------------------------------------------------------- |
| ----------------------------------------------- | ------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------- |
| REPLICA_HEALTHCHECK\_\_ETH_NETWORK | Ethereum Layer1 and Layer2 network (mainnet,kovan) | mainnet (change to `kovan` for the test network) |
| REPLICA_HEALTHCHECK\_\_ETH_NETWORK_RPC_PROVIDER | Layer2 source of truth endpoint, used for the sync check | https://mainnet.optimism.io (change to `https://kovan.optimism.io` for the test network) |
| REPLICA_HEALTHCHECK\_\_ETH_REPLICA_RPC_PROVIDER | Layer2 local replica endpoint, used for the sync check | http://localhost:9991 |
| REPLICA_HEALTHCHECK\_\_L2GETH_IMAGE_TAG | L2geth version | 0.4. |
| REPLICA_HEALTHCHECK\_\_L2GETH_IMAGE_TAG | L2geth version | 0.4.9 |
| REPLICA_HEALTHCHECK\_\_CHECK_TX_WRITE_LATENCY | Boolean for whether to perform the transaction latency check. Recommend to only use for testnets | false |
| REPLICA_HEALTHCHECK\_\_WALLET1_PRIVATE_KEY | Private key to one wallet for checking write latency | - |
| REPLICA_HEALTHCHECK\_\_WALLET2_PRIVATE_KEY | Private key to the other wallet for checking write latency | - |
packages/replica-healthcheck/package.json
View file @ 4a5833f7
......@@ -22,7 +22,7 @@
"devDependencies": {
"@types/express": "^4.17.12",
"@types/node": "^15.12.2",
"dotenv": "^10.0.0",
"@types/node-cron": "^2.0.4",
"supertest": "^6.1.4",
"ts-mocha": "^8.0.0",
"ts-node": "^10.0.0",
......@@ -32,8 +32,10 @@
"@eth-optimism/common-ts": "0.1.5",
"@eth-optimism/core-utils": "^0.5.1",
"ethers": "^5.4.5",
"dotenv": "^10.0.0",
"express": "^4.17.1",
"express-prom-bundle": "^6.3.6",
"node-cron": "^3.0.0",
"prom-client": "^13.1.0"
}
}
packages/replica-healthcheck/src/healthcheck-server.ts
View file @ 4a5833f7
import express from 'express'
import { Server } from 'net'
import promBundle from 'express-prom-bundle'
import { Gauge } from 'prom-client'
import { providers } from 'ethers'
import { Gauge, Histogram } from 'prom-client'
import cron from 'node-cron'
import { providers, Wallet } from 'ethers'
import { Metrics, Logger } from '@eth-optimism/common-ts'
import { injectL2Context, sleep } from '@eth-optimism/core-utils'
......@@ -13,13 +14,21 @@ export interface HealthcheckServerOptions {
gethRelease: string
sequencerRpcProvider: string
replicaRpcProvider: string
checkTxWriteLatency: boolean
txWriteOptions?: TxWriteOptions
logger: Logger
}
export interface TxWriteOptions {
wallet1PrivateKey: string
wallet2PrivateKey: string
}
export interface ReplicaMetrics {
lastMatchingStateRootHeight: Gauge<string>
replicaHeight: Gauge<string>
sequencerHeight: Gauge<string>
txWriteLatencyMs: Histogram<string>
}
export class HealthcheckServer {
......@@ -27,6 +36,7 @@ export class HealthcheckServer {
protected app: express.Express
protected logger: Logger
protected metrics: ReplicaMetrics
protected replicaProvider: providers.StaticJsonRpcProvider
server: Server
constructor(options: HealthcheckServerOptions) {
......@@ -38,6 +48,12 @@ export class HealthcheckServer {
init = () => {
this.metrics = this.initMetrics()
this.server = this.initServer()
this.replicaProvider = injectL2Context(
new providers.StaticJsonRpcProvider(this.options.replicaRpcProvider)
)
if (this.options.checkTxWriteLatency) {
this.initTxLatencyCheck()
}
}
initMetrics = (): ReplicaMetrics => {
......@@ -69,6 +85,11 @@ export class HealthcheckServer {
help: 'Block number of the latest block from the sequencer',
registers: [metrics.registry],
}),
txWriteLatencyMs: new metrics.client.Histogram({
name: 'tx_write_latency_in_ms',
help: 'The latency of sending a write transaction through a replica in ms',
registers: [metrics.registry],
}),
}
}
......@@ -91,17 +112,77 @@ export class HealthcheckServer {
return server
}
initTxLatencyCheck = () => {
// Check latency for every Monday
cron.schedule('0 0 * * 1', this.runTxLatencyCheck)
}
runTxLatencyCheck = async () => {
const wallet1 = new Wallet(
this.options.txWriteOptions.wallet1PrivateKey,
this.replicaProvider
)
const wallet2 = new Wallet(
this.options.txWriteOptions.wallet2PrivateKey,
this.replicaProvider
)
// Send funds between the 2 addresses
try {
const res1 = await this.getLatencyForSend(wallet1, wallet2)
this.logger.info('Sent transaction from wallet1 to wallet2', {
latencyMs: res1.latencyMs,
status: res1.status,
})
const res2 = await this.getLatencyForSend(wallet2, wallet2)
this.logger.info('Sent transaction from wallet2 to wallet1', {
latencyMs: res2.latencyMs,
status: res2.status,
})
} catch (err) {
this.logger.error('Failed to get tx write latency', {
message: err.toString(),
stack: err.stack,
code: err.code,
wallet1: wallet1.address,
wallet2: wallet2.address,
})
}
}
getLatencyForSend = async (
from: Wallet,
to: Wallet
): Promise<{
latencyMs: number
status: number
}> => {
const fromBal = await from.getBalance()
if (fromBal.isZero()) {
throw new Error('Wallet balance is zero, cannot make test transaction')
}
const startTime = new Date()
const tx = await from.sendTransaction({
to: to.address,
value: fromBal.div(2), // send half
})
const { status } = await tx.wait()
const endTime = new Date()
const latencyMs = endTime.getTime() - startTime.getTime()
this.metrics.txWriteLatencyMs.observe(latencyMs)
return { latencyMs, status }
}
runSyncCheck = async () => {
const sequencerProvider = injectL2Context(
new providers.JsonRpcProvider(this.options.sequencerRpcProvider)
)
const replicaProvider = injectL2Context(
new providers.JsonRpcBatchProvider(this.options.replicaRpcProvider)
new providers.StaticJsonRpcProvider(this.options.sequencerRpcProvider)
)
// Continuously loop while replica runs
while (true) {
let replicaLatest = (await replicaProvider.getBlock('latest')) as any
let replicaLatest = (await this.replicaProvider.getBlock('latest')) as any
const sequencerCorresponding = (await sequencerProvider.getBlock(
replicaLatest.number
)) as any
......@@ -112,7 +193,7 @@ export class HealthcheckServer {
)
const firstMismatch = await binarySearchForMismatch(
sequencerProvider,
replicaProvider,
this.replicaProvider,
replicaLatest.number,
this.logger
)
......@@ -129,7 +210,7 @@ export class HealthcheckServer {
})
this.metrics.lastMatchingStateRootHeight.set(replicaLatest.number)
replicaLatest = await replicaProvider.getBlock('latest')
replicaLatest = await this.replicaProvider.getBlock('latest')
const sequencerLatest = await sequencerProvider.getBlock('latest')
this.logger.info('Syncing from sequencer', {
sequencerHeight: sequencerLatest.number,
......@@ -145,7 +226,7 @@ export class HealthcheckServer {
'Replica caught up with sequencer, waiting for next block'
)
await sleep(1_000)
replicaLatest = await replicaProvider.getBlock('latest')
replicaLatest = await this.replicaProvider.getBlock('latest')
}
}
}
......
packages/replica-healthcheck/src/helpers.ts
View file @ 4a5833f7
......@@ -30,6 +30,19 @@ export const readConfig = (): HealthcheckServerOptions => {
process.exit(1)
}
const checkTxWriteLatency =
process.env['REPLICA_HEALTHCHECK__CHECK_TX_WRITE_LATENCY'] === 'true'
let txWriteOptions
if (checkTxWriteLatency) {
const wallet1PrivateKey = readEnvOrQuitProcess(
'REPLICA_HEALTHCHECK__WALLET1_PRIVATE_KEY'
)
const wallet2PrivateKey = readEnvOrQuitProcess(
'REPLICA_HEALTHCHECK__WALLET2_PRIVATE_KEY'
)
txWriteOptions = { wallet1PrivateKey, wallet2PrivateKey }
}
const logger = new Logger({ name: 'replica-healthcheck' })
return {
......@@ -37,6 +50,8 @@ export const readConfig = (): HealthcheckServerOptions => {
gethRelease,
sequencerRpcProvider,
replicaRpcProvider,
checkTxWriteLatency,
txWriteOptions,
logger,
}
}
......
yarn.lock
View file @ 4a5833f7
......@@ -2635,6 +2635,13 @@
resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-8.2.3.tgz#bbeb55fbc73f28ea6de601fbfa4613f58d785323"
integrity sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw==
"@types/node-cron@^2.0.4":
version "2.0.4"
resolved "https://registry.yarnpkg.com/@types/node-cron/-/node-cron-2.0.4.tgz#6d467440762e7d3539890d477b33670c020c458f"
integrity sha512-vXzgDRWCZpuut5wJVZtluEnkNhzGojYlyMch2c4kMj7H74L8xTLytVlgQzj+/17wfcjs49aJDFBDglFSGt7GeA==
dependencies:
"@types/tz-offset" "*"
"@types/node-fetch@^2.5.10", "@types/node-fetch@^2.5.5", "@types/node-fetch@^2.5.8":
version "2.5.12"
resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.12.tgz#8a6f779b1d4e60b7a57fb6fd48d84fb545b9cc66"
......@@ -2816,6 +2823,11 @@
resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-6.0.3.tgz#79df6f358ae8f79e628fe35a63608a0ea8e7cf08"
integrity sha512-E1dU4fzC9wN2QK2Cr1MLCfyHM8BoNnRFvuf45LYMPNDA+WqbNzC45S4UzPxvp1fFJ1rvSGU0bPvdd35VLmXG8g==
"@types/tz-offset@*":
version "0.0.0"
resolved "https://registry.yarnpkg.com/@types/tz-offset/-/tz-offset-0.0.0.tgz#d58f1cebd794148d245420f8f0660305d320e565"
integrity sha512-XLD/llTSB6EBe3thkN+/I0L+yCTB6sjrcVovQdx2Cnl6N6bTzHmwe/J8mWnsXFgxLrj/emzdv8IR4evKYG2qxQ==
"@types/underscore@*":
version "1.11.3"
resolved "https://registry.yarnpkg.com/@types/underscore/-/underscore-1.11.3.tgz#d6734f3741ce41b2630018c6b61c6745f6188c07"
......@@ -10451,6 +10463,18 @@ modify-values@^1.0.0:
resolved "https://registry.yarnpkg.com/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022"
integrity sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw==
moment-timezone@^0.5.31:
version "0.5.33"
resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.33.tgz#b252fd6bb57f341c9b59a5ab61a8e51a73bbd22c"
integrity sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==
dependencies:
moment ">= 2.9.0"
"moment@>= 2.9.0":
version "2.29.1"
resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3"
integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==
mri@1.1.4:
version "1.1.4"
resolved "https://registry.yarnpkg.com/mri/-/mri-1.1.4.tgz#7cb1dd1b9b40905f1fac053abe25b6720f44744a"
......@@ -10651,6 +10675,13 @@ node-addon-api@^3.0.2:
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161"
integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==
node-cron@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/node-cron/-/node-cron-3.0.0.tgz#b33252803e430f9cd8590cf85738efa1497a9522"
integrity sha512-DDwIvvuCwrNiaU7HEivFDULcaQualDv7KoNlB/UU1wPW0n1tDEmBJKhEIE6DlF2FuoOHcNbLJ8ITL2Iv/3AWmA==
dependencies:
moment-timezone "^0.5.31"
node-emoji@^1.10.0, node-emoji@^1.4.1:
version "1.11.0"
resolved "https://registry.yarnpkg.com/node-emoji/-/node-emoji-1.11.0.tgz#69a0150e6946e2f115e9d7ea4df7971e2628301c"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment