Add highest L1 and L2 block number Gauge metrics to DTL (#1125)

* build: add prom-client to data-transport-layer

* refactor: thread metrics more carefully through data-transport-layer; add two new metrics

* style: fix some style issues

* refactor: make metrics mandatory

* refactor: move metrics register code to top of file

* style: apply linting

* refactor: move promethesus initialization after express

* refactor: move promBundle call up, provide registry

* build: add changeset

.changeset/rude-eels-count.md

+---
+'@eth-optimism/data-transport-layer': patch
+---
+
+Add highest L1 and L2 block number Gauge metrics to DTL

packages/data-transport-layer/package.json

@@ -68,6 +68,7 @@
     "mocha": "^8.3.2",
     "pino-pretty": "^4.7.1",
     "prettier": "^2.2.1",
+    "prom-client": "^13.1.0",
     "rimraf": "^3.0.2",
     "ts-node": "^9.1.1",
     "typescript": "^4.2.3"

packages/data-transport-layer/src/services/l1-ingestion/service.ts

 /* Imports: External */
 import { fromHexString, EventArgsAddressSet } from '@eth-optimism/core-utils'
-import { BaseService } from '@eth-optimism/common-ts'
+import { BaseService, Metrics } from '@eth-optimism/common-ts'
 import { JsonRpcProvider } from '@ethersproject/providers'
 import { LevelUp } from 'levelup'
 import { ethers, constants } from 'ethers'
+import { Gauge } from 'prom-client'
 
 /* Imports: Internal */
 import { TransportDB } from '../../db/transport-db'
@@ -21,9 +22,25 @@ import { handleEventsStateBatchAppended } from './handlers/state-batch-appended'
 import { L1DataTransportServiceOptions } from '../main/service'
 import { MissingElementError, EventName } from './handlers/errors'
 
+interface L1IngestionMetrics {
+  highestSyncedL1Block: Gauge<string>
+}
+
+const registerMetrics = ({
+  client,
+  registry,
+}: Metrics): L1IngestionMetrics => ({
+  highestSyncedL1Block: new client.Gauge({
+    name: 'data_transport_layer_highest_synced_l1_block',
+    help: 'Highest Synced L1 Block Number',
+    registers: [registry],
+  }),
+})
+
 export interface L1IngestionServiceOptions
   extends L1DataTransportServiceOptions {
   db: LevelUp
+  metrics: Metrics
 }
 
 const optionSettings = {
@@ -61,6 +78,8 @@ export class L1IngestionService extends BaseService<L1IngestionServiceOptions> {
     super('L1_Ingestion_Service', options, optionSettings)
   }
 
+  private l1IngestionMetrics: L1IngestionMetrics
+
   private state: {
     db: TransportDB
     contracts: OptimismContracts
@@ -72,6 +91,8 @@ export class L1IngestionService extends BaseService<L1IngestionServiceOptions> {
   protected async _init(): Promise<void> {
     this.state.db = new TransportDB(this.options.db)
 
+    this.l1IngestionMetrics = registerMetrics(this.metrics)
+
     this.state.l1RpcProvider =
       typeof this.options.l1RpcProvider === 'string'
         ? new JsonRpcProvider(this.options.l1RpcProvider)
@@ -199,6 +220,8 @@ export class L1IngestionService extends BaseService<L1IngestionServiceOptions> {
 
         await this.state.db.setHighestSyncedL1Block(targetL1Block)
 
+        this.l1IngestionMetrics.highestSyncedL1Block.set(targetL1Block)
+
         if (
           currentL1Block - highestSyncedL1Block <
           this.options.logsPerPollingInterval
@@ -240,6 +263,10 @@ export class L1IngestionService extends BaseService<L1IngestionServiceOptions> {
             lastGoodElement.blockNumber
           )
 
+          this.l1IngestionMetrics.highestSyncedL1Block.set(
+            lastGoodElement.blockNumber
+          )
+
           // Something we should be keeping track of.
           this.logger.warn('recovering from a missing event', {
             eventName,

packages/data-transport-layer/src/services/l2-ingestion/service.ts

 /* Imports: External */
-import { BaseService } from '@eth-optimism/common-ts'
+import { BaseService, Metrics } from '@eth-optimism/common-ts'
 import { JsonRpcProvider } from '@ethersproject/providers'
 import { BigNumber } from 'ethers'
 import { LevelUp } from 'levelup'
 import axios from 'axios'
 import bfj from 'bfj'
+import { Gauge } from 'prom-client'
 
 /* Imports: Internal */
 import { TransportDB } from '../../db/transport-db'
@@ -12,6 +13,21 @@ import { sleep, toRpcHexString, validators } from '../../utils'
 import { L1DataTransportServiceOptions } from '../main/service'
 import { handleSequencerBlock } from './handlers/transaction'
 
+interface L2IngestionMetrics {
+  highestSyncedL2Block: Gauge<string>
+}
+
+const registerMetrics = ({
+  client,
+  registry,
+}: Metrics): L2IngestionMetrics => ({
+  highestSyncedL2Block: new client.Gauge({
+    name: 'data_transport_layer_highest_synced_l2_block',
+    help: 'Highest Synced L2 Block Number',
+    registers: [registry],
+  }),
+})
+
 export interface L2IngestionServiceOptions
   extends L1DataTransportServiceOptions {
   db: LevelUp
@@ -52,6 +68,8 @@ export class L2IngestionService extends BaseService<L2IngestionServiceOptions> {
     super('L2_Ingestion_Service', options, optionSettings)
   }
 
+  private l2IngestionMetrics: L2IngestionMetrics
+
   private state: {
     db: TransportDB
     l2RpcProvider: JsonRpcProvider
@@ -64,6 +82,8 @@ export class L2IngestionService extends BaseService<L2IngestionServiceOptions> {
       )
     }
 
+    this.l2IngestionMetrics = registerMetrics(this.metrics)
+
     this.state.db = new TransportDB(this.options.db)
 
     this.state.l2RpcProvider =
@@ -113,6 +133,8 @@ export class L2IngestionService extends BaseService<L2IngestionServiceOptions> {
 
         await this.state.db.setHighestSyncedUnconfirmedBlock(targetL2Block)
 
+        this.l2IngestionMetrics.highestSyncedL2Block.set(targetL2Block)
+
         if (
           currentL2Block - highestSyncedL2BlockNumber <
           this.options.transactionsPerPollingInterval

packages/data-transport-layer/src/services/main/service.ts

 /* Imports: External */
-import { BaseService, Logger } from '@eth-optimism/common-ts'
+import { BaseService, Logger, Metrics } from '@eth-optimism/common-ts'
 import { LevelUp } from 'levelup'
 import level from 'level'
 
@@ -31,7 +31,6 @@ export interface L1DataTransportServiceOptions {
   useSentry?: boolean
   sentryDsn?: string
   sentryTraceRate?: number
-  enableMetrics?: boolean
   defaultBackend: string
 }
 
@@ -65,8 +64,18 @@ export class L1DataTransportService extends BaseService<L1DataTransportServiceOp
     this.state.db = level(this.options.dbPath)
     await this.state.db.open()
 
+    const metrics = new Metrics({
+      labels: {
+        environment: this.options.nodeEnv,
+        network: this.options.ethNetworkName,
+        release: this.options.release,
+        service: this.name,
+      }
+    })
+
     this.state.l1TransportServer = new L1TransportServer({
       ...this.options,
+      metrics,
       db: this.state.db,
     })
 
@@ -74,6 +83,7 @@ export class L1DataTransportService extends BaseService<L1DataTransportServiceOp
     if (this.options.syncFromL1) {
       this.state.l1IngestionService = new L1IngestionService({
         ...this.options,
+        metrics,
         db: this.state.db,
       })
     }
@@ -82,6 +92,7 @@ export class L1DataTransportService extends BaseService<L1DataTransportServiceOp
     if (this.options.syncFromL2) {
       this.state.l2IngestionService = new L2IngestionService({
         ...(this.options as any), // TODO: Correct thing to do here is to assert this type.
+        metrics,
         db: this.state.db,
       })
     }

packages/data-transport-layer/src/services/run.ts

@@ -49,7 +49,6 @@ type ethNetwork = 'mainnet' | 'kovan' | 'goerli'
       useSentry: config.bool('use-sentry', false),
       sentryDsn: config.str('sentry-dsn'),
       sentryTraceRate: config.ufloat('sentry-trace-rate', 0.05),
-      enableMetrics: config.bool('enable-metrics', false),
     })
 
     await service.start()

packages/data-transport-layer/src/services/server/service.ts

@@ -2,6 +2,7 @@
 import { BaseService, Logger, Metrics } from '@eth-optimism/common-ts'
 import express, { Request, Response } from 'express'
 import promBundle from 'express-prom-bundle'
+import { Gauge } from 'prom-client'
 import cors from 'cors'
 import { BigNumber } from 'ethers'
 import { JsonRpcProvider } from '@ethersproject/providers'
@@ -27,6 +28,7 @@ import { L1DataTransportServiceOptions } from '../main/service'
 export interface L1TransportServerOptions
   extends L1DataTransportServiceOptions {
   db: LevelUp
+  metrics: Metrics
 }
 
 const optionSettings = {
@@ -106,14 +108,26 @@ export class L1TransportServer extends BaseService<L1TransportServerOptions> {
   private _initializeApp() {
     // TODO: Maybe pass this in as a parameter instead of creating it here?
     this.state.app = express()
+
     if (this.options.useSentry) {
       this._initSentry()
     }
-    if (this.options.enableMetrics) {
-      this._initMetrics()
-    }
+
     this.state.app.use(cors())
+
+    // Add prometheus middleware to express BEFORE route registering
+    this.state.app.use(
+      // This also serves metrics on port 3000 at /metrics
+      promBundle({
+        // Provide metrics registry that other metrics uses
+        promRegistry: this.metrics.registry,
+        includeMethod: true,
+        includePath: true,
+      })
+    )
+
     this._registerAllRoutes()
+
     // Sentry error handling must be after all controllers
     // and before other error middleware
     if (this.options.useSentry) {
@@ -148,25 +162,6 @@ export class L1TransportServer extends BaseService<L1TransportServerOptions> {
     this.state.app.use(Sentry.Handlers.tracingHandler())
   }
 
-  /**
-   * Initialize Prometheus metrics collection and endpoint
-   */
-  private _initMetrics() {
-    this.metrics = new Metrics({
-      labels: {
-        environment: this.options.nodeEnv,
-        network: this.options.ethNetworkName,
-        release: this.options.release,
-        service: this.name,
-      },
-    })
-    const metricsMiddleware = promBundle({
-      includeMethod: true,
-      includePath: true,
-    })
-    this.state.app.use(metricsMiddleware)
-  }
-
   /**
    * Registers a route on the server.
    *