feat[batch-submitter]: add lightweight metrics server (#792)

* feat[batch-submitter]: add lightweight metrics server

* add changeset

* move metrics server to common

.changeset/old-ladybugs-camp.md

+---
+'@eth-optimism/batch-submitter': patch
+'@eth-optimism/common-ts': patch
+---
+
+add metrics server to common-ts and batch submitter

packages/batch-submitter/.env.example

@@ -4,6 +4,9 @@ NODE_ENV=development
 ETH_NETWORK_NAME=
 # Logging & monitoring
 DEBUG=info*,error*,warn*,debug*
+RUN_PROMETHEUS_SERVER=
+PROMETHEUS_PORT=
+PROMETHEUS_HOSTNAME=
 # Leave the SENTRY_DSN variable unset during local development
 SENTRY_DSN=
 SENTRY_TRACE_RATE=

packages/batch-submitter/src/exec/run-batch-submitter.ts

 /* External Imports */
 import { injectL2Context, Bcfg } from '@eth-optimism/core-utils'
-import { Logger, Metrics } from '@eth-optimism/common-ts'
+import { Logger, Metrics, createMetricsServer } from '@eth-optimism/common-ts'
 import { exit } from 'process'
 import { Signer, Wallet } from 'ethers'
 import { JsonRpcProvider, TransactionReceipt } from '@ethersproject/providers'
@@ -67,6 +67,7 @@ interface RequiredEnvVars {
  * USE_HARDHAT
  * DEBUG_IMPERSONATE_SEQUENCER_ADDRESS
  * DEBUG_IMPERSONATE_PROPOSER_ADDRESS
+ * RUN_PROMETHEUS_SERVER
  */
 
 export const run = async () => {
@@ -362,10 +363,7 @@ export const run = async () => {
     GAS_THRESHOLD_IN_GWEI,
     BLOCK_OFFSET,
     logger.child({ name: TX_BATCH_SUBMITTER_LOG_TAG }),
-    new Metrics({
-      prefix: TX_BATCH_SUBMITTER_LOG_TAG,
-      labels: { environment, release, network },
-    }),
+    metrics,
     DISABLE_QUEUE_BATCH_APPEND,
     autoFixBatchOptions
   )
@@ -388,10 +386,7 @@ export const run = async () => {
     GAS_THRESHOLD_IN_GWEI,
     BLOCK_OFFSET,
     logger.child({ name: STATE_BATCH_SUBMITTER_LOG_TAG }),
-    new Metrics({
-      prefix: STATE_BATCH_SUBMITTER_LOG_TAG,
-      labels: { environment, release, network },
-    }),
+    metrics,
     FRAUD_SUBMISSION_ADDRESS
   )
 
@@ -462,4 +457,16 @@ export const run = async () => {
   if (requiredEnvVars.RUN_STATE_BATCH_SUBMITTER) {
     loop(() => stateBatchSubmitter.submitNextBatch())
   }
+
+  if (
+    config.bool('run-prometheus-server', env.RUN_PROMETHEUS_SERVER === 'true')
+  ) {
+    // Initialize metrics server
+    await createMetricsServer({
+      logger,
+      registry: metrics.registry,
+      port: config.uint('prometheus-port', 7300),
+      hostname: config.str('prometheus-hostname', '127.0.0.1'),
+    })
+  }
 }

packages/common-ts/package.json

@@ -14,13 +14,21 @@
     "clean": "rimraf dist/ ./tsconfig.build.tsbuildinfo",
     "lint": "yarn lint:fix && yarn lint:check",
     "lint:fix": "prettier --config .prettierrc.json --write '{src,test}/**/*.ts'",
-    "lint:check": "tslint --format stylish --project ."
+    "lint:check": "tslint --format stylish --project .",
+    "test": "ts-mocha test/*.spec.ts"
   },
   "devDependencies": {
+    "@types/chai": "^4.2.18",
+    "@types/express": "^4.17.11",
+    "@types/mocha": "^8.2.2",
     "@types/pino": "^6.3.6",
     "@types/pino-multi-stream": "^5.1.1",
     "@types/prettier": "^2.2.3",
+    "chai": "^4.3.4",
+    "mocha": "^8.4.0",
     "prettier": "^2.2.1",
+    "supertest": "^6.1.3",
+    "ts-mocha": "^8.0.0",
     "tslint": "^6.1.3",
     "tslint-config-prettier": "^1.18.0",
     "tslint-no-focused-test": "^0.5.0",
@@ -29,6 +37,7 @@
   },
   "dependencies": {
     "@sentry/node": "^6.2.5",
+    "express": "^4.17.1",
     "pino": "^6.11.3",
     "pino-multi-stream": "^5.3.0",
     "pino-sentry": "^0.7.0",

packages/common-ts/src/common/metrics.ts

@@ -3,6 +3,10 @@ import prometheus, {
   DefaultMetricsCollectorConfiguration,
   Registry,
 } from 'prom-client'
+import express from 'express'
+import { Server } from 'net'
+
+import { Logger } from './logger'
 
 export interface MetricsOptions {
   prefix: string
@@ -29,3 +33,36 @@ export class Metrics {
     collectDefaultMetrics(metricsOptions)
   }
 }
+
+export interface MetricsServerOptions {
+  logger: Logger
+  registry: Registry
+  port?: number
+  route?: string
+  hostname?: string
+}
+
+export const createMetricsServer = async (
+  options: MetricsServerOptions
+): Promise<Server> => {
+  const logger = options.logger.child({ component: 'MetricsServer' })
+
+  const app = express()
+
+  const route = options.route || '/metrics'
+  app.get(route, async (_, res) => {
+    res.status(200).send(await options.registry.metrics())
+  })
+
+  const port = options.port || 7300
+  const hostname = options.hostname || '127.0.0.1'
+  const server = app.listen(port, hostname, () => {
+    logger.info('Metrics server started', {
+      port,
+      hostname,
+      route,
+    })
+  })
+
+  return server
+}

packages/common-ts/test/metrics.spec.ts

+import request from 'supertest'
+// Setup
+import chai = require('chai')
+const expect = chai.expect
+
+import { Logger, Metrics, createMetricsServer } from '../src'
+
+describe('Metrics', () => {
+  it('shoud serve metrics', async () => {
+    const metrics = new Metrics({
+      prefix: 'test_metrics',
+    })
+    const registry = metrics.registry
+    const logger = new Logger({ name: 'test_logger' })
+
+    const server = await createMetricsServer({
+      logger,
+      registry,
+      port: 42069,
+    })
+
+    try {
+      // Create two metrics for testing
+      const counter = new metrics.client.Counter({
+        name: 'counter',
+        help: 'counter help',
+        registers: [registry],
+      })
+      const gauge = new metrics.client.Gauge({
+        name: 'gauge',
+        help: 'gauge help',
+        registers: [registry],
+      })
+
+      counter.inc()
+      counter.inc()
+      gauge.set(100)
+
+      // Verify that the registered metrics are served at `/`
+      const response = await request(server).get('/metrics').send()
+      expect(response.status).eq(200)
+      expect(response.text).match(/counter 2/)
+      expect(response.text).match(/gauge 100/)
+    } finally {
+      server.close()
+      registry.clear()
+    }
+  })
+})

yarn.lock

@@ -3903,6 +3903,11 @@
   resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.2.17.tgz#85f9f0610f514b22a94125d441f73eef65bde5cc"
   integrity sha512-LaiwWNnYuL8xJlQcE91QB2JoswWZckq9A4b+nMPq8dt8AP96727Nb3X4e74u+E3tm4NLTILNI9MYFsyVc30wSA==
 
+"@types/chai@^4.2.18":
+  version "4.2.18"
+  resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.2.18.tgz#0c8e298dbff8205e2266606c1ea5fbdba29b46e4"
+  integrity sha512-rS27+EkB/RE1Iz3u0XtVL5q36MGDWbgYe7zWiodyKNUnthxY0rukK5V36eiUCtCisB7NN8zKYH6DO2M37qxFEQ==
+
 "@types/concat-stream@^1.6.0":
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/@types/concat-stream/-/concat-stream-1.6.0.tgz#394dbe0bb5fee46b38d896735e8b68ef2390d00d"
@@ -7003,7 +7008,7 @@ component-emitter@1.2.1:
   resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.2.1.tgz#137918d6d78283f7df7a6b7c5a63e140e69425e6"
   integrity sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=
 
-component-emitter@^1.2.1:
+component-emitter@^1.2.1, component-emitter@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0"
   integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==
@@ -7208,7 +7213,7 @@ cookie@^0.4.1:
   resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.1.tgz#afd713fe26ebd21ba95ceb61f9a8116e50a537d1"
   integrity sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA==
 
-cookiejar@^2.1.1:
+cookiejar@^2.1.1, cookiejar@^2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.2.tgz#dd8a235530752f988f9a0844f3fc589e3111125c"
   integrity sha512-Mw+adcfzPxcPeI+0WlvRrr/3lGVO0bD75SxX6811cxSh1Wbxx7xZBGK1eVtDf6si8rg2lhnUjsVLMFMfbRIuwA==
@@ -9694,6 +9699,11 @@ form-data@~2.3.2:
     combined-stream "^1.0.6"
     mime-types "^2.1.12"
 
+formidable@^1.2.2:
+  version "1.2.2"
+  resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.2.tgz#bf69aea2972982675f00865342b982986f6b8dd9"
+  integrity sha512-V8gLm+41I/8kguQ4/o1D3RIHRmhYFG4pnNyonvua+40rqcEmT4+V71yaZ3B457xbbgCsCfjSPi65u/W6vK1U5Q==
+
 forwarded@~0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84"
@@ -13316,7 +13326,7 @@ meros@1.1.4:
   resolved "https://registry.yarnpkg.com/meros/-/meros-1.1.4.tgz#c17994d3133db8b23807f62bec7f0cb276cfd948"
   integrity sha512-E9ZXfK9iQfG9s73ars9qvvvbSIkJZF5yOo9j4tcwM5tN8mUKfj/EKN5PzOr3ZH0y5wL7dLAHw3RVEfpQV9Q7VQ==
 
-methods@~1.1.2:
+methods@^1.1.2, methods@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
   integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=
@@ -13392,6 +13402,11 @@ mime@1.6.0:
   resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
   integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
 
+mime@^2.4.6:
+  version "2.5.2"
+  resolved "https://registry.yarnpkg.com/mime/-/mime-2.5.2.tgz#6e3dc6cc2b9510643830e5f19d5cb753da5eeabe"
+  integrity sha512-tqkh47FzKeCPD2PUiPB6pkbMzsCasjxAfC62/Wap5qrUWcb+sFasXUC5I3gYM5iBM8v/Qpn4UK0x+j0iHyFPDg==
+
 mimic-fn@^1.0.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022"
@@ -13730,6 +13745,37 @@ mocha@^8.2.1, mocha@^8.3.0, mocha@^8.3.1, mocha@^8.3.2:
     yargs-parser "20.2.4"
     yargs-unparser "2.0.0"
 
+mocha@^8.4.0:
+  version "8.4.0"
+  resolved "https://registry.yarnpkg.com/mocha/-/mocha-8.4.0.tgz#677be88bf15980a3cae03a73e10a0fc3997f0cff"
+  integrity sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==
+  dependencies:
+    "@ungap/promise-all-settled" "1.1.2"
+    ansi-colors "4.1.1"
+    browser-stdout "1.3.1"
+    chokidar "3.5.1"
+    debug "4.3.1"
+    diff "5.0.0"
+    escape-string-regexp "4.0.0"
+    find-up "5.0.0"
+    glob "7.1.6"
+    growl "1.10.5"
+    he "1.2.0"
+    js-yaml "4.0.0"
+    log-symbols "4.0.0"
+    minimatch "3.0.4"
+    ms "2.1.3"
+    nanoid "3.1.20"
+    serialize-javascript "5.0.1"
+    strip-json-comments "3.1.1"
+    supports-color "8.1.1"
+    which "2.0.2"
+    wide-align "1.1.3"
+    workerpool "6.1.0"
+    yargs "16.2.0"
+    yargs-parser "20.2.4"
+    yargs-unparser "2.0.0"
+
 mock-fs@^4.1.0:
   version "4.13.0"
   resolved "https://registry.yarnpkg.com/mock-fs/-/mock-fs-4.13.0.tgz#31c02263673ec3789f90eb7b6963676aa407a598"
@@ -17910,6 +17956,31 @@ subscriptions-transport-ws@^0.9.11, subscriptions-transport-ws@^0.9.16, subscrip
     symbol-observable "^1.0.4"
     ws "^5.2.0"
 
+superagent@^6.1.0:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/superagent/-/superagent-6.1.0.tgz#09f08807bc41108ef164cfb4be293cebd480f4a6"
+  integrity sha512-OUDHEssirmplo3F+1HWKUrUjvnQuA+nZI6i/JJBdXb5eq9IyEQwPyPpqND+SSsxf6TygpBEkUjISVRN4/VOpeg==
+  dependencies:
+    component-emitter "^1.3.0"
+    cookiejar "^2.1.2"
+    debug "^4.1.1"
+    fast-safe-stringify "^2.0.7"
+    form-data "^3.0.0"
+    formidable "^1.2.2"
+    methods "^1.1.2"
+    mime "^2.4.6"
+    qs "^6.9.4"
+    readable-stream "^3.6.0"
+    semver "^7.3.2"
+
+supertest@^6.1.3:
+  version "6.1.3"
+  resolved "https://registry.yarnpkg.com/supertest/-/supertest-6.1.3.tgz#3f49ea964514c206c334073e8dc4e70519c7403f"
+  integrity sha512-v2NVRyP73XDewKb65adz+yug1XMtmvij63qIWHZzSX8tp6wiq6xBLUy4SUAd2NII6wIipOmHT/FD9eicpJwdgQ==
+  dependencies:
+    methods "^1.1.2"
+    superagent "^6.1.0"
+
 supports-color@6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.0.0.tgz#76cfe742cf1f41bb9b1c29ad03068c05b4c0e40a"