Monitoring: social preview bots request counter (#1781)

* add counter metric and logger for bot request

* add script for grafana local instance

* add log endpoint

* add ENV to enable monitoring

* [skip ci] change metric labels and add telegram bot

* [skip ci] remove 404 from /metrics route

.gitignore

@@ -25,6 +25,7 @@
 .DS_Store
 *.pem
 .tools
+grafana
 
 # debug
 npm-debug.log*

deploy/values/review/values.yaml.gotmpl

@@ -90,6 +90,7 @@ frontend:
     NEXT_PUBLIC_AD_ADBUTLER_CONFIG_DESKTOP: "{ \"id\": \"632019\", \"width\": \"728\", \"height\": \"90\" }"
     NEXT_PUBLIC_AD_ADBUTLER_CONFIG_MOBILE: "{ \"id\": \"632018\", \"width\": \"320\", \"height\": \"100\" }"
     NEXT_PUBLIC_DATA_AVAILABILITY_ENABLED: true
+    PROMETHEUS_METRICS_ENABLED: true
   envFromSecret:
     NEXT_PUBLIC_SENTRY_DSN: ref+vault://deployment-values/blockscout/dev/review?token_env=VAULT_TOKEN&address=https://vault.k8s.blockscout.com#/NEXT_PUBLIC_SENTRY_DSN
     SENTRY_CSP_REPORT_URI: ref+vault://deployment-values/blockscout/dev/review?token_env=VAULT_TOKEN&address=https://vault.k8s.blockscout.com#/SENTRY_CSP_REPORT_URI

lib/metadata/getApiDataForSocialPreview.ts

+import type { IncomingMessage, ServerResponse } from 'http';
+
+import { httpLogger } from 'nextjs/utils/logger';
+
+import metrics from 'lib/monitoring/metrics';
+
+export default async function getApiDataForSocialPreview(req: IncomingMessage | undefined, res: ServerResponse<IncomingMessage> | undefined, pathname: string) {
+  if (!req || !res || !metrics) {
+    return;
+  }
+
+  const userAgent = req.headers['user-agent'];
+
+  if (!userAgent) {
+    return;
+  }
+
+  if (userAgent.toLowerCase().includes('twitter')) {
+    httpLogger(req, res);
+    metrics.requestCounter.inc({ route: pathname, bot: 'twitter' });
+  }
+
+  if (userAgent.toLowerCase().includes('facebook')) {
+    httpLogger(req, res);
+    metrics.requestCounter.inc({ route: pathname, bot: 'facebook' });
+  }
+
+  if (userAgent.toLowerCase().includes('telegram')) {
+    httpLogger(req, res);
+    metrics.requestCounter.inc({ route: pathname, bot: 'telegram' });
+  }
+}

lib/metadata/getPageOgType.ts

@@ -48,6 +48,8 @@ const OG_TYPE_DICT: Record<Route['pathname'], OGPageType> = {
 
   // service routes, added only to make typescript happy
   '/login': 'Regular page',
+  '/api/metrics': 'Regular page',
+  '/api/log': 'Regular page',
   '/api/media-type': 'Regular page',
   '/api/proxy': 'Regular page',
   '/api/csrf': 'Regular page',

lib/metadata/templates/description.ts

@@ -51,6 +51,8 @@ const TEMPLATE_MAP: Record<Route['pathname'], string> = {
 
   // service routes, added only to make typescript happy
   '/login': DEFAULT_TEMPLATE,
+  '/api/metrics': DEFAULT_TEMPLATE,
+  '/api/log': DEFAULT_TEMPLATE,
   '/api/media-type': DEFAULT_TEMPLATE,
   '/api/proxy': DEFAULT_TEMPLATE,
   '/api/csrf': DEFAULT_TEMPLATE,

lib/metadata/templates/title.ts

@@ -46,6 +46,8 @@ const TEMPLATE_MAP: Record<Route['pathname'], string> = {
 
   // service routes, added only to make typescript happy
   '/login': 'login',
+  '/api/metrics': 'node API prometheus metrics',
+  '/api/log': 'node API request log',
   '/api/media-type': 'node API media type',
   '/api/proxy': 'node API proxy',
   '/api/csrf': 'node API CSRF token',

lib/mixpanel/getPageType.ts

@@ -46,6 +46,8 @@ export const PAGE_TYPE_DICT: Record<Route['pathname'], string> = {
 
   // service routes, added only to make typescript happy
   '/login': 'Login',
+  '/api/metrics': 'Node API: Prometheus metrics',
+  '/api/log': 'Node API: Request log',
   '/api/media-type': 'Node API: Media type',
   '/api/proxy': 'Node API: Proxy',
   '/api/csrf': 'Node API: CSRF token',

lib/monitoring/metrics.ts

+import * as promClient from 'prom-client';
+
+const metrics = (() => {
+  // eslint-disable-next-line no-restricted-properties
+  if (process.env.PROMETHEUS_METRICS_ENABLED !== 'true') {
+    return;
+  }
+
+  promClient.register.clear();
+
+  const requestCounter = new promClient.Counter({
+    name: 'request_counter',
+    help: 'Number of incoming requests',
+    labelNames: [ 'route', 'bot' ] as const,
+  });
+
+  return { requestCounter };
+})();
+
+export default metrics;

nextjs/nextjs-routes.d.ts

@@ -18,7 +18,9 @@ declare module "nextjs-routes" {
     | DynamicRoute<"/address/[hash]", { "hash": string }>
     | StaticRoute<"/api/csrf">
     | StaticRoute<"/api/healthz">
+    | StaticRoute<"/api/log">
     | StaticRoute<"/api/media-type">
+    | StaticRoute<"/api/metrics">
     | StaticRoute<"/api/proxy">
     | StaticRoute<"/api-docs">
     | DynamicRoute<"/apps/[id]", { "id": string }>

package.json

@@ -30,7 +30,9 @@
     "test:pw:detect-affected": "node ./deploy/tools/affected-tests/index.js",
     "test:jest": "jest",
     "test:jest:watch": "jest --watch",
-    "favicon:generate:dev": "./tools/scripts/favicon-generator.dev.sh"
+    "favicon:generate:dev": "./tools/scripts/favicon-generator.dev.sh",
+    "monitoring:prometheus:local": "docker run --name blockscout_prometheus -d -p 127.0.0.1:9090:9090 -v $(pwd)/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus",
+    "monitoring:grafana:local": "docker run -d -p 4000:3000 --name=blockscout_grafana --user $(id -u) --volume $(pwd)/grafana:/var/lib/grafana grafana/grafana-enterprise"
   },
   "dependencies": {
     "@chakra-ui/react": "2.7.1",
@@ -87,6 +89,7 @@
     "phoenix": "^1.6.15",
     "pino-http": "^8.2.1",
     "pino-pretty": "^9.1.1",
+    "prom-client": "15.1.1",
     "qrcode": "^1.5.1",
     "react": "18.2.0",
     "react-device-detect": "^2.2.3",

pages/_document.tsx

@@ -5,6 +5,7 @@ import React from 'react';
 
 import * as serverTiming from 'nextjs/utils/serverTiming';
 
+import getApiDataForSocialPreview from 'lib/metadata/getApiDataForSocialPreview';
 import theme from 'theme';
 import * as svgSprite from 'ui/shared/IconSvg';
 
@@ -21,6 +22,8 @@ class MyDocument extends Document {
       return result;
     };
 
+    await getApiDataForSocialPreview(ctx.req, ctx.res, ctx.pathname);
+
     const initialProps = await Document.getInitialProps(ctx);
 
     return initialProps;

pages/api/log.ts

+import type { NextApiRequest, NextApiResponse } from 'next';
+
+import { httpLogger } from 'nextjs/utils/logger';
+
+export default async function logHandler(req: NextApiRequest, res: NextApiResponse) {
+  httpLogger(req, res);
+
+  res.status(200).send('ok');
+}

pages/api/metrics.ts

+import type { NextApiRequest, NextApiResponse } from 'next';
+import * as promClient from 'prom-client';
+
+// eslint-disable-next-line no-restricted-properties
+const isEnabled = process.env.PROMETHEUS_METRICS_ENABLED === 'true';
+
+isEnabled && promClient.collectDefaultMetrics();
+
+export default async function metricsHandler(req: NextApiRequest, res: NextApiResponse) {
+  const metrics = await promClient.register.metrics();
+  res.setHeader('Content-type', promClient.register.contentType);
+  res.send(metrics);
+}

prometheus.yml

+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+rule_files:
+  # - "first.rules"
+  # - "second.rules"
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:9090']
+  - job_name: frontend
+    metrics_path: /node-api/metrics
+    static_configs:
+      - targets: ['host.docker.internal:3000']
\ No newline at end of file

yarn.lock

@@ -3528,6 +3528,11 @@
   resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.7.0.tgz#b139c81999c23e3c8d3c0a7234480e945920fc40"
   integrity sha512-AdY5wvN0P2vXBi3b29hxZgSFvdhdxPB9+f0B6s//P9Q8nibRWeA3cHm8UmLpio9ABigkVHJ5NMPk+Mz8VCCyrw==
 
+"@opentelemetry/api@^1.4.0":
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.8.0.tgz#5aa7abb48f23f693068ed2999ae627d2f7d902ec"
+  integrity sha512-I/s6F7yKUDdtMsoBWXJe8Qz40Tui5vsuKCWJEWVL+5q9sSWRzzx6v2KeNsOBEwd94j0eWkpWCH4yB6rZg9Mf0w==
+
 "@opentelemetry/auto-instrumentations-node@^0.39.4":
   version "0.39.4"
   resolved "https://registry.yarnpkg.com/@opentelemetry/auto-instrumentations-node/-/auto-instrumentations-node-0.39.4.tgz#c772201ba224e6ebffaaf17c9f2234e6e8343328"
@@ -7726,6 +7731,11 @@ binary-extensions@^2.0.0:
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
   integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==
 
+bintrees@1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8"
+  integrity sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==
+
 bl@^4.0.3:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a"
@@ -13300,6 +13310,14 @@ progress@^2.0.3:
   resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
   integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
 
+prom-client@15.1.1:
+  version "15.1.1"
+  resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.1.tgz#71ba84371241acd173181b04a436782c246f3652"
+  integrity sha512-GVA2H96QCg2q71rjc3VYvSrVG7OpnJxyryC7dMzvfJfpJJHzQVwF3TJLfHzKORcwJpElWs1TwXLthlJAFJxq2A==
+  dependencies:
+    "@opentelemetry/api" "^1.4.0"
+    tdigest "^0.1.1"
+
 promise-polyfill@^8.1.3:
   version "8.3.0"
   resolved "https://registry.yarnpkg.com/promise-polyfill/-/promise-polyfill-8.3.0.tgz#9284810268138d103807b11f4e23d5e945a4db63"
@@ -14901,6 +14919,13 @@ tar-stream@^2.1.4:
     inherits "^2.0.3"
     readable-stream "^3.1.1"
 
+tdigest@^0.1.1:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.2.tgz#96c64bac4ff10746b910b0e23b515794e12faced"
+  integrity sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==
+  dependencies:
+    bintrees "1.0.2"
+
 test-exclude@^6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-6.0.0.tgz#04a8698661d805ea6fa293b6cb9e63ac044ef15e"