interop-devnet: add grafana/loki/promtrail (#12773)

* interop-devnet: add grafana/loki/promtrail

Revival of PR https://github.com/ethereum-optimism/optimism/pull/11628
to enhance local interop devnet.
Co-authored-by: Ethen Pociask <ethenpo@gmail.com>
Co-authored-by: Samuel Laferriere <samlaf92@gmail.com>

* interop-devnet: use custom grafana port

* interop-devnet: fix ports

* interop-devnet: enable op-supervisor metrics

---------
Co-authored-by: Ethen Pociask <ethenpo@gmail.com>
Co-authored-by: Samuel Laferriere <samlaf92@gmail.com>

interop-devnet/docker-compose.yml

@@ -94,6 +94,8 @@ services:
       --rpc.port=8545
       --rpc.enable-admin
       --l2-rpcs="ws://l2-a:8546,ws://l2-b:8546"
+    environment:
+      OP_SUPERVISOR_METRICS_ENABLED: "true"
 
   l2-a:
     depends_on:
@@ -352,3 +354,44 @@ services:
       # OP_BATCHER_DATA_AVAILABILITY_TYPE: blobs
     env_file:
       - "${PWD}/../.devnet-interop/env/l2/900201/op-batcher.env"
+
+  grafana:
+    image: grafana/grafana:11.1.0
+    restart: unless-stopped
+    env_file:
+      - monitoring/grafana.env
+    volumes:
+      - ./monitoring/grafana/provisioning/:/etc/grafana/provisioning/:ro
+      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards
+      # - grafana_data:/var/lib/grafana
+    ports:
+      - 3300:3000
+
+  prometheus:
+    image: prom/prometheus:latest
+    restart: unless-stopped
+    volumes:
+      - ./monitoring/prometheus:/etc/prometheus
+      # - prometheus_data:/prometheus
+    ports:
+      - 3090:9090
+    command: --config.file=/etc/prometheus/prometheus.yml --log.level=debug
+
+  loki:
+    image: grafana/loki:3.1.1
+    restart: unless-stopped
+    volumes:
+      - ./monitoring/loki:/etc/loki
+    ports:
+      - 3200:3200
+    command: -config.file=/etc/loki/config.yaml
+
+  promtail:
+    image: grafana/promtail:3.1.1
+    restart: unless-stopped
+    volumes:
+      # uncomment to scrape system logs
+      # - /var/log:/var/log
+      - ./monitoring/promtail:/etc/promtail
+      - /var/run/docker.sock:/var/run/docker.sock # Mount Docker socket to read container logs
+    command: -config.file=/etc/promtail/config.yaml

interop-devnet/justfile

@@ -20,14 +20,21 @@ devnet-down:
   # stops services, does not remove containers/networks
   docker compose stop
 
+devnet-metrics-up:
+  docker compose up -d prometheus grafana loki promtail
+
+devnet-metrics-down:
+  docker compose down -d prometheus grafana loki promtail
+
 devnet-clean:
-  rm -rf ../.devnet-interop
   # Stops services, and removes containers/networks
   docker compose down
   # Now manually clean up the related images and volumes
   # Note: `justfile` interprets the curly brackets. So we escape them, by wrapping it with more, as a string, like Jinja2.
   docker image ls 'interop-devnet*' --format='{{ '{{.Repository}}' }}' | xargs -r docker rmi
   docker volume ls --filter name=interop-devnet --format='{{ '{{.Name}}' }}' | xargs -r docker volume rm
+  # docker compose down needs the env files before being able to shut down, so remove the devnet config resources last
+  rm -rf ../.devnet-interop
 
 devnet-logs:
   docker compose logs -f

interop-devnet/monitoring/grafana.env

+GF_SECURITY_ADMIN_PASSWORD=admin

interop-devnet/monitoring/grafana/provisioning/dashboards/all.yml

+apiVersion: 1
+
+providers:
+  - name: 'default'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: true
+    editable: true
+    options:
+      path: /var/lib/grafana/dashboards

interop-devnet/monitoring/grafana/provisioning/datasources/all.yml

+apiVersion: 1
+
+deleteDatasources:
+  - name: "Prometheus"
+
+datasources:
+  - access: "proxy"
+    editable: true
+    is_default: true
+    name: "Prometheus"
+    uid: "ddshms3dlineoe"
+    org_id: 1
+    type: "prometheus"
+    url: "http://prometheus:9090"
+    version: 1
+
+  - access: "proxy"
+    editable: true
+    name: "Loki"
+    uid: "loki-datasource"
+    org_id: 1
+    type: "loki"
+    url: "http://loki:3200"
+    version: 1

interop-devnet/monitoring/loki/config.yaml

+auth_enabled: false
+
+server:
+  http_listen_port: 3200
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+analytics:
+  reporting_enabled: false

interop-devnet/monitoring/prometheus/prometheus.yml

+global:
+  scrape_interval: 4s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: "l1-geth"
+    static_configs:
+      - targets: ["l1:6060"]
+    metrics_path: /debug/metrics/prometheus
+
+  - job_name: "l2-a-geth"
+    static_configs:
+      - targets: ["l2-a:6060"]
+    metrics_path: /debug/metrics/prometheus
+  - job_name: "l2-b-geth"
+    static_configs:
+      - targets: ["l2-b:6060"]
+    metrics_path: /debug/metrics/prometheus
+
+  - job_name: "op-node-a"
+    static_configs:
+      - targets: ["op-node-a:7300"]
+  - job_name: "op-node-b"
+    static_configs:
+      - targets: ["op-node-b:7300"]
+
+  - job_name: "op-batcher-a"
+    static_configs:
+      - targets: ["op-batcher-a:7300"]
+  - job_name: "op-batcher-b"
+    static_configs:
+      - targets: ["op-batcher-b:7300"]
+
+  - job_name: "op-proposer-a"
+    static_configs:
+      - targets: ["op-proposer-a:7300"]
+  - job_name: "op-proposer-b"
+    static_configs:
+      - targets: ["op-proposer-b:7300"]
+
+  - job_name: "op-supervisor"
+    static_configs:
+      - targets: ["op-supervisor:7300"]

interop-devnet/monitoring/promtail/config.yaml

+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3200/loki/api/v1/push
+
+scrape_configs:
+  # Uncomment to scrape system logs
+  # - job_name: system
+  #   static_configs:
+  #     - targets:
+  #         - localhost
+  #       labels:
+  #         job: varlogs
+  #         __path__: /var/log/*log
+  # This scrapes docker container logs
+  # copied from https://stackoverflow.com/questions/74776432/with-promtail-how-do-i-only-keep-log-messages-for-specified-docker-containers
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        filters:
+          - name: name
+            # Filter logging to just our containers
+            values: ["op-batcher-*", "op-proposer-*", "op-node-*", "op-supervisor-*"]
+    relabel_configs:
+      - source_labels: ["__meta_docker_container_name"]
+        regex: "/(.*)"
+        target_label: "container"