Merge pull request #1210 from ethereum-optimism/bwilson/op_exporter

Add op_exporter for sequencer metrics and health endpoint

.github/workflows/release.yml

@@ -92,6 +92,14 @@ jobs:
           push: true
           tags: ethereumoptimism/l2geth:${{ needs.release.outputs.l2geth }}
 
+      - name: Publish op_exporter
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./ops/docker/Dockerfile.op_exporter
+          push: true
+          tags: ethereumoptimism/op_exporter:${{ needs.release.outputs.l2geth }}
+
       - name: Publish rpc-proxy
         uses: docker/build-push-action@v2
         with:

go/op_exporter/.gitignore

+op_exporter
+.env
\ No newline at end of file

go/op_exporter/Makefile

+SHELL := /bin/bash
+
+VERSION := `git describe --abbrev=0`
+GITCOMMIT := `git rev-parse HEAD`
+BUILDDATE := `date +%Y-%m-%d`
+BUILDUSER := `whoami`
+
+LDFLAGSSTRING :=-X github.com/ethereum-optimism/optimism/go/op_exporter/version.Version=$(VERSION)
+LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/go/op_exporter/version.GitCommit=$(GITCOMMIT)
+LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/go/op_exporter/version.BuildDate=$(BUILDDATE)
+LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/go/op_exporter/version.BuildUser=$(BUILDUSER)
+
+LDFLAGS :=-ldflags "$(LDFLAGSSTRING)"
+
+.PHONY: all build
+
+all: build
+
+# Build binary
+build:
+	CGO_ENABLED=0 go build $(LDFLAGS)
\ No newline at end of file

go/op_exporter/README.md

+# op_exporter
+
+A prometheus exporter to collect information from an Optimistic Ethereum node and serve metrics for collection
+
+## Usage
+
+```
+make build && ./op_exporter --rpc.provider="https://kovan-sequencer.optimism.io" --label.network="kovan"
+```
+
+## Health endpoint `/health`
+
+Returns json describing the health of the sequencer based on the time since a block height update.
+
+```
+$ curl http://localhost:9100/health
+{ "healthy": "false" }
+```
+
+## Metrics endpoint `/metrics`
+
+```
+# HELP op_gasPrice Gas price.
+# TYPE op_gasPrice gauge
+op_gasPrice{layer="layer1",network="kovan"} 6.9e+09
+op_gasPrice{layer="layer2",network="kovan"} 1
+```

go/op_exporter/collector.go

+package main
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+//Define the metrics we wish to expose
+var (
+	gasPrice = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "op_gasPrice",
+			Help: "Gas price."},
+		[]string{"network", "layer"},
+	)
+	blockNumber = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "op_blocknumber",
+			Help: "Current block number."},
+		[]string{"network", "layer"},
+	)
+	healthySequencer = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "op_healthy_sequencer",
+			Help: "Is the sequencer healthy?"},
+		[]string{"network"},
+	)
+)
+
+func init() {
+	//Register metrics with prometheus
+	prometheus.MustRegister(gasPrice)
+	prometheus.MustRegister(blockNumber)
+	prometheus.MustRegister(healthySequencer)
+}

go/op_exporter/go.mod

+module github.com/ethereum-optimism/optimism/go/op_exporter
+
+go 1.16
+
+require (
+	github.com/ethereum/go-ethereum v1.10.4
+	github.com/prometheus/client_golang v1.4.0
+	github.com/sirupsen/logrus v1.4.2
+	github.com/ybbus/jsonrpc v2.1.2+incompatible
+	gopkg.in/alecthomas/kingpin.v2 v2.2.6
+)

go/op_exporter/main.go

+package main
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/go/op_exporter/version"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	log "github.com/sirupsen/logrus"
+	"github.com/ybbus/jsonrpc"
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+var (
+	listenAddress = kingpin.Flag(
+		"web.listen-address",
+		"Address on which to expose metrics and web interface.",
+	).Default(":9100").String()
+	rpcProvider = kingpin.Flag(
+		"rpc.provider",
+		"Address for RPC provider.",
+	).Default("http://127.0.0.1:8545").String()
+	networkLabel = kingpin.Flag(
+		"label.network",
+		"Label to apply to the metrics to identify the network.",
+	).Default("mainnet").String()
+	versionFlag = kingpin.Flag(
+		"version",
+		"Display binary version.",
+	).Default("False").Bool()
+	unhealthyTimePeriod = kingpin.Flag(
+		"wait.minutes",
+		"Number of minutes to wait for the next block before marking provider unhealthy.",
+	).Default("10").Int()
+	//unhealthyTimePeriod = time.Minute * 10
+)
+
+type healthCheck struct {
+	mu         *sync.RWMutex
+	height     uint64
+	healthy    bool
+	updateTime time.Time
+}
+
+func healthHandler(health *healthCheck) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		health.mu.RLock()
+		defer health.mu.RUnlock()
+		w.Write([]byte(fmt.Sprintf(`{ "healthy": "%t" }`, health.healthy)))
+	}
+}
+
+func main() {
+	kingpin.HelpFlag.Short('h')
+	kingpin.Parse()
+	if *versionFlag {
+		fmt.Printf("(version=%s, gitcommit=%s)\n", version.Version, version.GitCommit)
+		fmt.Printf("(go=%s, user=%s, date=%s)\n", version.GoVersion, version.BuildUser, version.BuildDate)
+		os.Exit(0)
+	}
+	log.Infoln("exporter config", *listenAddress, *rpcProvider, *networkLabel)
+	log.Infoln("Starting op_exporter", version.Info())
+	log.Infoln("Build context", version.BuildContext())
+
+	health := healthCheck{
+		mu:         new(sync.RWMutex),
+		height:     0,
+		healthy:    false,
+		updateTime: time.Now(),
+	}
+	http.Handle("/metrics", promhttp.Handler())
+	http.Handle("/health", healthHandler(&health))
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`<html>
+		<head><title>OP Exporter</title></head>
+		<body>
+		<h1>OP Exporter</h1>
+		<p><a href="/metrics">Metrics</a></p>
+		<p><a href="/health">Health</a></p>
+		</body>
+		</html>`))
+	})
+	go getRollupGasPrices()
+	go getBlockNumber(&health)
+	log.Infoln("Listening on", *listenAddress)
+	if err := http.ListenAndServe(*listenAddress, nil); err != nil {
+		log.Fatal(err)
+	}
+
+}
+
+func getBlockNumber(health *healthCheck) {
+	rpcClient := jsonrpc.NewClientWithOpts(*rpcProvider, &jsonrpc.RPCClientOpts{})
+	var blockNumberResponse *string
+	for {
+		if err := rpcClient.CallFor(&blockNumberResponse, "eth_blockNumber"); err != nil {
+			health.mu.Lock()
+			health.healthy = false
+			health.mu.Unlock()
+			log.Warnln("Error calling eth_blockNumber, setting unhealthy", err)
+		} else {
+			log.Infoln("Got block number: ", *blockNumberResponse)
+			health.mu.Lock()
+			currentHeight, err := hexutil.DecodeUint64(*blockNumberResponse)
+			blockNumber.WithLabelValues(
+				*networkLabel, "layer2").Set(float64(currentHeight))
+			if err != nil {
+				log.Warnln("Error decoding block height", err)
+				continue
+			}
+			lastHeight := health.height
+			// If the currentHeight is the same as the lastHeight, check that
+			// the unhealthyTimePeriod has passed and update health.healthy
+			if currentHeight == lastHeight {
+				currentTime := time.Now()
+				lastTime := health.updateTime
+				log.Warnln(fmt.Sprintf("Heights are the same, %v, %v", currentTime, lastTime))
+				if lastTime.Add(time.Duration(*unhealthyTimePeriod) * time.Minute).Before(currentTime) {
+					health.healthy = false
+					log.Warnln("Heights are the same for the unhealthyTimePeriod, setting unhealthy")
+				}
+			} else {
+				log.Warnln("New block height detected, setting healthy")
+				health.height = currentHeight
+				health.updateTime = time.Now()
+				health.healthy = true
+			}
+			if health.healthy {
+				healthySequencer.WithLabelValues(
+					*networkLabel).Set(1)
+			} else {
+				healthySequencer.WithLabelValues(
+					*networkLabel).Set(0)
+			}
+
+			health.mu.Unlock()
+		}
+		time.Sleep(time.Duration(30) * time.Second)
+	}
+}
+
+func getRollupGasPrices() {
+	rpcClient := jsonrpc.NewClientWithOpts(*rpcProvider, &jsonrpc.RPCClientOpts{})
+	var rollupGasPrices *GetRollupGasPrices
+	for {
+		if err := rpcClient.CallFor(&rollupGasPrices, "rollup_gasPrices"); err != nil {
+			log.Warnln("Error calling rollup_gasPrices", err)
+		} else {
+			l1GasPriceString := rollupGasPrices.L1GasPrice
+			l1GasPrice, err := hexutil.DecodeUint64(l1GasPriceString)
+			if err != nil {
+				log.Warnln("Error converting gasPrice " + l1GasPriceString)
+			}
+			gasPrice.WithLabelValues(
+				*networkLabel, "layer1").Set(float64(l1GasPrice))
+			l2GasPriceString := rollupGasPrices.L2GasPrice
+			l2GasPrice, err := hexutil.DecodeUint64(l2GasPriceString)
+			if err != nil {
+				log.Warnln("Error converting gasPrice " + l2GasPriceString)
+			}
+			gasPrice.WithLabelValues(
+				*networkLabel, "layer2").Set(float64(l2GasPrice))
+			log.Infoln("Got L1 gas string: ", l1GasPriceString)
+			log.Infoln("Got L1 gas prices: ", l1GasPrice)
+			log.Infoln("Got L2 gas string: ", l2GasPriceString)
+			log.Infoln("Got L2 gas prices: ", l2GasPrice)
+		}
+		time.Sleep(time.Duration(30) * time.Second)
+	}
+}

go/op_exporter/rpc.go

+package main
+
+// GetRollupGasPrices returns the rpc `rollup_gasPrices` status
+type GetRollupGasPrices struct {
+	L1GasPrice string `json:"l1GasPrice"`
+	L2GasPrice string `json:"l2GasPrice"`
+}
+
+type GetBlockNumber struct {
+	BlockNumber string `json:"result"`
+}

go/op_exporter/version/version.go

+package version
+
+import (
+	"fmt"
+	"runtime"
+)
+
+var (
+	Version   string
+	GitCommit string
+	BuildUser string
+	BuildDate string
+	GoVersion = runtime.Version()
+)
+
+func Info() string {
+	return fmt.Sprintf("(version=%s, gitcommit=%s)", Version, GitCommit)
+}
+
+func BuildContext() string {
+	return fmt.Sprintf("(go=%s, user=%s, date=%s)", GoVersion, BuildUser, BuildDate)
+}

ops/docker/Dockerfile.op_exporter

+FROM golang:1.16 as builder
+
+ADD ./go/op_exporter /app/
+WORKDIR /app/
+RUN make build
+
+FROM alpine:latest
+RUN apk --no-cache add ca-certificates
+WORKDIR /root/
+COPY --from=builder /app/op_exporter /usr/local/bin/
+ENTRYPOINT ["op_exporter"]
+CMD ["--help"]