teleportr: Improve metrics (#2690)

- Update metrics collection to happen on an interval in addition to on each tick of the driver loop. This lets us continue collecting on-chain metrics if the driver loop halts for some reason.
- Adds an additional metric to track the last on-chian deposit ID. This will let add Teleportr halting alerts more easily.

.changeset/curly-swans-crash.md

+---
+'@eth-optimism/teleportr': patch
+---
+
+Improve metrics

teleportr/drivers/disburser/driver.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"math/big"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/ethereum-optimism/optimism/bss-core/metrics"
@@ -48,6 +49,7 @@ type Config struct {
 	DisburserAddr        common.Address
 	ChainID              *big.Int
 	PrivKey              *ecdsa.PrivateKey
+	ChainMetricsEnable   bool
 }
 
 type Driver struct {
@@ -58,10 +60,12 @@ type Driver struct {
 	walletAddr           common.Address
 	metrics              *Metrics
 
-	currentDepositIDs []uint64
+	currentDepositIDs   []uint64
+	chainMetricsEnabled bool
+	metricsMu           sync.Mutex
 }
 
-func NewDriver(cfg Config) (*Driver, error) {
+func NewDriver(cfg Config, parentCtx context.Context) (*Driver, error) {
 	if cfg.NumConfirmations == 0 {
 		panic("NumConfirmations cannot be zero")
 	}
@@ -95,15 +99,21 @@ func NewDriver(cfg Config) (*Driver, error) {
 	)
 
 	walletAddr := crypto.PubkeyToAddress(cfg.PrivKey.PublicKey)
+	metricsInst := NewMetrics(cfg.Name)
 
-	return &Driver{
+	d := &Driver{
 		cfg:                  cfg,
 		depositContract:      depositContract,
 		disburserContract:    disburserContract,
 		rawDisburserContract: rawDisburserContract,
 		walletAddr:           walletAddr,
-		metrics:              NewMetrics(cfg.Name),
-	}, nil
+		metrics:              metricsInst,
+		chainMetricsEnabled:  cfg.ChainMetricsEnable,
+	}
+	if d.chainMetricsEnabled {
+		go d.collectChainMetricsBackground(parentCtx)
+	}
+	return d, nil
 }
 
 // Name is an identifier used to prefix logs for a particular service.
@@ -140,8 +150,8 @@ func (d *Driver) ClearPendingTx(
 func (d *Driver) GetBatchBlockRange(
 	ctx context.Context) (*big.Int, *big.Int, error) {
 
-	// Update balance metrics on each iteration.
-	d.updateBalanceMetrics(ctx)
+	// Update metrics on each iteration.
+	d.collectChainMetrics(ctx)
 
 	// Clear the current deposit IDs from any prior iteration.
 	d.currentDepositIDs = nil
@@ -720,19 +730,63 @@ func (d *Driver) deletePendingTx(startID, endID uint64) error {
 	return nil
 }
 
-func (d *Driver) updateBalanceMetrics(ctx context.Context) {
-	disburserBal, err := d.cfg.L2Client.BalanceAt(ctx, d.walletAddr, nil)
+func (d *Driver) collectChainMetricsBackground(ctx context.Context) {
+	tick := time.NewTicker(time.Minute)
+	defer tick.Stop()
+
+	for {
+		select {
+		case <-tick.C:
+			log.Info("collecting background metrics")
+			d.collectChainMetrics(ctx)
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (d *Driver) collectChainMetrics(ctx context.Context) {
+	if !d.chainMetricsEnabled {
+		return
+	}
+
+	subCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	disburserBal, err := d.cfg.L2Client.BalanceAt(subCtx, d.cfg.DisburserAddr, nil)
 	if err != nil {
 		log.Error("Error getting disburser wallet balance", "err", err)
 		disburserBal = big.NewInt(0)
 	}
 
-	depositBal, err := d.cfg.L1Client.BalanceAt(ctx, d.cfg.DepositAddr, nil)
+	depositBal, err := d.cfg.L1Client.BalanceAt(subCtx, d.cfg.DepositAddr, nil)
 	if err != nil {
 		log.Error("Error getting deposit contract balance", "err", err)
 		depositBal = big.NewInt(0)
 	}
 
+	nextDepositID, err := d.depositContract.TotalDeposits(&bind.CallOpts{
+		Context: subCtx,
+	})
+	if err != nil {
+		log.Error("Error getting deposit contract total deposits")
+		nextDepositID = big.NewInt(0)
+	}
+
+	nextDisbursementID, err := d.disburserContract.TotalDisbursements(
+		&bind.CallOpts{
+			Context: subCtx,
+		},
+	)
+	if err != nil {
+		log.Error("Error getting deposit contract total disbursements")
+		nextDisbursementID = big.NewInt(0)
+	}
+
+	d.metricsMu.Lock()
 	d.metrics.DisburserBalance.Set(float64(disburserBal.Uint64()))
 	d.metrics.DepositContractBalance.Set(float64(depositBal.Uint64()))
+	d.metrics.ContractNextDepositID.Set(float64(nextDepositID.Uint64()))
+	d.metrics.ContractNextDisbursementID.Set(float64(nextDisbursementID.Uint64()))
+	d.metricsMu.Unlock()
 }

teleportr/drivers/disburser/metrics.go

@@ -66,6 +66,10 @@ type Metrics struct {
 	// the disburser contract.
 	ContractNextDisbursementID prometheus.Gauge
 
+	// ContractNextDepositID tracks the next deposit id expected by the deposit
+	// contract.
+	ContractNextDepositID prometheus.Gauge
+
 	// DisburserBalance tracks Teleportr's disburser account balance.
 	DisburserBalance prometheus.Gauge
 
@@ -117,6 +121,11 @@ func NewMetrics(subsystem string) *Metrics {
 			Help:      "Next disbursement id expected by the disburser contract",
 			Subsystem: base.SubsystemName(),
 		}),
+		ContractNextDepositID: promauto.NewGauge(prometheus.GaugeOpts{
+			Name:      "contract_next_deposit_id",
+			Help:      "next deposit id expected by the deposit contract",
+			Subsystem: base.SubsystemName(),
+		}),
 		DisburserBalance: promauto.NewGauge(prometheus.GaugeOpts{
 			Name:      "disburser_balance",
 			Help:      "Balance in Wei of Teleportr's disburser wallet",

teleportr/teleportr.go

@@ -110,7 +110,8 @@ func Main(gitVersion string) func(ctx *cli.Context) error {
 			DisburserAddr:        disburserAddr,
 			ChainID:              chainID,
 			PrivKey:              disburserPrivKey,
-		})
+			ChainMetricsEnable:   cfg.MetricsServerEnable,
+		}, ctx)
 		if err != nil {
 			return err
 		}