Merge pull request #5322 from ethereum-optimism/jg/extended_metrics

txmgr: Add more metrics

op-service/metrics/event.go

@@ -52,7 +52,6 @@ func NewEventVec(factory Factory, ns string, name string, displayName string, la
 			Namespace: ns,
 			Name:      fmt.Sprintf("last_%s_unix", name),
 			Help:      fmt.Sprintf("Timestamp of last %s event", displayName),
-		},
-			labelNames),
+		}, labelNames),
 	}
 }

op-service/txmgr/metrics/noop.go

@@ -4,6 +4,9 @@ import "github.com/ethereum/go-ethereum/core/types"
 
 type NoopTxMetrics struct{}
 
-func (*NoopTxMetrics) RecordL1GasFee(*types.Receipt)     {}
+func (*NoopTxMetrics) RecordNonce(uint64)                {}
 func (*NoopTxMetrics) RecordGasBumpCount(int)            {}
 func (*NoopTxMetrics) RecordTxConfirmationLatency(int64) {}
+func (*NoopTxMetrics) TxConfirmed(*types.Receipt)        {}
+func (*NoopTxMetrics) TxPublished(string)                {}
+func (*NoopTxMetrics) RPCError()                         {}

op-service/txmgr/metrics/tx_metrics.go

@@ -9,15 +9,34 @@ import (
 )
 
 type TxMetricer interface {
-	RecordL1GasFee(receipt *types.Receipt)
-	RecordGasBumpCount(times int)
-	RecordTxConfirmationLatency(latency int64)
+	RecordGasBumpCount(int)
+	RecordTxConfirmationLatency(int64)
+	RecordNonce(uint64)
+	TxConfirmed(*types.Receipt)
+	TxPublished(string)
+	RPCError()
 }
 
 type TxMetrics struct {
 	TxL1GasFee         prometheus.Gauge
 	TxGasBump          prometheus.Gauge
 	LatencyConfirmedTx prometheus.Gauge
+	currentNonce       prometheus.Gauge
+	txPublishError     *prometheus.CounterVec
+	publishEvent       metrics.Event
+	confirmEvent       metrics.EventVec
+	rpcError           prometheus.Counter
+}
+
+func receiptStatusString(receipt *types.Receipt) string {
+	switch receipt.Status {
+	case types.ReceiptStatusSuccessful:
+		return "success"
+	case types.ReceiptStatusFailed:
+		return "failed"
+	default:
+		return "unknown_status"
+	}
 }
 
 var _ TxMetricer = (*TxMetrics)(nil)
@@ -42,10 +61,36 @@ func MakeTxMetrics(ns string, factory metrics.Factory) TxMetrics {
 			Help:      "Latency of a confirmed transaction in milliseconds",
 			Subsystem: "txmgr",
 		}),
+		currentNonce: factory.NewGauge(prometheus.GaugeOpts{
+			Namespace: ns,
+			Name:      "current_nonce",
+			Help:      "Current nonce of the from address",
+			Subsystem: "txmgr",
+		}),
+		txPublishError: factory.NewCounterVec(prometheus.CounterOpts{
+			Namespace: ns,
+			Name:      "tx_publish_error_count",
+			Help:      "Count of publish errors. Labells are sanitized error strings",
+			Subsystem: "txmgr",
+		}, []string{"error"}),
+		confirmEvent: metrics.NewEventVec(factory, ns, "confirm", "tx confirm", []string{"status"}),
+		publishEvent: metrics.NewEvent(factory, ns, "publish", "tx publish"),
+		rpcError: factory.NewCounter(prometheus.CounterOpts{
+			Namespace: ns,
+			Name:      "rpc_error_count",
+			Help:      "Temporrary: Count of RPC errors (like timeouts) that have occurrred",
+			Subsystem: "txmgr",
+		}),
 	}
 }
 
-func (t *TxMetrics) RecordL1GasFee(receipt *types.Receipt) {
+func (t *TxMetrics) RecordNonce(nonce uint64) {
+	t.currentNonce.Set(float64(nonce))
+}
+
+// TxConfirmed records lots of information about the confirmed transaction
+func (t *TxMetrics) TxConfirmed(receipt *types.Receipt) {
+	t.confirmEvent.Record(receiptStatusString(receipt))
 	t.TxL1GasFee.Set(float64(receipt.EffectiveGasPrice.Uint64() * receipt.GasUsed / params.GWei))
 }
 
@@ -56,3 +101,15 @@ func (t *TxMetrics) RecordGasBumpCount(times int) {
 func (t *TxMetrics) RecordTxConfirmationLatency(latency int64) {
 	t.LatencyConfirmedTx.Set(float64(latency))
 }
+
+func (t *TxMetrics) TxPublished(errString string) {
+	if errString != "" {
+		t.txPublishError.WithLabelValues(errString).Inc()
+	} else {
+		t.publishEvent.Record()
+	}
+}
+
+func (t *TxMetrics) RPCError() {
+	t.rpcError.Inc()
+}

op-service/txmgr/txmgr.go

@@ -148,6 +148,7 @@ func (m *SimpleTxManager) Send(ctx context.Context, candidate TxCandidate) (*typ
 func (m *SimpleTxManager) craftTx(ctx context.Context, candidate TxCandidate) (*types.Transaction, error) {
 	gasTipCap, basefee, err := m.suggestGasPriceCaps(ctx)
 	if err != nil {
+		m.metr.RPCError()
 		return nil, fmt.Errorf("failed to get gas price info: %w", err)
 	}
 	gasFeeCap := calcGasFeeCap(basefee, gasTipCap)
@@ -157,8 +158,10 @@ func (m *SimpleTxManager) craftTx(ctx context.Context, candidate TxCandidate) (*
 	defer cancel()
 	nonce, err := m.backend.NonceAt(childCtx, m.cfg.From, nil)
 	if err != nil {
+		m.metr.RPCError()
 		return nil, fmt.Errorf("failed to get nonce: %w", err)
 	}
+	m.metr.RecordNonce(nonce)
 
 	rawTx := &types.DynamicFeeTx{
 		ChainID:   m.chainID,
@@ -240,6 +243,7 @@ func (m *SimpleTxManager) send(ctx context.Context, tx *types.Transaction) (*typ
 
 		case receipt := <-receiptChan:
 			m.metr.RecordGasBumpCount(bumpCounter)
+			m.metr.TxConfirmed(receipt)
 			return receipt, nil
 		}
 	}
@@ -263,19 +267,28 @@ func (m *SimpleTxManager) publishAndWaitForTx(ctx context.Context, tx *types.Tra
 		switch {
 		case errStringMatch(err, core.ErrNonceTooLow):
 			log.Warn("nonce too low", "err", err)
+			m.metr.TxPublished("nonce_to_low")
 		case errStringMatch(err, context.Canceled):
+			m.metr.RPCError()
 			log.Warn("transaction send cancelled", "err", err)
+			m.metr.TxPublished("context_cancelled")
 		case errStringMatch(err, txpool.ErrAlreadyKnown):
 			log.Warn("resubmitted already known transaction", "err", err)
+			m.metr.TxPublished("tx_already_known")
 		case errStringMatch(err, txpool.ErrReplaceUnderpriced):
 			log.Warn("transaction replacement is underpriced", "err", err)
+			m.metr.TxPublished("tx_replacement_underpriced")
 		case errStringMatch(err, txpool.ErrUnderpriced):
 			log.Warn("transaction is underpriced", "err", err)
+			m.metr.TxPublished("tx_underpriced")
 		default:
+			m.metr.RPCError()
 			log.Error("unable to publish transaction", "err", err)
+			m.metr.TxPublished("unknown_error")
 		}
 		return
 	}
+	m.metr.TxPublished("")
 
 	log.Info("Transaction successfully published")
 	// Poll for the transaction to be ready & then send the result to receiptChan
@@ -287,7 +300,6 @@ func (m *SimpleTxManager) publishAndWaitForTx(ctx context.Context, tx *types.Tra
 	select {
 	case receiptChan <- receipt:
 		m.metr.RecordTxConfirmationLatency(time.Since(t).Milliseconds())
-		m.metr.RecordL1GasFee(receipt)
 	default:
 	}
 }
@@ -319,9 +331,11 @@ func (m *SimpleTxManager) queryReceipt(ctx context.Context, txHash common.Hash,
 		m.l.Trace("Transaction not yet mined", "hash", txHash)
 		return nil
 	} else if err != nil {
+		m.metr.RPCError()
 		m.l.Info("Receipt retrieval failed", "hash", txHash, "err", err)
 		return nil
 	} else if receipt == nil {
+		m.metr.RPCError()
 		m.l.Warn("Receipt and error are both nil", "hash", txHash)
 		return nil
 	}
@@ -405,6 +419,7 @@ func (m *SimpleTxManager) suggestGasPriceCaps(ctx context.Context) (*big.Int, *b
 	defer cancel()
 	tip, err := m.backend.SuggestGasTipCap(cCtx)
 	if err != nil {
+		m.metr.RPCError()
 		return nil, nil, fmt.Errorf("failed to fetch the suggested gas tip cap: %w", err)
 	} else if tip == nil {
 		return nil, nil, errors.New("the suggested tip was nil")
@@ -413,6 +428,7 @@ func (m *SimpleTxManager) suggestGasPriceCaps(ctx context.Context) (*big.Int, *b
 	defer cancel()
 	head, err := m.backend.HeaderByNumber(cCtx, nil)
 	if err != nil {
+		m.metr.RPCError()
 		return nil, nil, fmt.Errorf("failed to fetch the suggested basefee: %w", err)
 	} else if head.BaseFee == nil {
 		return nil, nil, errors.New("txmgr does not support pre-london blocks that do not have a basefee")

op-service/txmgr/txmgr_test.go

@@ -691,6 +691,7 @@ func TestWaitMinedReturnsReceiptAfterFailure(t *testing.T) {
 		name:    "TEST",
 		backend: &borkedBackend,
 		l:       testlog.Logger(t, log.LvlCrit),
+		metr:    &metrics.NoopTxMetrics{},
 	}
 
 	// Don't mine the tx with the default backend. The failingBackend will
@@ -726,6 +727,7 @@ func doGasPriceIncrease(t *testing.T, txTipCap, txFeeCap, newTip, newBaseFee int
 		name:    "TEST",
 		backend: &borkedBackend,
 		l:       testlog.Logger(t, log.LvlCrit),
+		metr:    &metrics.NoopTxMetrics{},
 	}
 
 	tx := types.NewTx(&types.DynamicFeeTx{
@@ -829,6 +831,7 @@ func TestIncreaseGasPriceNotExponential(t *testing.T) {
 		name:    "TEST",
 		backend: &borkedBackend,
 		l:       testlog.Logger(t, log.LvlCrit),
+		metr:    &metrics.NoopTxMetrics{},
 	}
 	tx := types.NewTx(&types.DynamicFeeTx{
 		GasTipCap: big.NewInt(10),