feat(op-dispute-mon): WithdrawalRequests Metrics (#9942)

* feat(op-challenger): Delayed Weth Withdrawal Request Caller

* fix(op-challenger): withdrawal request field ordering

* fix(op-challenger): encapsulate delayed weth behind the fault dispute game contract binding

* feat(op-dispute-mon): get withdrawals game caller method

* fix(op-dispute-mon): revert service name changes

* fix(op-dispute-mon): revert extractor test name

* fix(op-dispute-mon): revert extractor changes

* fix(op-dispute-mon): revert newline add

* fix(op-dispute-mon): revert caller field reordering

* feat(op-dispute-mon): weth caller creation and pass it through sub components

* chore(op-dispute-mon): weth caller creation unit test

* fix(op-dispute-mon): extractor test for weth caller creation error

* feat(op-dispute-mon): wire up the withdrawals extractor

* fix(op-dispute-mon): rebases

* fix(op-dispute-mon): revert caller test change

* fix(op-dispute-mon): recipient enricher

* feat(op-dispute-mon): withdrawal request metrics

* Update op-dispute-mon/mon/withdrawals.go
Co-authored-by: Adrian Sutton <adrian@oplabs.co>

* fix(op-dispute-mon): withdrawal request amounts

---------
Co-authored-by: Adrian Sutton <adrian@oplabs.co>

op-dispute-mon/metrics/metrics.go

@@ -39,6 +39,8 @@ type Metricer interface {
 	RecordInfo(version string)
 	RecordUp()
 
+	RecordWithdrawalRequests(delayedWeth common.Address, matches bool, count int)
+
 	RecordClaimResolutionDelayMax(delay float64)
 
 	RecordOutputFetchTime(timestamp float64)
@@ -62,6 +64,8 @@ type Metrics struct {
 	*opmetrics.CacheMetrics
 	*contractMetrics.ContractMetrics
 
+	withdrawalRequests prometheus.GaugeVec
+
 	info prometheus.GaugeVec
 	up   prometheus.Gauge
 
@@ -115,6 +119,14 @@ func NewMetrics() *Metrics {
 			Name:      "claim_resolution_delay_max",
 			Help:      "Maximum claim resolution delay in seconds",
 		}),
+		withdrawalRequests: *factory.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Name:      "withdrawal_requests",
+			Help:      "Number of withdrawal requests categorised by the source DelayedWETH contract and whether the withdrawal request amount matches or diverges from its fault dispute game credits",
+		}, []string{
+			"delayedWETH",
+			"credits",
+		}),
 		gamesAgreement: *factory.NewGaugeVec(prometheus.GaugeOpts{
 			Namespace: Namespace,
 			Name:      "games_agreement",
@@ -172,6 +184,14 @@ func (m *Metrics) RecordUp() {
 	m.up.Set(1)
 }
 
+func (m *Metrics) RecordWithdrawalRequests(delayedWeth common.Address, matches bool, count int) {
+	credits := "matching"
+	if !matches {
+		credits = "divergent"
+	}
+	m.withdrawalRequests.WithLabelValues(delayedWeth.Hex(), credits).Set(float64(count))
+}
+
 func (m *Metrics) RecordClaimResolutionDelayMax(delay float64) {
 	m.claimResolutionDelayMax.Set(delay)
 }

op-dispute-mon/metrics/noop.go

@@ -19,6 +19,7 @@ func (*NoopMetricsImpl) RecordUp()                 {}
 func (*NoopMetricsImpl) CacheAdd(_ string, _ int, _ bool) {}
 func (*NoopMetricsImpl) CacheGet(_ string, _ bool)        {}
 
+func (*NoopMetricsImpl) RecordWithdrawalRequests(_ common.Address, _ bool, _ int) {}
 func (*NoopMetricsImpl) RecordClaimResolutionDelayMax(delay float64)              {}
 
 func (*NoopMetricsImpl) RecordOutputFetchTime(timestamp float64) {}

op-dispute-mon/mon/monitor.go

@@ -15,6 +15,7 @@ import (
 
 type Forecast func(ctx context.Context, games []*types.EnrichedGameData)
 type Bonds func(games []*types.EnrichedGameData)
+type MonitorWithdrawals func(games []*types.EnrichedGameData)
 type BlockHashFetcher func(ctx context.Context, number *big.Int) (common.Hash, error)
 type BlockNumberFetcher func(ctx context.Context) (uint64, error)
 type Extract func(ctx context.Context, blockHash common.Hash, minTimestamp uint64) ([]*types.EnrichedGameData, error)
@@ -34,6 +35,7 @@ type gameMonitor struct {
 	delays           RecordClaimResolutionDelayMax
 	forecast         Forecast
 	bonds            Bonds
+	withdrawals      MonitorWithdrawals
 	extract          Extract
 	fetchBlockHash   BlockHashFetcher
 	fetchBlockNumber BlockNumberFetcher
@@ -48,6 +50,7 @@ func newGameMonitor(
 	delays RecordClaimResolutionDelayMax,
 	forecast Forecast,
 	bonds Bonds,
+	withdrawals MonitorWithdrawals,
 	extract Extract,
 	fetchBlockNumber BlockNumberFetcher,
 	fetchBlockHash BlockHashFetcher,
@@ -62,6 +65,7 @@ func newGameMonitor(
 		delays:           delays,
 		forecast:         forecast,
 		bonds:            bonds,
+		withdrawals:      withdrawals,
 		extract:          extract,
 		fetchBlockNumber: fetchBlockNumber,
 		fetchBlockHash:   fetchBlockHash,
@@ -86,6 +90,7 @@ func (m *gameMonitor) monitorGames() error {
 	m.delays(enrichedGames)
 	m.forecast(m.ctx, enrichedGames)
 	m.bonds(enrichedGames)
+	m.withdrawals(enrichedGames)
 	return nil
 }
 

op-dispute-mon/mon/monitor_test.go

@@ -24,7 +24,7 @@ func TestMonitor_MonitorGames(t *testing.T) {
 	t.Parallel()
 
 	t.Run("FailedFetchBlocknumber", func(t *testing.T) {
-		monitor, _, _, _, _ := setupMonitorTest(t)
+		monitor, _, _, _, _, _ := setupMonitorTest(t)
 		boom := errors.New("boom")
 		monitor.fetchBlockNumber = func(ctx context.Context) (uint64, error) {
 			return 0, boom
@@ -34,7 +34,7 @@ func TestMonitor_MonitorGames(t *testing.T) {
 	})
 
 	t.Run("FailedFetchBlockHash", func(t *testing.T) {
-		monitor, _, _, _, _ := setupMonitorTest(t)
+		monitor, _, _, _, _, _ := setupMonitorTest(t)
 		boom := errors.New("boom")
 		monitor.fetchBlockHash = func(ctx context.Context, number *big.Int) (common.Hash, error) {
 			return common.Hash{}, boom
@@ -44,23 +44,25 @@ func TestMonitor_MonitorGames(t *testing.T) {
 	})
 
 	t.Run("MonitorsWithNoGames", func(t *testing.T) {
-		monitor, factory, forecast, delays, bonds := setupMonitorTest(t)
+		monitor, factory, forecast, delays, bonds, withdrawals := setupMonitorTest(t)
 		factory.games = []*monTypes.EnrichedGameData{}
 		err := monitor.monitorGames()
 		require.NoError(t, err)
 		require.Equal(t, 1, forecast.calls)
 		require.Equal(t, 1, delays.calls)
 		require.Equal(t, 1, bonds.calls)
+		require.Equal(t, 1, withdrawals.calls)
 	})
 
 	t.Run("MonitorsMultipleGames", func(t *testing.T) {
-		monitor, factory, forecast, delays, bonds := setupMonitorTest(t)
+		monitor, factory, forecast, delays, bonds, withdrawals := setupMonitorTest(t)
 		factory.games = []*monTypes.EnrichedGameData{{}, {}, {}}
 		err := monitor.monitorGames()
 		require.NoError(t, err)
 		require.Equal(t, 1, forecast.calls)
 		require.Equal(t, 1, delays.calls)
 		require.Equal(t, 1, bonds.calls)
+		require.Equal(t, 1, withdrawals.calls)
 	})
 }
 
@@ -68,7 +70,7 @@ func TestMonitor_StartMonitoring(t *testing.T) {
 	t.Run("MonitorsGames", func(t *testing.T) {
 		addr1 := common.Address{0xaa}
 		addr2 := common.Address{0xbb}
-		monitor, factory, forecaster, _, _ := setupMonitorTest(t)
+		monitor, factory, forecaster, _, _, _ := setupMonitorTest(t)
 		factory.games = []*monTypes.EnrichedGameData{newEnrichedGameData(addr1, 9999), newEnrichedGameData(addr2, 9999)}
 		factory.maxSuccess = len(factory.games) // Only allow two successful fetches
 
@@ -81,7 +83,7 @@ func TestMonitor_StartMonitoring(t *testing.T) {
 	})
 
 	t.Run("FailsToFetchGames", func(t *testing.T) {
-		monitor, factory, forecaster, _, _ := setupMonitorTest(t)
+		monitor, factory, forecaster, _, _, _ := setupMonitorTest(t)
 		factory.fetchErr = errors.New("boom")
 
 		monitor.StartMonitoring()
@@ -103,7 +105,7 @@ func newEnrichedGameData(proxy common.Address, timestamp uint64) *monTypes.Enric
 	}
 }
 
-func setupMonitorTest(t *testing.T) (*gameMonitor, *mockExtractor, *mockForecast, *mockDelayCalculator, *mockBonds) {
+func setupMonitorTest(t *testing.T) (*gameMonitor, *mockExtractor, *mockForecast, *mockDelayCalculator, *mockBonds, *mockWithdrawalMonitor) {
 	logger := testlog.Logger(t, log.LvlDebug)
 	fetchBlockNum := func(ctx context.Context) (uint64, error) {
 		return 1, nil
@@ -117,6 +119,7 @@ func setupMonitorTest(t *testing.T) (*gameMonitor, *mockExtractor, *mockForecast
 	extractor := &mockExtractor{}
 	forecast := &mockForecast{}
 	bonds := &mockBonds{}
+	withdrawals := &mockWithdrawalMonitor{}
 	delays := &mockDelayCalculator{}
 	monitor := newGameMonitor(
 		context.Background(),
@@ -127,11 +130,20 @@ func setupMonitorTest(t *testing.T) (*gameMonitor, *mockExtractor, *mockForecast
 		delays.RecordClaimResolutionDelayMax,
 		forecast.Forecast,
 		bonds.CheckBonds,
+		withdrawals.CheckWithdrawals,
 		extractor.Extract,
 		fetchBlockNum,
 		fetchBlockHash,
 	)
-	return monitor, extractor, forecast, delays, bonds
+	return monitor, extractor, forecast, delays, bonds, withdrawals
+}
+
+type mockWithdrawalMonitor struct {
+	calls int
+}
+
+func (m *mockWithdrawalMonitor) CheckWithdrawals(games []*monTypes.EnrichedGameData) {
+	m.calls++
 }
 
 type mockDelayCalculator struct {

op-dispute-mon/mon/service.go

@@ -42,6 +42,7 @@ type Service struct {
 	forecast     *forecast
 	bonds        *bonds.Bonds
 	game         *extract.GameCallerCreator
+	withdrawals  *WithdrawalMonitor
 	rollupClient *sources.RollupClient
 	validator    *outputValidator
 
@@ -85,6 +86,8 @@ func (s *Service) initFromConfig(ctx context.Context, cfg *config.Config) error
 		return fmt.Errorf("failed to init rollup client: %w", err)
 	}
 
+	s.initWithdrawalMonitor()
+
 	s.initOutputValidator()   // Must be called before initForecast
 	s.initGameCallerCreator() // Must be called before initForecast
 
@@ -102,6 +105,10 @@ func (s *Service) initFromConfig(ctx context.Context, cfg *config.Config) error
 	return nil
 }
 
+func (s *Service) initWithdrawalMonitor() {
+	s.withdrawals = NewWithdrawalMonitor(s.logger, s.metrics)
+}
+
 func (s *Service) initOutputValidator() {
 	s.validator = newOutputValidator(s.logger, s.metrics, s.rollupClient)
 }
@@ -216,6 +223,7 @@ func (s *Service) initMonitor(ctx context.Context, cfg *config.Config) {
 		s.delays.RecordClaimResolutionDelayMax,
 		s.forecast.Forecast,
 		s.bonds.CheckBonds,
+		s.withdrawals.CheckWithdrawals,
 		s.extractor.Extract,
 		s.l1Client.BlockNumber,
 		blockHashFetcher,

op-dispute-mon/mon/withdrawals.go

+package mon
+
+import (
+	"github.com/ethereum-optimism/optimism/op-dispute-mon/mon/types"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+type WithdrawalMetrics interface {
+	RecordWithdrawalRequests(delayedWeth common.Address, matches bool, count int)
+}
+
+type WithdrawalMonitor struct {
+	logger  log.Logger
+	metrics WithdrawalMetrics
+}
+
+func NewWithdrawalMonitor(logger log.Logger, metrics WithdrawalMetrics) *WithdrawalMonitor {
+	return &WithdrawalMonitor{
+		logger:  logger,
+		metrics: metrics,
+	}
+}
+
+func (w *WithdrawalMonitor) CheckWithdrawals(games []*types.EnrichedGameData) {
+	matching := make(map[common.Address]int)
+	divergent := make(map[common.Address]int)
+	for _, game := range games {
+		matches, diverges := w.validateGameWithdrawals(game)
+		matching[game.WETHContract] += matches
+		divergent[game.WETHContract] += diverges
+	}
+	for contract, count := range matching {
+		w.metrics.RecordWithdrawalRequests(contract, true, count)
+	}
+	for contract, count := range divergent {
+		w.metrics.RecordWithdrawalRequests(contract, false, count)
+	}
+}
+
+func (w *WithdrawalMonitor) validateGameWithdrawals(game *types.EnrichedGameData) (int, int) {
+	matching := 0
+	divergent := 0
+	for recipient, withdrawalAmount := range game.WithdrawalRequests {
+		if withdrawalAmount.Amount != nil && withdrawalAmount.Amount.Cmp(game.Credits[recipient]) == 0 {
+			matching++
+		} else {
+			divergent++
+			w.logger.Error("Withdrawal request amount does not match credit", "game", game.Proxy, "recipient", recipient, "credit", game.Credits[recipient], "withdrawal", game.WithdrawalRequests[recipient].Amount)
+		}
+	}
+	return matching, divergent
+}

op-dispute-mon/mon/withdrawals_test.go

+package mon
+
+import (
+	"math/big"
+	"testing"
+
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/contracts"
+	monTypes "github.com/ethereum-optimism/optimism/op-dispute-mon/mon/types"
+	"github.com/ethereum-optimism/optimism/op-service/testlog"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	weth1 = common.Address{0x1a}
+	weth2 = common.Address{0x2b}
+)
+
+func makeGames() []*monTypes.EnrichedGameData {
+	weth1Balance := big.NewInt(4200)
+	weth2Balance := big.NewInt(6000)
+	game1 := &monTypes.EnrichedGameData{
+		Credits: map[common.Address]*big.Int{
+			common.Address{0x01}: big.NewInt(3),
+			common.Address{0x02}: big.NewInt(1),
+		},
+		WithdrawalRequests: map[common.Address]*contracts.WithdrawalRequest{
+			common.Address{0x01}: &contracts.WithdrawalRequest{Amount: big.NewInt(3)},
+			common.Address{0x02}: &contracts.WithdrawalRequest{Amount: big.NewInt(1)},
+		},
+		WETHContract:  weth1,
+		ETHCollateral: weth1Balance,
+	}
+	game2 := &monTypes.EnrichedGameData{
+		Credits: map[common.Address]*big.Int{
+			common.Address{0x01}: big.NewInt(46),
+			common.Address{0x02}: big.NewInt(1),
+		},
+		WithdrawalRequests: map[common.Address]*contracts.WithdrawalRequest{
+			common.Address{0x01}: &contracts.WithdrawalRequest{Amount: big.NewInt(3)},
+			common.Address{0x02}: &contracts.WithdrawalRequest{Amount: big.NewInt(1)},
+		},
+		WETHContract:  weth2,
+		ETHCollateral: weth2Balance,
+	}
+	game3 := &monTypes.EnrichedGameData{
+		Credits: map[common.Address]*big.Int{
+			common.Address{0x03}: big.NewInt(2),
+			common.Address{0x04}: big.NewInt(4),
+		},
+		WithdrawalRequests: map[common.Address]*contracts.WithdrawalRequest{
+			common.Address{0x03}: &contracts.WithdrawalRequest{Amount: big.NewInt(2)},
+			common.Address{0x04}: &contracts.WithdrawalRequest{Amount: big.NewInt(4)},
+		},
+		WETHContract:  weth2,
+		ETHCollateral: weth2Balance,
+	}
+	return []*monTypes.EnrichedGameData{game1, game2, game3}
+}
+
+func TestCheckWithdrawals(t *testing.T) {
+	logger := testlog.Logger(t, log.LvlInfo)
+	metrics := &stubWithdrawalsMetrics{
+		matching:  make(map[common.Address]int),
+		divergent: make(map[common.Address]int),
+	}
+	withdrawals := NewWithdrawalMonitor(logger, metrics)
+	withdrawals.CheckWithdrawals(makeGames())
+
+	require.Equal(t, metrics.matchCalls, 2)
+	require.Equal(t, metrics.divergeCalls, 2)
+	require.Len(t, metrics.matching, 2)
+	require.Len(t, metrics.divergent, 2)
+	require.Contains(t, metrics.matching, weth1)
+	require.Contains(t, metrics.matching, weth2)
+	require.Contains(t, metrics.divergent, weth1)
+	require.Contains(t, metrics.divergent, weth2)
+	require.Equal(t, metrics.matching[weth1], 2)
+	require.Equal(t, metrics.matching[weth2], 3)
+	require.Equal(t, metrics.divergent[weth1], 0)
+	require.Equal(t, metrics.divergent[weth2], 1)
+}
+
+type stubWithdrawalsMetrics struct {
+	matchCalls   int
+	divergeCalls int
+	matching     map[common.Address]int
+	divergent    map[common.Address]int
+}
+
+func (s *stubWithdrawalsMetrics) RecordWithdrawalRequests(addr common.Address, matches bool, count int) {
+	if matches {
+		s.matchCalls++
+		s.matching[addr] = count
+	} else {
+		s.divergeCalls++
+		s.divergent[addr] = count
+	}
+}