feat(op-dispute-mon): Track and Metrice Credit Balances (#9928)

* feat(op-dispute-mon): track and metrice credit balances

* fix(op-dispute-mon): Enrich Required Bonds (#9930)

* fix(op-dispute-mon): bad merge

* fix(op-dispute-mon): credit balance check final touches and logging

* Update op-dispute-mon/mon/bonds/monitor.go
Co-authored-by: Inphi <mlaw2501@gmail.com>

* fix(op-dispute-mon): iterative logic

* fix(op-dispute-mon): lints

---------
Co-authored-by: Inphi <mlaw2501@gmail.com>

op-challenger/game/fault/contracts/faultdisputegame.go

@@ -170,6 +170,22 @@ func (f *FaultDisputeGameContract) GetCredit(ctx context.Context, recipient comm
 	return credit, status, nil
 }
 
+func (f *FaultDisputeGameContract) GetRequiredBonds(ctx context.Context, block rpcblock.Block, positions ...*big.Int) ([]*big.Int, error) {
+	calls := make([]batching.Call, 0, len(positions))
+	for _, position := range positions {
+		calls = append(calls, f.contract.Call(methodRequiredBond, position))
+	}
+	results, err := f.multiCaller.Call(ctx, block, calls...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve required bonds: %w", err)
+	}
+	requiredBonds := make([]*big.Int, 0, len(positions))
+	for _, result := range results {
+		requiredBonds = append(requiredBonds, result.GetBigInt(0))
+	}
+	return requiredBonds, nil
+}
+
 func (f *FaultDisputeGameContract) GetCredits(ctx context.Context, block rpcblock.Block, recipients ...common.Address) ([]*big.Int, error) {
 	defer f.metrics.StartContractRequest("GetCredits")()
 	calls := make([]batching.Call, 0, len(recipients))

op-dispute-mon/metrics/metrics.go

@@ -19,6 +19,20 @@ import (
 
 const Namespace = "op_dispute_mon"
 
+type CreditExpectation uint8
+
+const (
+	// Max Duration reached
+	CreditBelowMaxDuration CreditExpectation = iota
+	CreditEqualMaxDuration
+	CreditAboveMaxDuration
+
+	// Max Duration not reached
+	CreditBelowNonMaxDuration
+	CreditEqualNonMaxDuration
+	CreditAboveNonMaxDuration
+)
+
 type GameAgreementStatus uint8
 
 const (
@@ -55,6 +69,8 @@ type Metricer interface {
 	RecordInfo(version string)
 	RecordUp()
 
+	RecordCredit(expectation CreditExpectation, count int)
+
 	RecordClaims(status ClaimStatus, count int)
 
 	RecordWithdrawalRequests(delayedWeth common.Address, matches bool, count int)
@@ -89,6 +105,8 @@ type Metrics struct {
 	info prometheus.GaugeVec
 	up   prometheus.Gauge
 
+	credits prometheus.GaugeVec
+
 	lastOutputFetch prometheus.Gauge
 
 	claimResolutionDelayMax prometheus.Gauge
@@ -139,6 +157,14 @@ func NewMetrics() *Metrics {
 			Name:      "claim_resolution_delay_max",
 			Help:      "Maximum claim resolution delay in seconds",
 		}),
+		credits: *factory.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Name:      "credits",
+			Help:      "Cumulative credits",
+		}, []string{
+			"credit",
+			"max_duration",
+		}),
 		claims: *factory.NewGaugeVec(prometheus.GaugeOpts{
 			Namespace: Namespace,
 			Name:      "claims",
@@ -213,6 +239,28 @@ func (m *Metrics) RecordUp() {
 	m.up.Set(1)
 }
 
+func (m *Metrics) RecordCredit(expectation CreditExpectation, count int) {
+	asLabels := func(expectation CreditExpectation) []string {
+		switch expectation {
+		case CreditBelowMaxDuration:
+			return []string{"below", "max_duration"}
+		case CreditEqualMaxDuration:
+			return []string{"expected", "max_duration"}
+		case CreditAboveMaxDuration:
+			return []string{"above", "max_duration"}
+		case CreditBelowNonMaxDuration:
+			return []string{"below", "non_max_duration"}
+		case CreditEqualNonMaxDuration:
+			return []string{"expected", "non_max_duration"}
+		case CreditAboveNonMaxDuration:
+			return []string{"above", "non_max_duration"}
+		default:
+			panic(fmt.Errorf("unknown credit expectation: %v", expectation))
+		}
+	}
+	m.credits.WithLabelValues(asLabels(expectation)...).Set(float64(count))
+}
+
 func (m *Metrics) RecordClaims(status ClaimStatus, count int) {
 	asLabels := func(status ClaimStatus) []string {
 		switch status {

op-dispute-mon/metrics/noop.go

@@ -19,6 +19,8 @@ func (*NoopMetricsImpl) RecordUp()           {}
 func (*NoopMetricsImpl) CacheAdd(_ string, _ int, _ bool) {}
 func (*NoopMetricsImpl) CacheGet(_ string, _ bool)        {}
 
+func (*NoopMetricsImpl) RecordCredit(_ CreditExpectation, _ int) {}
+
 func (*NoopMetricsImpl) RecordClaims(_ ClaimStatus, _ int) {}
 
 func (*NoopMetricsImpl) RecordWithdrawalRequests(_ common.Address, _ bool, _ int) {}

op-dispute-mon/mon/bonds/monitor.go

@@ -2,24 +2,33 @@ package bonds
 
 import (
 	"math/big"
+	"time"
 
+	"github.com/ethereum-optimism/optimism/op-dispute-mon/metrics"
 	"github.com/ethereum-optimism/optimism/op-dispute-mon/mon/types"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 )
 
+type RClock interface {
+	Now() time.Time
+}
+
 type BondMetrics interface {
+	RecordCredit(expectation metrics.CreditExpectation, count int)
 	RecordBondCollateral(addr common.Address, required *big.Int, available *big.Int)
 }
 
 type Bonds struct {
 	logger  log.Logger
+	clock   RClock
 	metrics BondMetrics
 }
 
-func NewBonds(logger log.Logger, metrics BondMetrics) *Bonds {
+func NewBonds(logger log.Logger, metrics BondMetrics, clock RClock) *Bonds {
 	return &Bonds{
 		logger:  logger,
+		clock:   clock,
 		metrics: metrics,
 	}
 }
@@ -29,4 +38,62 @@ func (b *Bonds) CheckBonds(games []*types.EnrichedGameData) {
 	for addr, collateral := range data {
 		b.metrics.RecordBondCollateral(addr, collateral.Required, collateral.Actual)
 	}
+
+	for _, game := range games {
+		b.checkCredits(game)
+	}
+}
+
+func (b *Bonds) checkCredits(game *types.EnrichedGameData) {
+	// Check if the max duration has been reached for this game
+	duration := uint64(b.clock.Now().Unix()) - game.Timestamp
+	maxDurationReached := duration >= game.Duration
+
+	// Iterate over claims and filter out resolved ones
+	recipients := make(map[int]common.Address)
+	for i, claim := range game.Claims {
+		// Skip unresolved claims since these bonds will not appear in the credits.
+		if !claim.Resolved {
+			continue
+		}
+		// The recipient of a resolved claim is the claimant unless it's been countered.
+		recipient := claim.Claimant
+		if claim.CounteredBy != (common.Address{}) {
+			recipient = claim.CounteredBy
+		}
+		recipients[i] = recipient
+	}
+
+	creditMetrics := make(map[metrics.CreditExpectation]int)
+	for i, recipient := range recipients {
+		expected := game.Credits[recipient]
+		comparison := expected.Cmp(game.RequiredBonds[i])
+		if maxDurationReached {
+			if comparison > 0 {
+				creditMetrics[metrics.CreditBelowMaxDuration] += 1
+			} else if comparison == 0 {
+				creditMetrics[metrics.CreditEqualMaxDuration] += 1
+			} else {
+				creditMetrics[metrics.CreditAboveMaxDuration] += 1
+				b.logger.Warn("credit above expected amount", "recipient", recipient, "expected", expected, "gameAddr", game.Proxy, "duration", "max_duration")
+			}
+		} else {
+			if comparison > 0 {
+				creditMetrics[metrics.CreditBelowNonMaxDuration] += 1
+			} else if comparison == 0 {
+				creditMetrics[metrics.CreditEqualNonMaxDuration] += 1
+			} else {
+				creditMetrics[metrics.CreditAboveNonMaxDuration] += 1
+				b.logger.Warn("credit above expected amount", "recipient", recipient, "expected", expected, "gameAddr", game.Proxy, "duration", "non_max_duration")
+			}
+		}
+	}
+
+	b.metrics.RecordCredit(metrics.CreditBelowMaxDuration, creditMetrics[metrics.CreditBelowMaxDuration])
+	b.metrics.RecordCredit(metrics.CreditEqualMaxDuration, creditMetrics[metrics.CreditEqualMaxDuration])
+	b.metrics.RecordCredit(metrics.CreditAboveMaxDuration, creditMetrics[metrics.CreditAboveMaxDuration])
+
+	b.metrics.RecordCredit(metrics.CreditBelowNonMaxDuration, creditMetrics[metrics.CreditBelowNonMaxDuration])
+	b.metrics.RecordCredit(metrics.CreditEqualNonMaxDuration, creditMetrics[metrics.CreditEqualNonMaxDuration])
+	b.metrics.RecordCredit(metrics.CreditAboveNonMaxDuration, creditMetrics[metrics.CreditAboveNonMaxDuration])
 }

op-dispute-mon/mon/bonds/monitor_test.go

@@ -3,14 +3,21 @@ package bonds
 import (
 	"math/big"
 	"testing"
+	"time"
 
+	"github.com/ethereum-optimism/optimism/op-dispute-mon/metrics"
 	monTypes "github.com/ethereum-optimism/optimism/op-dispute-mon/mon/types"
+	"github.com/ethereum-optimism/optimism/op-service/clock"
 	"github.com/ethereum-optimism/optimism/op-service/testlog"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/stretchr/testify/require"
 )
 
+var (
+	frozen = time.Unix(int64(time.Hour.Seconds()), 0)
+)
+
 func TestCheckBonds(t *testing.T) {
 	weth1 := common.Address{0x1a}
 	weth1Balance := big.NewInt(4200)
@@ -32,8 +39,11 @@ func TestCheckBonds(t *testing.T) {
 	}
 
 	logger := testlog.Logger(t, log.LvlInfo)
-	metrics := &stubBondMetrics{recorded: make(map[common.Address]Collateral)}
-	bonds := NewBonds(logger, metrics)
+	metrics := &stubBondMetrics{
+		credits:  make(map[metrics.CreditExpectation]int),
+		recorded: make(map[common.Address]Collateral),
+	}
+	bonds := NewBonds(logger, metrics, clock.NewDeterministicClock(frozen))
 
 	bonds.CheckBonds([]*monTypes.EnrichedGameData{game1, game2})
 
@@ -47,6 +57,7 @@ func TestCheckBonds(t *testing.T) {
 }
 
 type stubBondMetrics struct {
+	credits  map[metrics.CreditExpectation]int
 	recorded map[common.Address]Collateral
 }
 
@@ -56,3 +67,7 @@ func (s *stubBondMetrics) RecordBondCollateral(addr common.Address, required *bi
 		Actual:   available,
 	}
 }
+
+func (s *stubBondMetrics) RecordCredit(expectation metrics.CreditExpectation, count int) {
+	s.credits[expectation] = count
+}

op-dispute-mon/mon/extract/bond_enricher.go

@@ -18,6 +18,7 @@ var ErrIncorrectCreditCount = errors.New("incorrect credit count")
 
 type BondCaller interface {
 	GetCredits(context.Context, rpcblock.Block, ...common.Address) ([]*big.Int, error)
+	GetRequiredBonds(context.Context, rpcblock.Block, ...*big.Int) ([]*big.Int, error)
 }
 
 type BondEnricher struct{}
@@ -27,8 +28,26 @@ func NewBondEnricher() *BondEnricher {
 }
 
 func (b *BondEnricher) Enrich(ctx context.Context, block rpcblock.Block, caller GameCaller, game *monTypes.EnrichedGameData) error {
-	recipients := maps.Keys(game.Recipients)
-	credits, err := caller.GetCredits(ctx, block, recipients...)
+	if err := b.enrichCredits(ctx, block, caller, game); err != nil {
+		return err
+	}
+	if err := b.enrichRequiredBonds(ctx, block, caller, game); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (b *BondEnricher) enrichCredits(ctx context.Context, block rpcblock.Block, caller GameCaller, game *monTypes.EnrichedGameData) error {
+	recipients := make(map[common.Address]bool)
+	for _, claim := range game.Claims {
+		if claim.CounteredBy != (common.Address{}) {
+			recipients[claim.CounteredBy] = true
+		} else {
+			recipients[claim.Claimant] = true
+		}
+	}
+	recipientAddrs := maps.Keys(recipients)
+	credits, err := caller.GetCredits(ctx, block, recipientAddrs...)
 	if err != nil {
 		return err
 	}
@@ -37,7 +56,37 @@ func (b *BondEnricher) Enrich(ctx context.Context, block rpcblock.Block, caller
 	}
 	game.Credits = make(map[common.Address]*big.Int)
 	for i, credit := range credits {
-		game.Credits[recipients[i]] = credit
+		game.Credits[recipientAddrs[i]] = credit
+	}
+	return nil
+}
+
+func (b *BondEnricher) enrichRequiredBonds(ctx context.Context, block rpcblock.Block, caller GameCaller, game *monTypes.EnrichedGameData) error {
+	positions := make([]*big.Int, len(game.Claims))
+	for _, claim := range game.Claims {
+		// If the claim is not resolved, we don't need to get the bond
+		// for it since the Bond field in the claim will be accurate.
+		if !claim.Resolved {
+			continue
+		}
+		positions = append(positions, claim.Position.ToGIndex())
+	}
+	bonds, err := caller.GetRequiredBonds(ctx, block, positions...)
+	if err != nil {
+		return err
+	}
+	if len(bonds) != len(positions) {
+		return fmt.Errorf("%w, requested %v values but got %v", ErrIncorrectCreditCount, len(positions), len(bonds))
+	}
+	game.RequiredBonds = make(map[int]*big.Int)
+	bondIndex := 0
+	for i, claim := range game.Claims {
+		if !claim.Resolved {
+			game.RequiredBonds[i] = claim.Bond
+			continue
+		}
+		game.RequiredBonds[i] = bonds[bondIndex]
+		bondIndex++
 	}
 	return nil
 }

op-dispute-mon/mon/extract/bond_enricher_test.go

@@ -82,7 +82,13 @@ func TestBondEnricher(t *testing.T) {
 			recipients[1]: big.NewInt(30),
 			recipients[2]: big.NewInt(40),
 		}
-		caller := &mockGameCaller{credits: expectedCredits}
+		requiredBonds := []*big.Int{
+			big.NewInt(10),
+			big.NewInt(20),
+			big.NewInt(30),
+			big.NewInt(40),
+		}
+		caller := &mockGameCaller{credits: expectedCredits, requiredBonds: requiredBonds}
 		err := enricher.Enrich(context.Background(), rpcblock.Latest, caller, game)
 		require.NoError(t, err)
 

op-dispute-mon/mon/extract/extractor_test.go

@@ -190,11 +190,22 @@ type mockGameCaller struct {
 	balanceErr        error
 	balance           *big.Int
 	balanceAddr       common.Address
+	requiredBondCalls int
+	requiredBondErr   error
+	requiredBonds     []*big.Int
 	withdrawalsCalls  int
 	withdrawalsErr    error
 	withdrawals       []*contracts.WithdrawalRequest
 }
 
+func (m *mockGameCaller) GetRequiredBonds(ctx context.Context, block rpcblock.Block, positions ...*big.Int) ([]*big.Int, error) {
+	m.requiredBondCalls++
+	if m.requiredBondErr != nil {
+		return nil, m.requiredBondErr
+	}
+	return m.requiredBonds, nil
+}
+
 func (m *mockGameCaller) GetWithdrawals(_ context.Context, _ rpcblock.Block, _ common.Address, _ ...common.Address) ([]*contracts.WithdrawalRequest, error) {
 	m.withdrawalsCalls++
 	if m.withdrawalsErr != nil {

op-dispute-mon/mon/service.go

@@ -146,7 +146,7 @@ func (s *Service) initForecast(cfg *config.Config) {
 }
 
 func (s *Service) initBonds() {
-	s.bonds = bonds.NewBonds(s.logger, s.metrics)
+	s.bonds = bonds.NewBonds(s.logger, s.metrics, s.cl)
 }
 
 func (s *Service) initOutputRollupClient(ctx context.Context, cfg *config.Config) error {

op-dispute-mon/mon/types/types.go

@@ -31,6 +31,11 @@ type EnrichedGameData struct {
 	// Credits records the paid out bonds for the game, keyed by recipient.
 	Credits map[common.Address]*big.Int
 
+	// RequiredBonds maps *resolved* claim indices to their required bond amounts.
+	// Required bonds are not needed for unresolved claims since
+	// the `Bond` field in the claim is the required bond amount.
+	RequiredBonds map[int]*big.Int
+
 	// WithdrawalRequests maps recipients with withdrawal requests in DelayedWETH for this game.
 	WithdrawalRequests map[common.Address]*contracts.WithdrawalRequest
 
@@ -39,7 +44,8 @@ type EnrichedGameData struct {
 	WETHContract common.Address
 
 	// ETHCollateral is the ETH balance of the (potentially shared) WETHContract
-	// This ETH balance will be used to pay out any bonds required by the games that use the same DelayedWETH contract.
+	// This ETH balance will be used to pay out any bonds required by the games
+	// that use the same DelayedWETH contract.
 	ETHCollateral *big.Int
 }