feat(op-dispute-mon): Resolution Status Monitor Component (#10020)

* feat(op-dispute-mon): resolution status

* fix(op-dispute-mon): merge fixes

op-dispute-mon/metrics/metrics.go

@@ -69,6 +69,8 @@ type Metricer interface {
 	RecordInfo(version string)
 	RecordUp()
 
+	RecordGameResolutionStatus(complete bool, maxDurationReached bool, count int)
+
 	RecordCredit(expectation CreditExpectation, count int)
 
 	RecordClaims(status ClaimStatus, count int)
@@ -98,6 +100,8 @@ type Metrics struct {
 	*opmetrics.CacheMetrics
 	*contractMetrics.ContractMetrics
 
+	resolutionStatus prometheus.GaugeVec
+
 	claims prometheus.GaugeVec
 
 	withdrawalRequests prometheus.GaugeVec
@@ -157,6 +161,14 @@ func NewMetrics() *Metrics {
 			Name:      "claim_resolution_delay_max",
 			Help:      "Maximum claim resolution delay in seconds",
 		}),
+		resolutionStatus: *factory.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Name:      "resolution_status",
+			Help:      "Number of games categorised by whether the game is complete and whether the maximum duration has been reached",
+		}, []string{
+			"completion",
+			"max_duration",
+		}),
 		credits: *factory.NewGaugeVec(prometheus.GaugeOpts{
 			Namespace: Namespace,
 			Name:      "credits",
@@ -239,6 +251,18 @@ func (m *Metrics) RecordUp() {
 	m.up.Set(1)
 }
 
+func (m *Metrics) RecordGameResolutionStatus(complete bool, maxDurationReached bool, count int) {
+	completion := "complete"
+	if !complete {
+		completion = "in_progress"
+	}
+	maxDuration := "reached"
+	if !maxDurationReached {
+		maxDuration = "not_reached"
+	}
+	m.resolutionStatus.WithLabelValues(completion, maxDuration).Set(float64(count))
+}
+
 func (m *Metrics) RecordCredit(expectation CreditExpectation, count int) {
 	asLabels := func(expectation CreditExpectation) []string {
 		switch expectation {

op-dispute-mon/metrics/noop.go

@@ -19,6 +19,8 @@ func (*NoopMetricsImpl) RecordUp()           {}
 func (*NoopMetricsImpl) CacheAdd(_ string, _ int, _ bool) {}
 func (*NoopMetricsImpl) CacheGet(_ string, _ bool)        {}
 
+func (*NoopMetricsImpl) RecordGameResolutionStatus(_ bool, _ bool, _ int) {}
+
 func (*NoopMetricsImpl) RecordCredit(_ CreditExpectation, _ int) {}
 
 func (*NoopMetricsImpl) RecordClaims(_ ClaimStatus, _ int) {}

op-dispute-mon/mon/monitor.go

@@ -15,6 +15,7 @@ import (
 
 type Forecast func(ctx context.Context, games []*types.EnrichedGameData)
 type Bonds func(games []*types.EnrichedGameData)
+type Resolutions func(games []*types.EnrichedGameData)
 type MonitorClaims func(games []*types.EnrichedGameData)
 type MonitorWithdrawals func(games []*types.EnrichedGameData)
 type BlockHashFetcher func(ctx context.Context, number *big.Int) (common.Hash, error)
@@ -36,6 +37,7 @@ type gameMonitor struct {
 	delays           RecordClaimResolutionDelayMax
 	forecast         Forecast
 	bonds            Bonds
+	resolutions      Resolutions
 	claims           MonitorClaims
 	withdrawals      MonitorWithdrawals
 	extract          Extract
@@ -52,6 +54,7 @@ func newGameMonitor(
 	delays RecordClaimResolutionDelayMax,
 	forecast Forecast,
 	bonds Bonds,
+	resolutions Resolutions,
 	claims MonitorClaims,
 	withdrawals MonitorWithdrawals,
 	extract Extract,
@@ -68,6 +71,7 @@ func newGameMonitor(
 		delays:           delays,
 		forecast:         forecast,
 		bonds:            bonds,
+		resolutions:      resolutions,
 		claims:           claims,
 		withdrawals:      withdrawals,
 		extract:          extract,
@@ -91,6 +95,7 @@ func (m *gameMonitor) monitorGames() error {
 	if err != nil {
 		return fmt.Errorf("failed to load games: %w", err)
 	}
+	m.resolutions(enrichedGames)
 	m.delays(enrichedGames)
 	m.forecast(m.ctx, enrichedGames)
 	m.bonds(enrichedGames)

op-dispute-mon/mon/monitor_test.go

@@ -24,7 +24,7 @@ func TestMonitor_MonitorGames(t *testing.T) {
 	t.Parallel()
 
 	t.Run("FailedFetchBlocknumber", func(t *testing.T) {
-		monitor, _, _, _, _, _, _ := setupMonitorTest(t)
+		monitor, _, _, _, _, _, _, _ := setupMonitorTest(t)
 		boom := errors.New("boom")
 		monitor.fetchBlockNumber = func(ctx context.Context) (uint64, error) {
 			return 0, boom
@@ -34,7 +34,7 @@ func TestMonitor_MonitorGames(t *testing.T) {
 	})
 
 	t.Run("FailedFetchBlockHash", func(t *testing.T) {
-		monitor, _, _, _, _, _, _ := setupMonitorTest(t)
+		monitor, _, _, _, _, _, _, _ := setupMonitorTest(t)
 		boom := errors.New("boom")
 		monitor.fetchBlockHash = func(ctx context.Context, number *big.Int) (common.Hash, error) {
 			return common.Hash{}, boom
@@ -44,25 +44,27 @@ func TestMonitor_MonitorGames(t *testing.T) {
 	})
 
 	t.Run("MonitorsWithNoGames", func(t *testing.T) {
-		monitor, factory, forecast, delays, bonds, withdrawals, claims := setupMonitorTest(t)
+		monitor, factory, forecast, delays, bonds, withdrawals, resolutions, claims := setupMonitorTest(t)
 		factory.games = []*monTypes.EnrichedGameData{}
 		err := monitor.monitorGames()
 		require.NoError(t, err)
 		require.Equal(t, 1, forecast.calls)
 		require.Equal(t, 1, delays.calls)
 		require.Equal(t, 1, bonds.calls)
+		require.Equal(t, 1, resolutions.calls)
 		require.Equal(t, 1, claims.calls)
 		require.Equal(t, 1, withdrawals.calls)
 	})
 
 	t.Run("MonitorsMultipleGames", func(t *testing.T) {
-		monitor, factory, forecast, delays, bonds, withdrawals, claims := setupMonitorTest(t)
+		monitor, factory, forecast, delays, bonds, withdrawals, resolutions, claims := setupMonitorTest(t)
 		factory.games = []*monTypes.EnrichedGameData{{}, {}, {}}
 		err := monitor.monitorGames()
 		require.NoError(t, err)
 		require.Equal(t, 1, forecast.calls)
 		require.Equal(t, 1, delays.calls)
 		require.Equal(t, 1, bonds.calls)
+		require.Equal(t, 1, resolutions.calls)
 		require.Equal(t, 1, claims.calls)
 		require.Equal(t, 1, withdrawals.calls)
 	})
@@ -72,7 +74,7 @@ func TestMonitor_StartMonitoring(t *testing.T) {
 	t.Run("MonitorsGames", func(t *testing.T) {
 		addr1 := common.Address{0xaa}
 		addr2 := common.Address{0xbb}
-		monitor, factory, forecaster, _, _, _, _ := setupMonitorTest(t)
+		monitor, factory, forecaster, _, _, _, _, _ := setupMonitorTest(t)
 		factory.games = []*monTypes.EnrichedGameData{newEnrichedGameData(addr1, 9999), newEnrichedGameData(addr2, 9999)}
 		factory.maxSuccess = len(factory.games) // Only allow two successful fetches
 
@@ -85,7 +87,7 @@ func TestMonitor_StartMonitoring(t *testing.T) {
 	})
 
 	t.Run("FailsToFetchGames", func(t *testing.T) {
-		monitor, factory, forecaster, _, _, _, _ := setupMonitorTest(t)
+		monitor, factory, forecaster, _, _, _, _, _ := setupMonitorTest(t)
 		factory.fetchErr = errors.New("boom")
 
 		monitor.StartMonitoring()
@@ -107,7 +109,7 @@ func newEnrichedGameData(proxy common.Address, timestamp uint64) *monTypes.Enric
 	}
 }
 
-func setupMonitorTest(t *testing.T) (*gameMonitor, *mockExtractor, *mockForecast, *mockDelayCalculator, *mockBonds, *mockWithdrawalMonitor, *mockClaimMonitor) {
+func setupMonitorTest(t *testing.T) (*gameMonitor, *mockExtractor, *mockForecast, *mockDelayCalculator, *mockBonds, *mockWithdrawalMonitor, *mockResolutionMonitor, *mockClaimMonitor) {
 	logger := testlog.Logger(t, log.LvlDebug)
 	fetchBlockNum := func(ctx context.Context) (uint64, error) {
 		return 1, nil
@@ -121,6 +123,7 @@ func setupMonitorTest(t *testing.T) (*gameMonitor, *mockExtractor, *mockForecast
 	extractor := &mockExtractor{}
 	forecast := &mockForecast{}
 	bonds := &mockBonds{}
+	resolutions := &mockResolutionMonitor{}
 	claims := &mockClaimMonitor{}
 	withdrawals := &mockWithdrawalMonitor{}
 	delays := &mockDelayCalculator{}
@@ -133,13 +136,22 @@ func setupMonitorTest(t *testing.T) (*gameMonitor, *mockExtractor, *mockForecast
 		delays.RecordClaimResolutionDelayMax,
 		forecast.Forecast,
 		bonds.CheckBonds,
+		resolutions.CheckResolutions,
 		claims.CheckClaims,
 		withdrawals.CheckWithdrawals,
 		extractor.Extract,
 		fetchBlockNum,
 		fetchBlockHash,
 	)
-	return monitor, extractor, forecast, delays, bonds, withdrawals, claims
+	return monitor, extractor, forecast, delays, bonds, withdrawals, resolutions, claims
+}
+
+type mockResolutionMonitor struct {
+	calls int
+}
+
+func (m *mockResolutionMonitor) CheckResolutions(games []*monTypes.EnrichedGameData) {
+	m.calls++
 }
 
 type mockClaimMonitor struct {

op-dispute-mon/mon/resolutions.go

+package mon
+
+import (
+	gameTypes "github.com/ethereum-optimism/optimism/op-challenger/game/types"
+	"github.com/ethereum-optimism/optimism/op-dispute-mon/mon/types"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+type ResolutionMetrics interface {
+	RecordGameResolutionStatus(complete bool, maxDurationReached bool, count int)
+}
+
+type ResolutionMonitor struct {
+	logger  log.Logger
+	clock   RClock
+	metrics ResolutionMetrics
+}
+
+func NewResolutionMonitor(logger log.Logger, metrics ResolutionMetrics, clock RClock) *ResolutionMonitor {
+	return &ResolutionMonitor{
+		logger:  logger,
+		clock:   clock,
+		metrics: metrics,
+	}
+}
+
+type resolutionStatus struct {
+	completeMaxDuration         int
+	completeBeforeMaxDuration   int
+	inProgressMaxDuration       int
+	inProgressBeforeMaxDuration int
+}
+
+func (r *resolutionStatus) Inc(complete, maxDuration bool) {
+	if complete {
+		if maxDuration {
+			r.completeMaxDuration++
+		} else {
+			r.completeBeforeMaxDuration++
+		}
+	} else {
+		if maxDuration {
+			r.inProgressMaxDuration++
+		} else {
+			r.inProgressBeforeMaxDuration++
+		}
+	}
+}
+
+func (r *ResolutionMonitor) CheckResolutions(games []*types.EnrichedGameData) {
+	status := &resolutionStatus{}
+	for _, game := range games {
+		complete := game.Status != gameTypes.GameStatusInProgress
+		duration := uint64(r.clock.Now().Unix()) - game.Timestamp
+		maxDurationReached := duration >= game.Duration
+		status.Inc(complete, maxDurationReached)
+	}
+	r.metrics.RecordGameResolutionStatus(true, true, status.completeMaxDuration)
+	r.metrics.RecordGameResolutionStatus(true, false, status.completeBeforeMaxDuration)
+	r.metrics.RecordGameResolutionStatus(false, true, status.inProgressMaxDuration)
+	r.metrics.RecordGameResolutionStatus(false, false, status.inProgressBeforeMaxDuration)
+}

op-dispute-mon/mon/resolutions_test.go

+package mon
+
+import (
+	"testing"
+	"time"
+
+	gameTypes "github.com/ethereum-optimism/optimism/op-challenger/game/types"
+	"github.com/ethereum-optimism/optimism/op-dispute-mon/mon/types"
+	"github.com/ethereum-optimism/optimism/op-service/clock"
+	"github.com/ethereum-optimism/optimism/op-service/testlog"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolutionMonitor_CheckResolutions(t *testing.T) {
+	r, cl, m := newTestResolutionMonitor(t)
+	games := newTestGames(uint64(cl.Now().Unix()))
+	r.CheckResolutions(games)
+
+	require.Equal(t, 1, m.calls[true][true])
+	require.Equal(t, 1, m.calls[true][false])
+	require.Equal(t, 1, m.calls[false][true])
+	require.Equal(t, 1, m.calls[false][false])
+}
+
+func newTestResolutionMonitor(t *testing.T) (*ResolutionMonitor, *clock.DeterministicClock, *stubResolutionMetrics) {
+	logger := testlog.Logger(t, log.LvlInfo)
+	cl := clock.NewDeterministicClock(time.Unix(int64(time.Hour.Seconds()), 0))
+	metrics := &stubResolutionMetrics{}
+	return NewResolutionMonitor(logger, metrics, cl), cl, metrics
+}
+
+type stubResolutionMetrics struct {
+	calls map[bool]map[bool]int // completed -> max duration reached -> call count
+}
+
+func (s *stubResolutionMetrics) RecordGameResolutionStatus(complete bool, maxDurationReached bool, count int) {
+	if s.calls == nil {
+		s.calls = make(map[bool]map[bool]int)
+		s.calls[true] = make(map[bool]int)
+		s.calls[false] = make(map[bool]int)
+	}
+	s.calls[complete][maxDurationReached] += count
+}
+
+func newTestGames(duration uint64) []*types.EnrichedGameData {
+	newTestGame := func(duration uint64, status gameTypes.GameStatus) *types.EnrichedGameData {
+		return &types.EnrichedGameData{Duration: duration, Status: status}
+	}
+	return []*types.EnrichedGameData{
+		newTestGame(duration, gameTypes.GameStatusInProgress),
+		newTestGame(duration*10, gameTypes.GameStatusInProgress),
+		newTestGame(duration, gameTypes.GameStatusDefenderWon),
+		newTestGame(duration*10, gameTypes.GameStatusChallengerWon),
+	}
+}

op-dispute-mon/mon/service.go

@@ -42,6 +42,7 @@ type Service struct {
 	forecast     *forecast
 	bonds        *bonds.Bonds
 	game         *extract.GameCallerCreator
+	resolutions  *ResolutionMonitor
 	claims       *ClaimMonitor
 	withdrawals  *WithdrawalMonitor
 	rollupClient *sources.RollupClient
@@ -87,6 +88,7 @@ func (s *Service) initFromConfig(ctx context.Context, cfg *config.Config) error
 		return fmt.Errorf("failed to init rollup client: %w", err)
 	}
 
+	s.initResolutionMonitor()
 	s.initClaimMonitor()
 	s.initWithdrawalMonitor()
 
@@ -107,6 +109,10 @@ func (s *Service) initFromConfig(ctx context.Context, cfg *config.Config) error
 	return nil
 }
 
+func (s *Service) initResolutionMonitor() {
+	s.resolutions = NewResolutionMonitor(s.logger, s.metrics, s.cl)
+}
+
 func (s *Service) initClaimMonitor() {
 	s.claims = NewClaimMonitor(s.logger, s.cl, s.metrics)
 }
@@ -229,6 +235,7 @@ func (s *Service) initMonitor(ctx context.Context, cfg *config.Config) {
 		s.delays.RecordClaimResolutionDelayMax,
 		s.forecast.Forecast,
 		s.bonds.CheckBonds,
+		s.resolutions.CheckResolutions,
 		s.claims.CheckClaims,
 		s.withdrawals.CheckWithdrawals,
 		s.extractor.Extract,