fix(op-dispute-mon): Improve Resolution Status Granularity (#10419)

* stash resolution status

* fix(op-dispute-mon): resolution status metric

* fix(op-dispute-mon): add logs for resolvable games

op-dispute-mon/metrics/metrics.go

@@ -19,6 +19,22 @@ import (
 
 const Namespace = "op_dispute_mon"
 
+type ResolutionStatus uint8
+
+const (
+	// In progress
+	CompleteMaxDuration ResolutionStatus = iota
+	CompleteBeforeMaxDuration
+
+	// Resolvable
+	ResolvableMaxDuration
+	ResolvableBeforeMaxDuration
+
+	// Not resolvable
+	InProgressMaxDuration
+	InProgressBeforeMaxDuration
+)
+
 type CreditExpectation uint8
 
 const (
@@ -93,7 +109,7 @@ type Metricer interface {
 
 	RecordHonestActorClaims(address common.Address, stats *HonestActorData)
 
-	RecordGameResolutionStatus(complete bool, maxDurationReached bool, count int)
+	RecordGameResolutionStatus(status ResolutionStatus, count int)
 
 	RecordCredit(expectation CreditExpectation, count int)
 
@@ -311,16 +327,26 @@ func (m *Metrics) RecordHonestActorClaims(address common.Address, stats *HonestA
 	m.honestActorBonds.WithLabelValues(address.Hex(), "won").Set(weiToEther(stats.WonBonds))
 }
 
-func (m *Metrics) RecordGameResolutionStatus(complete bool, maxDurationReached bool, count int) {
-	completion := "complete"
-	if !complete {
-		completion = "in_progress"
+func (m *Metrics) RecordGameResolutionStatus(status ResolutionStatus, count int) {
+	asLabels := func(status ResolutionStatus) []string {
+		switch status {
+		case CompleteMaxDuration:
+			return []string{"complete", "max_duration"}
+		case CompleteBeforeMaxDuration:
+			return []string{"complete", "before_max_duration"}
+		case ResolvableMaxDuration:
+			return []string{"resolvable", "max_duration"}
+		case ResolvableBeforeMaxDuration:
+			return []string{"resolvable", "before_max_duration"}
+		case InProgressMaxDuration:
+			return []string{"in_progress", "max_duration"}
+		case InProgressBeforeMaxDuration:
+			return []string{"in_progress", "before_max_duration"}
+		default:
+			panic(fmt.Errorf("unknown resolution status: %v", status))
 		}
-	maxDuration := "reached"
-	if !maxDurationReached {
-		maxDuration = "not_reached"
 	}
-	m.resolutionStatus.WithLabelValues(completion, maxDuration).Set(float64(count))
+	m.resolutionStatus.WithLabelValues(asLabels(status)...).Set(float64(count))
 }
 
 func (m *Metrics) RecordCredit(expectation CreditExpectation, count int) {

op-dispute-mon/metrics/noop.go

@@ -21,7 +21,7 @@ func (*NoopMetricsImpl) CacheGet(_ string, _ bool)        {}
 
 func (*NoopMetricsImpl) RecordHonestActorClaims(_ common.Address, _ *HonestActorData) {}
 
-func (*NoopMetricsImpl) RecordGameResolutionStatus(_ bool, _ bool, _ int) {}
+func (*NoopMetricsImpl) RecordGameResolutionStatus(_ ResolutionStatus, _ int) {}
 
 func (*NoopMetricsImpl) RecordCredit(_ CreditExpectation, _ int) {}
 

op-dispute-mon/mon/resolutions.go

 package mon
 
 import (
+	"time"
+
 	gameTypes "github.com/ethereum-optimism/optimism/op-challenger/game/types"
+	"github.com/ethereum-optimism/optimism/op-dispute-mon/metrics"
 	"github.com/ethereum-optimism/optimism/op-dispute-mon/mon/types"
 	"github.com/ethereum/go-ethereum/log"
 )
 
+const MaxResolveDelay = time.Minute
+
 type ResolutionMetrics interface {
-	RecordGameResolutionStatus(complete bool, maxDurationReached bool, count int)
+	RecordGameResolutionStatus(status metrics.ResolutionStatus, count int)
 }
 
 type ResolutionMonitor struct {
@@ -24,39 +29,48 @@ func NewResolutionMonitor(logger log.Logger, metrics ResolutionMetrics, clock RC
 	}
 }
 
-type resolutionStatus struct {
-	completeMaxDuration         int
-	completeBeforeMaxDuration   int
-	inProgressMaxDuration       int
-	inProgressBeforeMaxDuration int
-}
-
-func (r *resolutionStatus) Inc(complete, maxDuration bool) {
+func (r *ResolutionMonitor) CheckResolutions(games []*types.EnrichedGameData) {
+	statusMetrics := make(map[metrics.ResolutionStatus]int)
+	for _, game := range games {
+		complete := game.Status != gameTypes.GameStatusInProgress
+		duration := uint64(r.clock.Now().Unix()) - game.Timestamp
+		maxDurationReached := duration >= (2 * game.MaxClockDuration)
+		resolvable := true
+		for _, claim := range game.Claims {
+			// If any claim is not resolved, the game is not resolvable
+			resolvable = resolvable && claim.Resolved
+		}
 		if complete {
-		if maxDuration {
-			r.completeMaxDuration++
+			if maxDurationReached {
+				statusMetrics[metrics.CompleteMaxDuration]++
 			} else {
-			r.completeBeforeMaxDuration++
+				statusMetrics[metrics.CompleteBeforeMaxDuration]++
+			}
+		} else if resolvable {
+			if maxDurationReached {
+				// SAFETY: since maxDurationReached is true, this cannot underflow
+				delay := duration - (2 * game.MaxClockDuration)
+				if delay > uint64(MaxResolveDelay.Seconds()) {
+					r.logger.Warn("Resolvable game has taken too long to resolve", "game", game.Proxy, "delay", delay)
 				}
+				statusMetrics[metrics.ResolvableMaxDuration]++
 			} else {
-		if maxDuration {
-			r.inProgressMaxDuration++
+				statusMetrics[metrics.ResolvableBeforeMaxDuration]++
+			}
+		} else {
+			if maxDurationReached {
+				// Note: we don't need to log here since unresolved claims are logged and metriced in claims.go
+				statusMetrics[metrics.InProgressMaxDuration]++
 			} else {
-			r.inProgressBeforeMaxDuration++
+				statusMetrics[metrics.InProgressBeforeMaxDuration]++
 			}
 		}
-}
-
-func (r *ResolutionMonitor) CheckResolutions(games []*types.EnrichedGameData) {
-	status := &resolutionStatus{}
-	for _, game := range games {
-		complete := game.Status != gameTypes.GameStatusInProgress
-		duration := uint64(r.clock.Now().Unix()) - game.Timestamp
-		maxDurationReached := duration >= game.MaxClockDuration
-		status.Inc(complete, maxDurationReached)
 	}
-	r.metrics.RecordGameResolutionStatus(true, true, status.completeMaxDuration)
-	r.metrics.RecordGameResolutionStatus(true, false, status.completeBeforeMaxDuration)
-	r.metrics.RecordGameResolutionStatus(false, true, status.inProgressMaxDuration)
-	r.metrics.RecordGameResolutionStatus(false, false, status.inProgressBeforeMaxDuration)
+
+	r.metrics.RecordGameResolutionStatus(metrics.CompleteMaxDuration, statusMetrics[metrics.CompleteMaxDuration])
+	r.metrics.RecordGameResolutionStatus(metrics.CompleteBeforeMaxDuration, statusMetrics[metrics.CompleteBeforeMaxDuration])
+	r.metrics.RecordGameResolutionStatus(metrics.ResolvableMaxDuration, statusMetrics[metrics.ResolvableMaxDuration])
+	r.metrics.RecordGameResolutionStatus(metrics.ResolvableBeforeMaxDuration, statusMetrics[metrics.ResolvableBeforeMaxDuration])
+	r.metrics.RecordGameResolutionStatus(metrics.InProgressMaxDuration, statusMetrics[metrics.InProgressMaxDuration])
+	r.metrics.RecordGameResolutionStatus(metrics.InProgressBeforeMaxDuration, statusMetrics[metrics.InProgressBeforeMaxDuration])
 }

op-dispute-mon/mon/resolutions_test.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	gameTypes "github.com/ethereum-optimism/optimism/op-challenger/game/types"
+	"github.com/ethereum-optimism/optimism/op-dispute-mon/metrics"
 	"github.com/ethereum-optimism/optimism/op-dispute-mon/mon/types"
 	"github.com/ethereum-optimism/optimism/op-service/clock"
 	"github.com/ethereum-optimism/optimism/op-service/testlog"
@@ -17,10 +18,12 @@ func TestResolutionMonitor_CheckResolutions(t *testing.T) {
 	games := newTestGames(uint64(cl.Now().Unix()))
 	r.CheckResolutions(games)
 
-	require.Equal(t, 1, m.calls[true][true])
-	require.Equal(t, 1, m.calls[true][false])
-	require.Equal(t, 1, m.calls[false][true])
-	require.Equal(t, 1, m.calls[false][false])
+	require.Equal(t, 1, m.calls[metrics.CompleteMaxDuration])
+	require.Equal(t, 1, m.calls[metrics.CompleteBeforeMaxDuration])
+	require.Equal(t, 1, m.calls[metrics.ResolvableMaxDuration])
+	require.Equal(t, 1, m.calls[metrics.ResolvableBeforeMaxDuration])
+	require.Equal(t, 1, m.calls[metrics.InProgressMaxDuration])
+	require.Equal(t, 1, m.calls[metrics.InProgressBeforeMaxDuration])
 }
 
 func newTestResolutionMonitor(t *testing.T) (*ResolutionMonitor, *clock.DeterministicClock, *stubResolutionMetrics) {
@@ -31,26 +34,37 @@ func newTestResolutionMonitor(t *testing.T) (*ResolutionMonitor, *clock.Determin
 }
 
 type stubResolutionMetrics struct {
-	calls map[bool]map[bool]int // completed -> max duration reached -> call count
+	calls map[metrics.ResolutionStatus]int
 }
 
-func (s *stubResolutionMetrics) RecordGameResolutionStatus(complete bool, maxDurationReached bool, count int) {
+func (s *stubResolutionMetrics) RecordGameResolutionStatus(status metrics.ResolutionStatus, count int) {
 	if s.calls == nil {
-		s.calls = make(map[bool]map[bool]int)
-		s.calls[true] = make(map[bool]int)
-		s.calls[false] = make(map[bool]int)
+		s.calls = make(map[metrics.ResolutionStatus]int)
 	}
-	s.calls[complete][maxDurationReached] += count
+	s.calls[status] += count
 }
 
 func newTestGames(duration uint64) []*types.EnrichedGameData {
-	newTestGame := func(duration uint64, status gameTypes.GameStatus) *types.EnrichedGameData {
-		return &types.EnrichedGameData{MaxClockDuration: duration, Status: status}
+	newTestGame := func(duration uint64, status gameTypes.GameStatus, resolvable bool) *types.EnrichedGameData {
+		game := &types.EnrichedGameData{
+			MaxClockDuration: duration,
+			Status:           status,
+		}
+		if !resolvable {
+			game.Claims = []types.EnrichedClaim{
+				{
+					Resolved: false,
+				},
+			}
+		}
+		return game
 	}
 	return []*types.EnrichedGameData{
-		newTestGame(duration, gameTypes.GameStatusInProgress),
-		newTestGame(duration*10, gameTypes.GameStatusInProgress),
-		newTestGame(duration, gameTypes.GameStatusDefenderWon),
-		newTestGame(duration*10, gameTypes.GameStatusChallengerWon),
+		newTestGame(duration/2, gameTypes.GameStatusInProgress, false),
+		newTestGame(duration*5, gameTypes.GameStatusInProgress, false),
+		newTestGame(duration/2, gameTypes.GameStatusInProgress, true),
+		newTestGame(duration*5, gameTypes.GameStatusInProgress, true),
+		newTestGame(duration/2, gameTypes.GameStatusDefenderWon, false),
+		newTestGame(duration*5, gameTypes.GameStatusChallengerWon, false),
 	}
 }