fix(op-dispute-mon): forecast metrics and test enhancements (#9468)

op-dispute-mon/mon/detector_test.go

@@ -34,7 +34,8 @@ func TestDetector_Detect(t *testing.T) {
 	t.Run("CheckAgreementFails", func(t *testing.T) {
 		detector, metrics, creator, rollup, _ := setupDetectorTest(t)
 		rollup.err = errors.New("boom")
-		creator.caller = &mockGameCaller{status: types.GameStatusInProgress}
+		creator.caller.status = []types.GameStatus{types.GameStatusInProgress}
+		creator.caller.rootClaim = []common.Hash{{}}
 		detector.Detect(context.Background(), []types.GameMetadata{{}})
 		metrics.Equals(t, 1, 0, 0) // Status should still be metriced here!
 		metrics.Mapped(t, map[string]int{})
@@ -42,7 +43,8 @@ func TestDetector_Detect(t *testing.T) {
 
 	t.Run("SingleGame", func(t *testing.T) {
 		detector, metrics, creator, _, _ := setupDetectorTest(t)
-		creator.caller = &mockGameCaller{status: types.GameStatusInProgress}
+		creator.caller.status = []types.GameStatus{types.GameStatusInProgress}
+		creator.caller.rootClaim = []common.Hash{{}}
 		detector.Detect(context.Background(), []types.GameMetadata{{}})
 		metrics.Equals(t, 1, 0, 0)
 		metrics.Mapped(t, map[string]int{"in_progress": 1})
@@ -50,7 +52,12 @@ func TestDetector_Detect(t *testing.T) {
 
 	t.Run("MultipleGames", func(t *testing.T) {
 		detector, metrics, creator, _, _ := setupDetectorTest(t)
-		creator.caller = &mockGameCaller{status: types.GameStatusInProgress}
+		creator.caller.status = []types.GameStatus{
+			types.GameStatusInProgress,
+			types.GameStatusInProgress,
+			types.GameStatusInProgress,
+		}
+		creator.caller.rootClaim = []common.Hash{{}, {}, {}}
 		detector.Detect(context.Background(), []types.GameMetadata{{}, {}, {}})
 		metrics.Equals(t, 3, 0, 0)
 		metrics.Mapped(t, map[string]int{"in_progress": 3})
@@ -128,13 +135,16 @@ func TestDetector_FetchGameMetadata(t *testing.T) {
 	t.Run("GetGameMetadataFails", func(t *testing.T) {
 		detector, _, creator, _, _ := setupDetectorTest(t)
 		creator.caller = &mockGameCaller{err: errors.New("boom")}
+		creator.caller.status = []types.GameStatus{types.GameStatusInProgress}
+		creator.caller.rootClaim = []common.Hash{{}}
 		_, _, _, err := detector.fetchGameMetadata(context.Background(), types.GameMetadata{})
 		require.Error(t, err)
 	})
 
 	t.Run("Success", func(t *testing.T) {
 		detector, _, creator, _, _ := setupDetectorTest(t)
-		creator.caller = &mockGameCaller{status: types.GameStatusInProgress}
+		creator.caller.status = []types.GameStatus{types.GameStatusInProgress}
+		creator.caller.rootClaim = []common.Hash{{}}
 		_, _, status, err := detector.fetchGameMetadata(context.Background(), types.GameMetadata{})
 		require.NoError(t, err)
 		require.Equal(t, types.GameStatusInProgress, status)
@@ -266,27 +276,29 @@ func (m *mockGameCallerCreator) CreateContract(game types.GameMetadata) (GameCal
 type mockGameCaller struct {
 	calls       int
 	claimsCalls int
-	claims      []faultTypes.Claim
-	status      types.GameStatus
-	rootClaim   common.Hash
+	claims      [][]faultTypes.Claim
+	status      []types.GameStatus
+	rootClaim   []common.Hash
 	err         error
 	claimsErr   error
 }
 
 func (m *mockGameCaller) GetGameMetadata(ctx context.Context) (uint64, common.Hash, types.GameStatus, error) {
+	idx := m.calls
 	m.calls++
 	if m.err != nil {
-		return 0, m.rootClaim, m.status, m.err
+		return 0, m.rootClaim[idx], m.status[idx], m.err
 	}
-	return 0, m.rootClaim, m.status, nil
+	return 0, m.rootClaim[idx], m.status[idx], nil
 }
 
 func (m *mockGameCaller) GetAllClaims(ctx context.Context) ([]faultTypes.Claim, error) {
+	idx := m.claimsCalls
 	m.claimsCalls++
 	if m.claimsErr != nil {
 		return nil, m.claimsErr
 	}
-	return m.claims, nil
+	return m.claims[idx], nil
 }
 
 type mockDetectorMetricer struct {

op-dispute-mon/mon/forecast.go

@@ -18,32 +18,44 @@ var (
 	ErrRootAgreement    = errors.New("failed to check root agreement")
 )
 
+type ForecastMetrics interface {
+	RecordGameAgreement(status string, count int)
+}
+
 type forecast struct {
-	logger log.Logger
-	// TODO(client-pod#536): Add forecast metrics.
-	//                       These should only fire if a game is in progress.
-	//                       otherwise, the detector should record the game status.
+	logger    log.Logger
+	metrics   ForecastMetrics
 	creator   GameCallerCreator
 	validator OutputValidator
 }
 
-func newForecast(logger log.Logger, creator GameCallerCreator, validator OutputValidator) *forecast {
+func newForecast(logger log.Logger, metrics ForecastMetrics, creator GameCallerCreator, validator OutputValidator) *forecast {
 	return &forecast{
 		logger:    logger,
+		metrics:   metrics,
 		creator:   creator,
 		validator: validator,
 	}
 }
 
 func (f *forecast) Forecast(ctx context.Context, games []types.GameMetadata) {
+	batch := forecastBatch{}
 	for _, game := range games {
-		if err := f.forecastGame(ctx, game); err != nil {
+		if err := f.forecastGame(ctx, game, &batch); err != nil {
 			f.logger.Error("Failed to forecast game", "err", err)
 		}
 	}
+	f.recordBatch(batch)
+}
+
+func (f *forecast) recordBatch(batch forecastBatch) {
+	f.metrics.RecordGameAgreement("agree_challenger_ahead", batch.AgreeChallengerAhead)
+	f.metrics.RecordGameAgreement("disagree_challenger_ahead", batch.DisagreeChallengerAhead)
+	f.metrics.RecordGameAgreement("agree_defender_ahead", batch.AgreeDefenderAhead)
+	f.metrics.RecordGameAgreement("disagree_defender_ahead", batch.DisagreeDefenderAhead)
 }
 
-func (f *forecast) forecastGame(ctx context.Context, game types.GameMetadata) error {
+func (f *forecast) forecastGame(ctx context.Context, game types.GameMetadata, metrics *forecastBatch) error {
 	loader, err := f.creator.CreateContract(game)
 	if err != nil {
 		return fmt.Errorf("%w: %w", ErrContractCreation, err)
@@ -80,15 +92,19 @@ func (f *forecast) forecastGame(ctx context.Context, game types.GameMetadata) er
 	if agreement {
 		// If we agree with the output root proposal, the Defender should win, defending that claim.
 		if status == types.GameStatusChallengerWon {
+			metrics.AgreeChallengerAhead++
 			f.logger.Warn("Forecasting unexpected game result", "status", status, "game", game, "rootClaim", rootClaim, "expected", expected)
 		} else {
+			metrics.AgreeDefenderAhead++
 			f.logger.Debug("Forecasting expected game result", "status", status, "game", game, "rootClaim", rootClaim, "expected", expected)
 		}
 	} else {
 		// If we disagree with the output root proposal, the Challenger should win, challenging that claim.
 		if status == types.GameStatusDefenderWon {
+			metrics.DisagreeDefenderAhead++
 			f.logger.Warn("Forecasting unexpected game result", "status", status, "game", game, "rootClaim", rootClaim, "expected", expected)
 		} else {
+			metrics.DisagreeChallengerAhead++
 			f.logger.Debug("Forecasting expected game result", "status", status, "game", game, "rootClaim", rootClaim, "expected", expected)
 		}
 	}

op-dispute-mon/mon/service.go

@@ -79,11 +79,14 @@ func (s *Service) initFromConfig(ctx context.Context, cfg *config.Config) error
 	if err := s.initOutputRollupClient(ctx, cfg); err != nil {
 		return fmt.Errorf("failed to init rollup client: %w", err)
 	}
-	s.initOutputValidator()
-	s.initGameCallerCreator()
+
+	s.initOutputValidator()   // Must be called before initForecast
+	s.initGameCallerCreator() // Must be called before initForecast
+
 	s.initForecast(cfg)
 	s.initDetector()
-	s.initMonitor(ctx, cfg)
+
+	s.initMonitor(ctx, cfg) // Monitor must be initialized last
 
 	s.metrics.RecordInfo(version.SimpleWithMeta)
 	s.metrics.RecordUp()
@@ -100,7 +103,7 @@ func (s *Service) initGameCallerCreator() {
 }
 
 func (s *Service) initForecast(cfg *config.Config) {
-	s.forecast = newForecast(s.logger, s.game, s.validator)
+	s.forecast = newForecast(s.logger, s.metrics, s.game, s.validator)
 }
 
 func (s *Service) initDetector() {

op-dispute-mon/mon/types.go

@@ -19,6 +19,13 @@ func (s *statusBatch) Add(status types.GameStatus) {
 	}
 }
 
+type forecastBatch struct {
+	AgreeDefenderAhead      int
+	DisagreeDefenderAhead   int
+	AgreeChallengerAhead    int
+	DisagreeChallengerAhead int
+}
+
 type detectionBatch struct {
 	inProgress             int
 	agreeDefenderWins      int