op-dispute-mon: Publish latest valid and invalid proposal times (#10564)

Changes latest_invalid_proposal metric to latest_proposal with a root_agreement label. Will enable alerting if the proposer isn't creating games on the expected schedule.

op-dispute-mon/metrics/metrics.go

@@ -126,7 +126,7 @@ type Metricer interface {
 
 	RecordGameAgreement(status GameAgreementStatus, count int)
 
-	RecordLatestInvalidProposal(timestamp uint64)
+	RecordLatestProposals(latestValid, latestInvalid uint64)
 
 	RecordIgnoredGames(count int)
 
@@ -167,11 +167,11 @@ type Metrics struct {
 
 	lastOutputFetch prometheus.Gauge
 
-	gamesAgreement        prometheus.GaugeVec
-	latestInvalidProposal prometheus.Gauge
-	ignoredGames          prometheus.Gauge
-	failedGames           prometheus.Gauge
-	l2Challenges          prometheus.GaugeVec
+	gamesAgreement  prometheus.GaugeVec
+	latestProposals prometheus.GaugeVec
+	ignoredGames    prometheus.Gauge
+	failedGames     prometheus.Gauge
+	l2Challenges    prometheus.GaugeVec
 
 	requiredCollateral  prometheus.GaugeVec
 	availableCollateral prometheus.GaugeVec
@@ -277,11 +277,12 @@ func NewMetrics() *Metrics {
 			"result_correctness",
 			"root_agreement",
 		}),
-		latestInvalidProposal: factory.NewGauge(prometheus.GaugeOpts{
+		latestProposals: *factory.NewGaugeVec(prometheus.GaugeOpts{
 			Namespace: Namespace,
-			Name:      "latest_invalid_proposal",
-			Help:      "Timestamp of the most recent game with an invalid root claim in unix seconds",
-		}),
+			Name:      "latest_proposal",
+			Help:      "Timestamp of the most recent game with a valid or invalid root claim in unix seconds",
+		},
+			[]string{"root_agreement"}),
 		ignoredGames: factory.NewGauge(prometheus.GaugeOpts{
 			Namespace: Namespace,
 			Name:      "ignored_games",
@@ -452,8 +453,9 @@ func (m *Metrics) RecordGameAgreement(status GameAgreementStatus, count int) {
 	m.gamesAgreement.WithLabelValues(labelValuesFor(status)...).Set(float64(count))
 }
 
-func (m *Metrics) RecordLatestInvalidProposal(timestamp uint64) {
-	m.latestInvalidProposal.Set(float64(timestamp))
+func (m *Metrics) RecordLatestProposals(latestValid, latestInvalid uint64) {
+	m.latestProposals.WithLabelValues("agree").Set(float64(latestValid))
+	m.latestProposals.WithLabelValues("disagree").Set(float64(latestInvalid))
 }
 
 func (m *Metrics) RecordIgnoredGames(count int) {

op-dispute-mon/metrics/noop.go

@@ -36,7 +36,7 @@ func (*NoopMetricsImpl) RecordOutputFetchTime(_ float64) {}
 
 func (*NoopMetricsImpl) RecordGameAgreement(_ GameAgreementStatus, _ int) {}
 
-func (*NoopMetricsImpl) RecordLatestInvalidProposal(_ uint64) {}
+func (*NoopMetricsImpl) RecordLatestProposals(_ uint64, _ uint64) {}
 
 func (*NoopMetricsImpl) RecordIgnoredGames(_ int) {}
 

op-dispute-mon/mon/forecast.go

@@ -16,7 +16,7 @@ var (
 
 type ForecastMetrics interface {
 	RecordGameAgreement(status metrics.GameAgreementStatus, count int)
-	RecordLatestInvalidProposal(timestamp uint64)
+	RecordLatestProposals(validTimestamp, invalidTimestamp uint64)
 	RecordIgnoredGames(count int)
 	RecordFailedGames(count int)
 }
@@ -33,6 +33,7 @@ type forecastBatch struct {
 	DisagreeChallengerWins int
 
 	LatestInvalidProposal uint64
+	LatestValidProposal   uint64
 }
 
 type Forecast struct {
@@ -68,7 +69,7 @@ func (f *Forecast) recordBatch(batch forecastBatch, ignoredCount, failedCount in
 	f.metrics.RecordGameAgreement(metrics.AgreeDefenderAhead, batch.AgreeDefenderAhead)
 	f.metrics.RecordGameAgreement(metrics.DisagreeDefenderAhead, batch.DisagreeDefenderAhead)
 
-	f.metrics.RecordLatestInvalidProposal(batch.LatestInvalidProposal)
+	f.metrics.RecordLatestProposals(batch.LatestValidProposal, batch.LatestInvalidProposal)
 
 	f.metrics.RecordIgnoredGames(ignoredCount)
 	f.metrics.RecordFailedGames(failedCount)
@@ -85,6 +86,10 @@ func (f *Forecast) forecastGame(game *monTypes.EnrichedGameData, metrics *foreca
 		if metrics.LatestInvalidProposal < game.Timestamp {
 			metrics.LatestInvalidProposal = game.Timestamp
 		}
+	} else {
+		if metrics.LatestValidProposal < game.Timestamp {
+			metrics.LatestValidProposal = game.Timestamp
+		}
 	}
 
 	if game.Status != types.GameStatusInProgress {

op-dispute-mon/mon/forecast_test.go

@@ -263,8 +263,8 @@ func TestForecast_Forecast_MultipleGames(t *testing.T) {
 		{},
 		mockRootClaim,
 		{},
-		{}, // Expected latest invalid proposal (will have timestamp 7)
-		mockRootClaim,
+		{},            // Expected latest invalid proposal (will have timestamp 7)
+		mockRootClaim, // Expected latest valid proposal (will have timestamp 8)
 	}
 	games := make([]*monTypes.EnrichedGameData, 9)
 	for i := range games {
@@ -293,6 +293,7 @@ func TestForecast_Forecast_MultipleGames(t *testing.T) {
 	require.Equal(t, 3, m.ignoredGames)
 	require.Equal(t, 4, m.contractCreationFails)
 	require.EqualValues(t, 7, m.latestInvalidProposal)
+	require.EqualValues(t, 8, m.latestValidProposal)
 }
 
 func setupForecastTest(t *testing.T) (*Forecast, *mockForecastMetrics, *testlog.CapturingHandler) {
@@ -320,6 +321,7 @@ type mockForecastMetrics struct {
 	gameAgreement         map[metrics.GameAgreementStatus]int
 	ignoredGames          int
 	latestInvalidProposal uint64
+	latestValidProposal   uint64
 	contractCreationFails int
 }
 
@@ -331,8 +333,9 @@ func (m *mockForecastMetrics) RecordGameAgreement(status metrics.GameAgreementSt
 	m.gameAgreement[status] = count
 }
 
-func (m *mockForecastMetrics) RecordLatestInvalidProposal(timestamp uint64) {
-	m.latestInvalidProposal = timestamp
+func (m *mockForecastMetrics) RecordLatestProposals(valid, invalid uint64) {
+	m.latestValidProposal = valid
+	m.latestInvalidProposal = invalid
 }
 
 func (m *mockForecastMetrics) RecordIgnoredGames(count int) {