Track in flight games inside the scheduler component.

op-challenger/game/scheduler/coordinator.go

@@ -65,6 +65,7 @@ func (c *coordinator) schedule(ctx context.Context, games []common.Address) erro
 			errs = append(errs, err)
 		} else if j != nil {
 			jobs = append(jobs, *j)
+			c.m.RecordGameUpdateScheduled()
 		}
 		state, ok := c.states[addr]
 		if ok {
@@ -136,6 +137,7 @@ func (c *coordinator) processResult(j job) error {
 	state.inflight = false
 	state.status = j.status
 	c.deleteResolvedGameFiles()
+	c.m.RecordGameUpdateCompleted()
 	return nil
 }
 

op-challenger/game/scheduler/scheduler.go

@@ -13,6 +13,8 @@ var ErrBusy = errors.New("busy scheduling previous update")
 
 type SchedulerMetricer interface {
 	RecordGamesStatus(inProgress, defenderWon, challengerWon int)
+	RecordGameUpdateScheduled()
+	RecordGameUpdateCompleted()
 }
 
 type Scheduler struct {

op-challenger/metrics/metrics.go

@@ -26,6 +26,9 @@ type Metricer interface {
 	RecordCannonExecutionTime(t float64)
 
 	RecordGamesStatus(inProgress, defenderWon, challengerWon int)
+
+	RecordGameUpdateScheduled()
+	RecordGameUpdateCompleted()
 }
 
 type Metrics struct {
@@ -44,6 +47,7 @@ type Metrics struct {
 	cannonExecutionTime prometheus.Histogram
 
 	trackedGames  prometheus.GaugeVec
+	inflightGames prometheus.Gauge
 }
 
 var _ Metricer = (*Metrics)(nil)
@@ -85,7 +89,9 @@ func NewMetrics() *Metrics {
 			Namespace: Namespace,
 			Name:      "cannon_execution_time",
 			Help:      "Time (in seconds) to execute cannon",
-			Buckets:   append([]float64{1.0, 10.0}, prometheus.ExponentialBuckets(30.0, 2.0, 14)...),
+			Buckets: append(
+				[]float64{1.0, 10.0},
+				prometheus.ExponentialBuckets(30.0, 2.0, 14)...),
 		}),
 		trackedGames: *factory.NewGaugeVec(prometheus.GaugeOpts{
 			Namespace: Namespace,
@@ -94,6 +100,11 @@ func NewMetrics() *Metrics {
 		}, []string{
 			"status",
 		}),
+		inflightGames: factory.NewGauge(prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Name:      "inflight_games",
+			Help:      "Number of games being tracked by the challenger",
+		}),
 	}
 }
 
@@ -101,7 +112,12 @@ func (m *Metrics) Serve(ctx context.Context, host string, port int) error {
 	return opmetrics.ListenAndServe(ctx, m.registry, host, port)
 }
 
-func (m *Metrics) StartBalanceMetrics(ctx context.Context, l log.Logger, client *ethclient.Client, account common.Address) {
+func (m *Metrics) StartBalanceMetrics(
+	ctx context.Context,
+	l log.Logger,
+	client *ethclient.Client,
+	account common.Address,
+) {
 	opmetrics.LaunchBalanceMetrics(ctx, l, m.registry, m.ns, client, account)
 }
 
@@ -138,3 +154,11 @@ func (m *Metrics) RecordGamesStatus(inProgress, defenderWon, challengerWon int)
 	m.trackedGames.WithLabelValues("defender_won").Set(float64(defenderWon))
 	m.trackedGames.WithLabelValues("challenger_won").Set(float64(challengerWon))
 }
+
+func (m *Metrics) RecordGameUpdateScheduled() {
+	m.inflightGames.Add(1)
+}
+
+func (m *Metrics) RecordGameUpdateCompleted() {
+	m.inflightGames.Sub(1)
+}

op-challenger/metrics/noop.go

@@ -19,3 +19,6 @@ func (*noopMetrics) RecordGameStep() {}
 func (*noopMetrics) RecordCannonExecutionTime(t float64) {}
 
 func (*noopMetrics) RecordGamesStatus(inProgress, defenderWon, challengerWon int) {}
+
+func (*noopMetrics) RecordGameUpdateScheduled() {}
+func (*noopMetrics) RecordGameUpdateCompleted() {}