Merge branch 'develop' into op-service/http-util-cleanup

README.md

@@ -57,6 +57,7 @@ Refer to the Directory Structure section below to understand which packages are
 ├── <a href="./op-exporter">op-exporter</a>: Prometheus exporter client
 ├── <a href="./op-heartbeat">op-heartbeat</a>: Heartbeat monitor service
 ├── <a href="./op-node">op-node</a>: rollup consensus-layer client
+├── <a href="./op-preimage">op-preimage</a>: Go bindings for Preimage Oracle
 ├── <a href="./op-program">op-program</a>: Fault proof program
 ├── <a href="./op-proposer">op-proposer</a>: L2-Output Submitter, submits proposals to L1
 ├── <a href="./op-service">op-service</a>: Common codebase utilities

indexer/Dockerfile

@@ -10,8 +10,6 @@ COPY ./op-node /app/op-node
 COPY ./go.mod /app/go.mod
 COPY ./go.sum /app/go.sum
 
-COPY ./.git /app/.git
-
 WORKDIR /app/indexer
 
 RUN go mod download

indexer/cmd/indexer/cli.go

@@ -38,7 +38,7 @@ func runIndexer(ctx *cli.Context) error {
 	}
 	defer db.Close()
 
-	indexer, err := indexer.NewIndexer(log, db, cfg.Chain, cfg.RPCs, cfg.Metrics)
+	indexer, err := indexer.NewIndexer(log, db, cfg.Chain, cfg.RPCs, cfg.HTTPServer, cfg.MetricsServer)
 	if err != nil {
 		log.Error("failed to create indexer", "err", err)
 		return err
@@ -63,7 +63,7 @@ func runApi(ctx *cli.Context) error {
 	defer db.Close()
 
 	api := api.NewApi(log, db.BridgeTransfers)
-	return api.Listen(ctx.Context, cfg.API.Port)
+	return api.Listen(ctx.Context, cfg.HTTPServer.Port)
 }
 
 func runAll(ctx *cli.Context) error {

indexer/config/config.go

@@ -20,11 +20,11 @@ const (
 
 // Config represents the `indexer.toml` file used to configure the indexer
 type Config struct {
-	Chain   ChainConfig   `toml:"chain"`
-	RPCs    RPCsConfig    `toml:"rpcs"`
-	DB      DBConfig      `toml:"db"`
-	API     APIConfig     `toml:"api"`
-	Metrics MetricsConfig `toml:"metrics"`
+	Chain         ChainConfig  `toml:"chain"`
+	RPCs          RPCsConfig   `toml:"rpcs"`
+	DB            DBConfig     `toml:"db"`
+	HTTPServer    ServerConfig `toml:"http"`
+	MetricsServer ServerConfig `toml:"metrics"`
 }
 
 // fetch this via onchain config from RPCsConfig and remove from config in future
@@ -96,14 +96,8 @@ type DBConfig struct {
 	Password string `toml:"password"`
 }
 
-// APIConfig configures the API server
-type APIConfig struct {
-	Host string `toml:"host"`
-	Port int    `toml:"port"`
-}
-
-// MetricsConfig configures the metrics server
-type MetricsConfig struct {
+// Configures the a server
+type ServerConfig struct {
 	Host string `toml:"host"`
 	Port int    `toml:"port"`
 }

indexer/config/config_test.go

@@ -31,9 +31,9 @@ func TestLoadConfig(t *testing.T) {
 		port = 5432
 		user = "postgres"
 		password = "postgres"
-	  name = "indexer"
+	    name = "indexer"
 
-		[api]
+		[http]
 		host = "127.0.0.1"
 		port = 8080
 
@@ -65,10 +65,10 @@ func TestLoadConfig(t *testing.T) {
 	require.Equal(t, conf.DB.User, "postgres")
 	require.Equal(t, conf.DB.Password, "postgres")
 	require.Equal(t, conf.DB.Name, "indexer")
-	require.Equal(t, conf.API.Host, "127.0.0.1")
-	require.Equal(t, conf.API.Port, 8080)
-	require.Equal(t, conf.Metrics.Host, "127.0.0.1")
-	require.Equal(t, conf.Metrics.Port, 7300)
+	require.Equal(t, conf.HTTPServer.Host, "127.0.0.1")
+	require.Equal(t, conf.HTTPServer.Port, 8080)
+	require.Equal(t, conf.MetricsServer.Host, "127.0.0.1")
+	require.Equal(t, conf.MetricsServer.Port, 7300)
 }
 
 func TestLoadConfig_WithoutPreset(t *testing.T) {

indexer/e2e_tests/setup.go

@@ -82,10 +82,8 @@ func createE2ETestSuite(t *testing.T) E2ETestSuite {
 				L1StandardBridgeProxy:       opCfg.L1Deployments.L1StandardBridgeProxy,
 			},
 		},
-		Metrics: config.MetricsConfig{
-			Host: "127.0.0.1",
-			Port: 0,
-		},
+		HTTPServer:    config.ServerConfig{Host: "127.0.0.1", Port: 0},
+		MetricsServer: config.ServerConfig{Host: "127.0.0.1", Port: 0},
 	}
 
 	db, err := database.NewDB(indexerCfg.DB)
@@ -93,7 +91,7 @@ func createE2ETestSuite(t *testing.T) E2ETestSuite {
 	t.Cleanup(func() { db.Close() })
 
 	indexerLog := testlog.Logger(t, log.LvlInfo).New("role", "indexer")
-	indexer, err := indexer.NewIndexer(indexerLog, db, indexerCfg.Chain, indexerCfg.RPCs, indexerCfg.Metrics)
+	indexer, err := indexer.NewIndexer(indexerLog, db, indexerCfg.Chain, indexerCfg.RPCs, indexerCfg.HTTPServer, indexerCfg.MetricsServer)
 	require.NoError(t, err)
 
 	indexerCtx, indexerStop := context.WithCancel(context.Background())

indexer/indexer.go

@@ -4,10 +4,15 @@ import (
 	"context"
 	"fmt"
 	"math/big"
+	"net/http"
 	"runtime/debug"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/log"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/go-chi/chi/v5/middleware"
+
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/ethereum-optimism/optimism/indexer/config"
@@ -15,6 +20,7 @@ import (
 	"github.com/ethereum-optimism/optimism/indexer/etl"
 	"github.com/ethereum-optimism/optimism/indexer/node"
 	"github.com/ethereum-optimism/optimism/indexer/processors"
+	"github.com/ethereum-optimism/optimism/op-service/httputil"
 	"github.com/ethereum-optimism/optimism/op-service/metrics"
 )
 
@@ -24,7 +30,8 @@ type Indexer struct {
 	log log.Logger
 	db  *database.DB
 
-	metricsConfig   config.MetricsConfig
+	httpConfig      config.ServerConfig
+	metricsConfig   config.ServerConfig
 	metricsRegistry *prometheus.Registry
 
 	L1ETL           *etl.L1ETL
@@ -33,7 +40,14 @@ type Indexer struct {
 }
 
 // NewIndexer initializes an instance of the Indexer
-func NewIndexer(logger log.Logger, db *database.DB, chainConfig config.ChainConfig, rpcsConfig config.RPCsConfig, metricsConfig config.MetricsConfig) (*Indexer, error) {
+func NewIndexer(
+	log log.Logger,
+	db *database.DB,
+	chainConfig config.ChainConfig,
+	rpcsConfig config.RPCsConfig,
+	httpConfig config.ServerConfig,
+	metricsConfig config.ServerConfig,
+) (*Indexer, error) {
 	metricsRegistry := metrics.NewRegistry()
 
 	// L1
@@ -47,7 +61,7 @@ func NewIndexer(logger log.Logger, db *database.DB, chainConfig config.ChainConf
 		ConfirmationDepth: big.NewInt(int64(chainConfig.L1ConfirmationDepth)),
 		StartHeight:       big.NewInt(int64(chainConfig.L1StartingHeight)),
 	}
-	l1Etl, err := etl.NewL1ETL(l1Cfg, logger, db, etl.NewMetrics(metricsRegistry, "l1"), l1EthClient, chainConfig.L1Contracts)
+	l1Etl, err := etl.NewL1ETL(l1Cfg, log, db, etl.NewMetrics(metricsRegistry, "l1"), l1EthClient, chainConfig.L1Contracts)
 	if err != nil {
 		return nil, err
 	}
@@ -62,21 +76,22 @@ func NewIndexer(logger log.Logger, db *database.DB, chainConfig config.ChainConf
 		HeaderBufferSize:  chainConfig.L2HeaderBufferSize,
 		ConfirmationDepth: big.NewInt(int64(chainConfig.L2ConfirmationDepth)),
 	}
-	l2Etl, err := etl.NewL2ETL(l2Cfg, logger, db, etl.NewMetrics(metricsRegistry, "l2"), l2EthClient)
+	l2Etl, err := etl.NewL2ETL(l2Cfg, log, db, etl.NewMetrics(metricsRegistry, "l2"), l2EthClient)
 	if err != nil {
 		return nil, err
 	}
 
 	// Bridge
-	bridgeProcessor, err := processors.NewBridgeProcessor(logger, db, l1Etl, chainConfig)
+	bridgeProcessor, err := processors.NewBridgeProcessor(log, db, l1Etl, chainConfig)
 	if err != nil {
 		return nil, err
 	}
 
 	indexer := &Indexer{
-		log: logger,
+		log: log,
 		db:  db,
 
+		httpConfig:      httpConfig,
 		metricsConfig:   metricsConfig,
 		metricsRegistry: metricsRegistry,
 
@@ -88,6 +103,23 @@ func NewIndexer(logger log.Logger, db *database.DB, chainConfig config.ChainConf
 	return indexer, nil
 }
 
+func (i *Indexer) startHttpServer(ctx context.Context) error {
+	i.log.Info("starting http server...", "port", i.httpConfig.Host)
+
+	r := chi.NewRouter()
+	r.Use(middleware.Heartbeat("/healthz"))
+
+	server := http.Server{Addr: fmt.Sprintf("%s:%d", i.httpConfig.Host, i.httpConfig.Port), Handler: r}
+	err := httputil.ListenAndServeContext(ctx, &server)
+	if err != nil {
+		i.log.Error("http server stopped", "err", err)
+	} else {
+		i.log.Info("http server stopped")
+	}
+
+	return err
+}
+
 func (i *Indexer) startMetricsServer(ctx context.Context) error {
 	i.log.Info("starting metrics server...", "port", i.metricsConfig.Port)
 	err := metrics.ListenAndServe(ctx, i.metricsRegistry, i.metricsConfig.Host, i.metricsConfig.Port)
@@ -103,7 +135,7 @@ func (i *Indexer) startMetricsServer(ctx context.Context) error {
 // Start starts the indexing service on L1 and L2 chains
 func (i *Indexer) Run(ctx context.Context) error {
 	var wg sync.WaitGroup
-	errCh := make(chan error, 4)
+	errCh := make(chan error, 5)
 
 	// if any goroutine halts, we stop the entire indexer
 	processCtx, processCancel := context.WithCancel(ctx)
@@ -130,6 +162,7 @@ func (i *Indexer) Run(ctx context.Context) error {
 	runProcess(i.L2ETL.Start)
 	runProcess(i.BridgeProcessor.Start)
 	runProcess(i.startMetricsServer)
+	runProcess(i.startHttpServer)
 	wg.Wait()
 
 	err := <-errCh

indexer/indexer.toml

@@ -28,7 +28,7 @@ user = "db_username"
 password = "db_password"
 name = "db_name"
 
-[api]
+[http]
 host = "127.0.0.1"
 port = 8080
 

op-challenger/README.md

@@ -7,13 +7,18 @@ games, and validity games. To learn more about dispute games, visit the
 
 ## Quickstart
 
-First, clone this repo. Then, run `make`, which will build all required targets.
-Alternatively, run `make devnet` to bring up the [devnet](../ops-bedrock/devnet-up.sh)
-which deploys the [mock dispute game contracts](./contracts) as well as an
-`op-challenger` instance.
+First, clone this repo. Then run:
 
-Alternatively, you can build the `op-challenger` binary locally using the pre-configured
-[Makefile](./Makefile) target by running `make build`, and then running `./op-challenger --help`
+```shell
+cd op-challenger
+make alphabet
+```
+
+This creates a local devnet, starts a dispute game using the simple alphabet trace type and runs two op-challenger
+instances with differing views of the correct alphabet to play the game.
+
+Alternatively, you can build the `op-challenger` binary directly using the pre-configured
+[Makefile](./Makefile) target by running `make build`, and then running `./bin/op-challenger --help`
 to see a list of available options.
 
 ## Usage
@@ -21,6 +26,40 @@ to see a list of available options.
 `op-challenger` is configurable via command line flags and environment variables. The help menu
 shows the available config options and can be accessed by running `./op-challenger --help`.
 
+### Running with Cannon on Local Devnet
+
+To run `op-challenger` against the local devnet, first ensure the required components are built and the devnet is running.
+From the top level of the repository run:
+
+```shell
+make devnet-clean
+make cannon-prestate op-challenger
+make devnet-up
+```
+
+Then start `op-challenger` with:
+```shell
+DISPUTE_GAME_FACTORY=$(jq -r .DisputeGameFactoryProxy .devnet/addresses.json)
+./op-challenger/bin/op-challenger \
+  --trace-type cannon \
+  --l1-eth-rpc http://localhost:8545 \
+  --game-factory-address $DISPUTE_GAME_FACTORY \
+  --agree-with-proposed-output=true \
+  --datadir temp/challenger-data \
+  --cannon-rollup-config .devnet/rollup.json  \
+  --cannon-l2-genesis .devnet/genesis-l2.json \
+  --cannon-bin ./cannon/bin/cannon \
+  --cannon-server ./op-program/bin/op-program \
+  --cannon-prestate ./op-program/bin/prestate.json \
+  --cannon-l2 http://localhost:9545 \
+  --mnemonic "test test test test test test test test test test test junk" \
+  --hd-path "m/44'/60'/0'/0/8" \
+  --num-confirmations 1
+```
+
+The mnemonic and hd-path above is a prefunded address on the devnet. The challenger respond to any created games by
+posting the correct trace as the counter-claim. The scripts below can then be used to create and interact with games.
+
 ## Scripts
 
 The [scripts](scripts) directory contains a collection of scripts to assist with manually creating and playing games.
@@ -28,6 +67,23 @@ This are not intended to be used in production, only to support manual testing a
 dispute games work. They also serve as examples of how to use `cast` to manually interact with the dispute game
 contracts.
 
+### Understanding Revert Reasons
+
+When actions performed by these scripts fails, they typically print a message that includes the
+abi encoded revert reason provided by the contract. e.g.
+
+```
+Error:
+(code: 3, message: execution reverted, data: Some(String("0x67fe1950")))
+```
+
+The `cast 4byte` command can be used to decode these revert reasons. e.g.
+
+```shell
+$ cast 4byte 0x67fe1950
+GameNotInProgress()
+```
+
 ### Dependencies
 
 These scripts assume that the following tools are installed and available on the current `PATH`:
@@ -78,6 +134,32 @@ Performs a move to either attack or defend the latest claim in the specified gam
   These arguments must specify a way for `cast` to sign the transactions.
   See `cast send --help` for supported options.
 
+### [resolve.sh](scripts/resolve.sh)
+
+```shell
+./scripts/resolve.sh <RPC_URL> <GAME_ADDRESS> <SIGNER_ARGS>...
+```
+
+Resolves a dispute game. Note that this will fail if the dispute game has already been resolved
+or if the clocks have not yet expired and further moves are possible.
+If the game is resolved successfully, the result is printed.
+
+* `RPC_URL` - the RPC endpoint of the L1 endpoint to use (e.g. `http://localhost:8545`).
+* `GAME_ADDRESS` - the address of the dispute game to resolve.
+* `SIGNER_ARGS` the remaining args are past as arguments to `cast` when sending transactions.
+  These arguments must specify a way for `cast` to sign the transactions.
+  See `cast send --help` for supported options.
+
+### [list_games.sh](scripts/list_games.sh)
+
+```shell
+./scripts/list_games.sh <RPC> <GAME_FACTORY_ADDRESS>
+```
+
+Prints the games created by the game factory along with their current status.
+
+* `RPC_URL` - the RPC endpoint of the L1 endpoint to use (e.g. `http://localhost:8545`).
+* `GAME_FACTORY_ADDRESS` - the address of the dispute game factory contract on L1.
 
 ### [list_claims.sh](scripts/list_claims.sh)
 

op-challenger/game/fault/agent.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/solver"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum/go-ethereum/log"
 )
 
@@ -24,6 +25,7 @@ type ClaimLoader interface {
 }
 
 type Agent struct {
+	metrics                 metrics.Metricer
 	solver                  *solver.Solver
 	loader                  ClaimLoader
 	responder               Responder
@@ -33,8 +35,9 @@ type Agent struct {
 	log                     log.Logger
 }
 
-func NewAgent(loader ClaimLoader, maxDepth int, trace types.TraceProvider, responder Responder, updater types.OracleUpdater, agreeWithProposedOutput bool, log log.Logger) *Agent {
+func NewAgent(m metrics.Metricer, loader ClaimLoader, maxDepth int, trace types.TraceProvider, responder Responder, updater types.OracleUpdater, agreeWithProposedOutput bool, log log.Logger) *Agent {
 	return &Agent{
+		metrics:                 m,
 		solver:                  solver.NewSolver(maxDepth, trace),
 		loader:                  loader,
 		responder:               responder,
@@ -71,7 +74,7 @@ func (a *Agent) Act(ctx context.Context) error {
 
 // shouldResolve returns true if the agent should resolve the game.
 // This method will return false if the game is still in progress.
-func (a *Agent) shouldResolve(ctx context.Context, status types.GameStatus) bool {
+func (a *Agent) shouldResolve(status types.GameStatus) bool {
 	expected := types.GameStatusDefenderWon
 	if a.agreeWithProposedOutput {
 		expected = types.GameStatusChallengerWon
@@ -82,20 +85,19 @@ func (a *Agent) shouldResolve(ctx context.Context, status types.GameStatus) bool
 	return expected == status
 }
 
-// tryResolve resolves the game if it is in a terminal state
-// and returns true if the game resolves successfully.
+// tryResolve resolves the game if it is in a winning state
+// Returns true if the game is resolvable (regardless of whether it was actually resolved)
 func (a *Agent) tryResolve(ctx context.Context) bool {
 	status, err := a.responder.CallResolve(ctx)
-	if err != nil {
+	if err != nil || status == types.GameStatusInProgress {
 		return false
 	}
-	if !a.shouldResolve(ctx, status) {
-		return false
+	if !a.shouldResolve(status) {
+		return true
 	}
 	a.log.Info("Resolving game")
 	if err := a.responder.Resolve(ctx); err != nil {
 		a.log.Error("Failed to resolve the game", "err", err)
-		return false
 	}
 	return true
 }
@@ -134,6 +136,7 @@ func (a *Agent) move(ctx context.Context, claim types.Claim, game types.Game) er
 		log.Debug("Skipping duplicate move")
 		return nil
 	}
+	a.metrics.RecordGameMove()
 	log.Info("Performing move")
 	return a.responder.Respond(ctx, move)
 }
@@ -170,6 +173,7 @@ func (a *Agent) step(ctx context.Context, claim types.Claim, game types.Game) er
 
 	a.log.Info("Performing step", "is_attack", step.IsAttack,
 		"depth", step.LeafClaim.Depth(), "index_at_depth", step.LeafClaim.IndexAtDepth(), "value", step.LeafClaim.Value)
+	a.metrics.RecordGameStep()
 	callData := types.StepCallData{
 		ClaimIndex: uint64(step.LeafClaim.ContractIndex),
 		IsAttack:   step.IsAttack,

op-challenger/game/fault/agent_test.go

@@ -2,9 +2,13 @@ package fault
 
 import (
 	"context"
+	"errors"
 	"testing"
 
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/test"
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/alphabet"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/stretchr/testify/require"
 
@@ -13,19 +17,143 @@ import (
 
 // TestShouldResolve tests the resolution logic.
 func TestShouldResolve(t *testing.T) {
-	log := testlog.Logger(t, log.LvlCrit)
-
 	t.Run("AgreeWithProposedOutput", func(t *testing.T) {
-		agent := NewAgent(nil, 0, nil, nil, nil, true, log)
-		require.False(t, agent.shouldResolve(context.Background(), types.GameStatusDefenderWon))
-		require.True(t, agent.shouldResolve(context.Background(), types.GameStatusChallengerWon))
-		require.False(t, agent.shouldResolve(context.Background(), types.GameStatusInProgress))
+		agent, _, _ := setupTestAgent(t, true)
+		require.False(t, agent.shouldResolve(types.GameStatusDefenderWon))
+		require.True(t, agent.shouldResolve(types.GameStatusChallengerWon))
+		require.False(t, agent.shouldResolve(types.GameStatusInProgress))
 	})
 
 	t.Run("DisagreeWithProposedOutput", func(t *testing.T) {
-		agent := NewAgent(nil, 0, nil, nil, nil, false, log)
-		require.True(t, agent.shouldResolve(context.Background(), types.GameStatusDefenderWon))
-		require.False(t, agent.shouldResolve(context.Background(), types.GameStatusChallengerWon))
-		require.False(t, agent.shouldResolve(context.Background(), types.GameStatusInProgress))
+		agent, _, _ := setupTestAgent(t, false)
+		require.True(t, agent.shouldResolve(types.GameStatusDefenderWon))
+		require.False(t, agent.shouldResolve(types.GameStatusChallengerWon))
+		require.False(t, agent.shouldResolve(types.GameStatusInProgress))
 	})
 }
+
+func TestDoNotMakeMovesWhenGameIsResolvable(t *testing.T) {
+	ctx := context.Background()
+
+	tests := []struct {
+		name                    string
+		agreeWithProposedOutput bool
+		callResolveStatus       types.GameStatus
+		shouldResolve           bool
+	}{
+		{
+			name:                    "Agree_Losing",
+			agreeWithProposedOutput: true,
+			callResolveStatus:       types.GameStatusDefenderWon,
+			shouldResolve:           false,
+		},
+		{
+			name:                    "Agree_Winning",
+			agreeWithProposedOutput: true,
+			callResolveStatus:       types.GameStatusChallengerWon,
+			shouldResolve:           true,
+		},
+		{
+			name:                    "Disagree_Losing",
+			agreeWithProposedOutput: false,
+			callResolveStatus:       types.GameStatusChallengerWon,
+			shouldResolve:           false,
+		},
+		{
+			name:                    "Disagree_Winning",
+			agreeWithProposedOutput: false,
+			callResolveStatus:       types.GameStatusDefenderWon,
+			shouldResolve:           true,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			agent, claimLoader, responder := setupTestAgent(t, test.agreeWithProposedOutput)
+			responder.callResolveStatus = test.callResolveStatus
+
+			require.NoError(t, agent.Act(ctx))
+
+			require.Equal(t, 1, responder.callResolveCount, "should check if game is resolvable")
+			require.Zero(t, claimLoader.callCount, "should not fetch claims for resolvable game")
+
+			if test.shouldResolve {
+				require.EqualValues(t, 1, responder.resolveCount, "should resolve winning game")
+			} else {
+				require.Zero(t, responder.resolveCount, "should not resolve losing game")
+			}
+		})
+	}
+}
+
+func TestLoadClaimsWhenGameNotResolvable(t *testing.T) {
+	// Checks that if the game isn't resolvable, that the agent continues on to start checking claims
+	agent, claimLoader, responder := setupTestAgent(t, false)
+	responder.callResolveErr = errors.New("game is not resolvable")
+	depth := 4
+	claimBuilder := test.NewClaimBuilder(t, depth, alphabet.NewTraceProvider("abcdefg", uint64(depth)))
+
+	claimLoader.claims = []types.Claim{
+		claimBuilder.CreateRootClaim(true),
+	}
+
+	require.NoError(t, agent.Act(context.Background()))
+
+	require.EqualValues(t, 1, claimLoader.callCount, "should load claims for unresolvable game")
+}
+
+func setupTestAgent(t *testing.T, agreeWithProposedOutput bool) (*Agent, *stubClaimLoader, *stubResponder) {
+	logger := testlog.Logger(t, log.LvlInfo)
+	claimLoader := &stubClaimLoader{}
+	depth := 4
+	trace := alphabet.NewTraceProvider("abcd", uint64(depth))
+	responder := &stubResponder{}
+	updater := &stubUpdater{}
+	agent := NewAgent(metrics.NoopMetrics, claimLoader, depth, trace, responder, updater, agreeWithProposedOutput, logger)
+	return agent, claimLoader, responder
+}
+
+type stubClaimLoader struct {
+	callCount int
+	claims    []types.Claim
+}
+
+func (s *stubClaimLoader) FetchClaims(ctx context.Context) ([]types.Claim, error) {
+	s.callCount++
+	return s.claims, nil
+}
+
+type stubResponder struct {
+	callResolveCount  int
+	callResolveStatus types.GameStatus
+	callResolveErr    error
+
+	resolveCount int
+	resolveErr   error
+}
+
+func (s *stubResponder) CallResolve(ctx context.Context) (types.GameStatus, error) {
+	s.callResolveCount++
+	return s.callResolveStatus, s.callResolveErr
+}
+
+func (s *stubResponder) Resolve(ctx context.Context) error {
+	s.resolveCount++
+	return s.resolveErr
+}
+
+func (s *stubResponder) Respond(ctx context.Context, response types.Claim) error {
+	panic("Not implemented")
+}
+
+func (s *stubResponder) Step(ctx context.Context, stepData types.StepCallData) error {
+	panic("Not implemented")
+}
+
+type stubUpdater struct {
+}
+
+func (s *stubUpdater) UpdateOracle(ctx context.Context, data *types.PreimageOracleData) error {
+	panic("Not implemented")
+}

op-challenger/game/fault/player.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/alphabet"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/cannon"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum-optimism/optimism/op-service/txmgr"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
@@ -37,6 +38,7 @@ type GamePlayer struct {
 func NewGamePlayer(
 	ctx context.Context,
 	logger log.Logger,
+	m metrics.Metricer,
 	cfg *config.Config,
 	dir string,
 	addr common.Address,
@@ -79,7 +81,7 @@ func NewGamePlayer(
 	var updater types.OracleUpdater
 	switch cfg.TraceType {
 	case config.TraceTypeCannon:
-		cannonProvider, err := cannon.NewTraceProvider(ctx, logger, cfg, client, dir, addr)
+		cannonProvider, err := cannon.NewTraceProvider(ctx, logger, m, cfg, client, dir, addr)
 		if err != nil {
 			return nil, fmt.Errorf("create cannon trace provider: %w", err)
 		}
@@ -105,7 +107,7 @@ func NewGamePlayer(
 	}
 
 	return &GamePlayer{
-		act:                     NewAgent(loader, int(gameDepth), provider, responder, updater, cfg.AgreeWithProposedOutput, logger).Act,
+		act:                     NewAgent(m, loader, int(gameDepth), provider, responder, updater, cfg.AgreeWithProposedOutput, logger).Act,
 		agreeWithProposedOutput: cfg.AgreeWithProposedOutput,
 		loader:                  loader,
 		logger:                  logger,

op-challenger/game/fault/trace/cannon/executor.go

@@ -11,6 +11,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/ethereum-optimism/optimism/op-challenger/config"
 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
@@ -30,6 +31,7 @@ type cmdExecutor func(ctx context.Context, l log.Logger, binary string, args ...
 
 type Executor struct {
 	logger           log.Logger
+	metrics          CannonMetricer
 	l1               string
 	l2               string
 	inputs           LocalGameInputs
@@ -45,9 +47,10 @@ type Executor struct {
 	cmdExecutor      cmdExecutor
 }
 
-func NewExecutor(logger log.Logger, cfg *config.Config, inputs LocalGameInputs) *Executor {
+func NewExecutor(logger log.Logger, m CannonMetricer, cfg *config.Config, inputs LocalGameInputs) *Executor {
 	return &Executor{
 		logger:           logger,
+		metrics:          m,
 		l1:               cfg.L1EthRpc,
 		l2:               cfg.CannonL2,
 		inputs:           inputs,
@@ -119,7 +122,13 @@ func (e *Executor) GenerateProof(ctx context.Context, dir string, i uint64) erro
 		return fmt.Errorf("could not create proofs directory %v: %w", proofDir, err)
 	}
 	e.logger.Info("Generating trace", "proof", i, "cmd", e.cannon, "args", strings.Join(args, ", "))
-	return e.cmdExecutor(ctx, e.logger.New("proof", i), e.cannon, args...)
+	execStart := time.Now()
+	err = e.cmdExecutor(ctx, e.logger.New("proof", i), e.cannon, args...)
+	if err != nil {
+		execDuration := time.Since(execStart).Seconds()
+		e.metrics.RecordCannonExecutionTime(execDuration)
+	}
+	return err
 }
 
 func runCmd(ctx context.Context, l log.Logger, binary string, args ...string) error {

op-challenger/game/fault/trace/cannon/executor_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/ethereum-optimism/optimism/op-challenger/config"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum-optimism/optimism/op-node/testlog"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
@@ -39,7 +40,7 @@ func TestGenerateProof(t *testing.T) {
 		L2BlockNumber: big.NewInt(3333),
 	}
 	captureExec := func(t *testing.T, cfg config.Config, proofAt uint64) (string, string, map[string]string) {
-		executor := NewExecutor(testlog.Logger(t, log.LvlInfo), &cfg, inputs)
+		executor := NewExecutor(testlog.Logger(t, log.LvlInfo), metrics.NoopMetrics, &cfg, inputs)
 		executor.selectSnapshot = func(logger log.Logger, dir string, absolutePreState string, i uint64) (string, error) {
 			return input, nil
 		}

op-challenger/game/fault/trace/cannon/provider.go

@@ -33,6 +33,10 @@ type proofData struct {
 	OracleOffset uint32        `json:"oracle-offset,omitempty"`
 }
 
+type CannonMetricer interface {
+	RecordCannonExecutionTime(t float64)
+}
+
 type ProofGenerator interface {
 	// GenerateProof executes cannon to generate a proof at the specified trace index in dataDir.
 	GenerateProof(ctx context.Context, dataDir string, proofAt uint64) error
@@ -51,7 +55,7 @@ type CannonTraceProvider struct {
 	lastProof *proofData
 }
 
-func NewTraceProvider(ctx context.Context, logger log.Logger, cfg *config.Config, l1Client bind.ContractCaller, dir string, gameAddr common.Address) (*CannonTraceProvider, error) {
+func NewTraceProvider(ctx context.Context, logger log.Logger, m CannonMetricer, cfg *config.Config, l1Client bind.ContractCaller, dir string, gameAddr common.Address) (*CannonTraceProvider, error) {
 	l2Client, err := ethclient.DialContext(ctx, cfg.CannonL2)
 	if err != nil {
 		return nil, fmt.Errorf("dial l2 client %v: %w", cfg.CannonL2, err)
@@ -65,15 +69,15 @@ func NewTraceProvider(ctx context.Context, logger log.Logger, cfg *config.Config
 	if err != nil {
 		return nil, fmt.Errorf("fetch local game inputs: %w", err)
 	}
-	return NewTraceProviderFromInputs(logger, cfg, localInputs, dir), nil
+	return NewTraceProviderFromInputs(logger, m, cfg, localInputs, dir), nil
 }
 
-func NewTraceProviderFromInputs(logger log.Logger, cfg *config.Config, localInputs LocalGameInputs, dir string) *CannonTraceProvider {
+func NewTraceProviderFromInputs(logger log.Logger, m CannonMetricer, cfg *config.Config, localInputs LocalGameInputs, dir string) *CannonTraceProvider {
 	return &CannonTraceProvider{
 		logger:    logger,
 		dir:       dir,
 		prestate:  cfg.CannonAbsolutePreState,
-		generator: NewExecutor(logger, cfg, localInputs),
+		generator: NewExecutor(logger, m, cfg, localInputs),
 	}
 }
 

op-challenger/game/monitor.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/ethereum-optimism/optimism/op-challenger/game/scheduler"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum-optimism/optimism/op-service/clock"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
@@ -26,6 +27,7 @@ type gameScheduler interface {
 
 type gameMonitor struct {
 	logger           log.Logger
+	metrics          metrics.Metricer
 	clock            clock.Clock
 	source           gameSource
 	scheduler        gameScheduler
@@ -36,6 +38,7 @@ type gameMonitor struct {
 
 func newGameMonitor(
 	logger log.Logger,
+	m metrics.Metricer,
 	cl clock.Clock,
 	source gameSource,
 	scheduler gameScheduler,
@@ -45,6 +48,7 @@ func newGameMonitor(
 ) *gameMonitor {
 	return &gameMonitor{
 		logger:           logger,
+		metrics:          m,
 		clock:            cl,
 		scheduler:        scheduler,
 		source:           source,

op-challenger/game/monitor_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum-optimism/optimism/op-node/testlog"
 	"github.com/ethereum-optimism/optimism/op-service/clock"
 	"github.com/ethereum/go-ethereum/common"
@@ -100,7 +101,7 @@ func setupMonitorTest(t *testing.T, allowedGames []common.Address) (*gameMonitor
 		return i, nil
 	}
 	sched := &stubScheduler{}
-	monitor := newGameMonitor(logger, clock.SystemClock, source, sched, time.Duration(0), fetchBlockNum, allowedGames)
+	monitor := newGameMonitor(logger, metrics.NoopMetrics, clock.SystemClock, source, sched, time.Duration(0), fetchBlockNum, allowedGames)
 	return monitor, source, sched
 }
 

op-challenger/game/service.go

@@ -72,10 +72,10 @@ func NewService(ctx context.Context, logger log.Logger, cfg *config.Config) (*Se
 		disk,
 		cfg.MaxConcurrency,
 		func(addr common.Address, dir string) (scheduler.GamePlayer, error) {
-			return fault.NewGamePlayer(ctx, logger, cfg, dir, addr, txMgr, client)
+			return fault.NewGamePlayer(ctx, logger, m, cfg, dir, addr, txMgr, client)
 		})
 
-	monitor := newGameMonitor(logger, cl, loader, sched, cfg.GameWindow, client.BlockNumber, cfg.GameAllowlist)
+	monitor := newGameMonitor(logger, m, cl, loader, sched, cfg.GameWindow, client.BlockNumber, cfg.GameAllowlist)
 
 	m.RecordInfo(version.SimpleWithMeta)
 	m.RecordUp()

op-challenger/metrics/metrics.go

@@ -20,6 +20,10 @@ type Metricer interface {
 
 	// Record Tx metrics
 	txmetrics.TxMetricer
+
+	RecordGameStep()
+	RecordGameMove()
+	RecordCannonExecutionTime(t float64)
 }
 
 type Metrics struct {
@@ -31,6 +35,10 @@ type Metrics struct {
 
 	info prometheus.GaugeVec
 	up   prometheus.Gauge
+
+	moves               prometheus.Counter
+	steps               prometheus.Counter
+	cannonExecutionTime prometheus.Histogram
 }
 
 var _ Metricer = (*Metrics)(nil)
@@ -58,6 +66,22 @@ func NewMetrics() *Metrics {
 			Name:      "up",
 			Help:      "1 if the op-challenger has finished starting up",
 		}),
+		moves: factory.NewCounter(prometheus.CounterOpts{
+			Namespace: Namespace,
+			Name:      "moves",
+			Help:      "Number of game moves made by the challenge agent",
+		}),
+		steps: factory.NewCounter(prometheus.CounterOpts{
+			Namespace: Namespace,
+			Name:      "steps",
+			Help:      "Number of game steps made by the challenge agent",
+		}),
+		cannonExecutionTime: factory.NewHistogram(prometheus.HistogramOpts{
+			Namespace: Namespace,
+			Name:      "cannon_execution_time",
+			Help:      "Time (in seconds) to execute cannon",
+			Buckets:   append([]float64{1.0, 10.0}, prometheus.ExponentialBuckets(30.0, 2.0, 14)...),
+		}),
 	}
 }
 
@@ -84,3 +108,15 @@ func (m *Metrics) RecordUp() {
 func (m *Metrics) Document() []opmetrics.DocumentedMetric {
 	return m.factory.Document()
 }
+
+func (m *Metrics) RecordGameMove() {
+	m.moves.Add(1)
+}
+
+func (m *Metrics) RecordGameStep() {
+	m.steps.Add(1)
+}
+
+func (m *Metrics) RecordCannonExecutionTime(t float64) {
+	m.cannonExecutionTime.Observe(t)
+}

op-challenger/metrics/noop.go

@@ -10,5 +10,8 @@ type noopMetrics struct {
 
 var NoopMetrics Metricer = new(noopMetrics)
 
-func (*noopMetrics) RecordInfo(version string) {}
-func (*noopMetrics) RecordUp()                 {}
+func (*noopMetrics) RecordInfo(version string)           {}
+func (*noopMetrics) RecordUp()                           {}
+func (*noopMetrics) RecordGameMove()                     {}
+func (*noopMetrics) RecordGameStep()                     {}
+func (*noopMetrics) RecordCannonExecutionTime(t float64) {}

op-challenger/scripts/alphabet/init_game.sh

@@ -18,7 +18,7 @@ DEVNET_SPONSOR="ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
 DISPUTE_GAME_FACTORY=$(jq -r .DisputeGameFactoryProxy $MONOREPO_DIR/.devnet/addresses.json)
 
 echo "----------------------------------------------------------------"
-echo " Dispute Game Factory at $DISPUTE_GAME_PROXY"
+echo " Dispute Game Factory at $DISPUTE_GAME_FACTORY"
 echo "----------------------------------------------------------------"
 
 L2_OUTPUT_ORACLE_PROXY=$(jq -r .L2OutputOracleProxy $MONOREPO_DIR/.devnet/addresses.json)

op-challenger/scripts/list_games.sh

+#!/usr/bin/env bash
+
+set -euo pipefail
+
+RPC=${1:?Must specify RPC address}
+FACTORY_ADDR=${2:?Must specify dispute game factory address}
+
+COUNT=$(cast call --rpc-url "${RPC}" "${FACTORY_ADDR}" 'gameCount() returns(uint256)')
+echo "Game count: ${COUNT}"
+if [[ "${COUNT}" == "0" ]]
+then
+  exit
+fi
+((COUNT=COUNT-1))
+for i in $(seq 0 "${COUNT}")
+do
+  GAME=$(cast call --rpc-url "${RPC}" "${FACTORY_ADDR}" 'gameAtIndex(uint256) returns(uint8, uint64, address)' "${i}")
+
+  SAVEIFS=$IFS   # Save current IFS (Internal Field Separator)
+  IFS=$'\n'      # Change IFS to newline char
+  GAME=($GAME) # split the string into an array by the same name
+  IFS=$SAVEIFS   # Restore original IFS
+
+  GAME_ADDR="${GAME[2]}"
+  STATUS=$(cast call --rpc-url "${RPC}" "${GAME_ADDR}" "status() return(uint8)" | cast to-dec)
+  if [[ "${STATUS}" == "0" ]]
+  then
+    STATUS="In Progress"
+  elif [[ "${STATUS}" == "1" ]]
+  then
+    STATUS="Challenger Wins"
+  elif [[ "${STATUS}" == "2" ]]
+  then
+    STATUS="Defender Wins"
+  fi
+  echo "${i}  Game: ${GAME_ADDR} Type: ${GAME[0]} Created: ${GAME[1]} Status: ${STATUS}"
+done

op-challenger/scripts/resolve.sh

+#!/bin/bash
+set -euo pipefail
+
+RPC=${1:?Must specify RPC URL}
+GAME_ADDR=${2:?Must specify game address}
+SIGNER_ARGS="${@:3}"
+
+# Perform the move.
+RESULT_DATA=$(cast send --rpc-url "${RPC}" ${SIGNER_ARGS} "${GAME_ADDR}" "resolve()" --json)
+RESULT=$(echo "${RESULT_DATA}" | jq -r '.logs[0].topics[1]' | cast to-dec)
+
+if [[ "${RESULT}" == "0" ]]
+then
+  RESULT="In Progress"
+elif [[ "${RESULT}" == "1" ]]
+then
+  RESULT="Challenger Wins"
+elif [[ "${RESULT}" == "2" ]]
+then
+  RESULT="Defender Wins"
+fi
+
+echo "Result: $RESULT"

op-e2e/e2eutils/disputegame/cannon_helper.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/cannon"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/challenger"
 	"github.com/ethereum-optimism/optimism/op-node/rollup"
 	"github.com/ethereum-optimism/optimism/op-node/testlog"
@@ -40,7 +41,7 @@ func (g *CannonGameHelper) CreateHonestActor(ctx context.Context, rollupCfg *rol
 	opts = append(opts, options...)
 	cfg := challenger.NewChallengerConfig(g.t, l1Endpoint, opts...)
 	logger := testlog.Logger(g.t, log.LvlInfo).New("role", "CorrectTrace")
-	provider, err := cannon.NewTraceProvider(ctx, logger, cfg, l1Client, filepath.Join(cfg.Datadir, "honest"), g.addr)
+	provider, err := cannon.NewTraceProvider(ctx, logger, metrics.NoopMetrics, cfg, l1Client, filepath.Join(cfg.Datadir, "honest"), g.addr)
 	g.require.NoError(err, "create cannon trace provider")
 
 	return &HonestHelper{

op-e2e/e2eutils/disputegame/helper.go

@@ -14,6 +14,7 @@ import (
 	"github.com/ethereum-optimism/optimism/op-chain-ops/genesis"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/alphabet"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/cannon"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/challenger"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/transactions"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait"
@@ -175,7 +176,7 @@ func (h *FactoryHelper) StartCannonGameWithCorrectRoot(ctx context.Context, roll
 		L2Claim:       challengedOutput.OutputRoot,
 		L2BlockNumber: challengedOutput.L2BlockNumber,
 	}
-	provider := cannon.NewTraceProviderFromInputs(testlog.Logger(h.t, log.LvlInfo).New("role", "CorrectTrace"), cfg, inputs, cfg.Datadir)
+	provider := cannon.NewTraceProviderFromInputs(testlog.Logger(h.t, log.LvlInfo).New("role", "CorrectTrace"), metrics.NoopMetrics, cfg, inputs, cfg.Datadir)
 	rootClaim, err := provider.Get(ctx, math.MaxUint64)
 	h.require.NoError(err, "Compute correct root hash")
 

op-program/host/prefetcher/prefetcher.go

@@ -58,14 +58,18 @@ func (p *Prefetcher) Hint(hint string) error {
 func (p *Prefetcher) GetPreimage(ctx context.Context, key common.Hash) ([]byte, error) {
 	p.logger.Trace("Pre-image requested", "key", key)
 	pre, err := p.kvStore.Get(key)
-	if errors.Is(err, kvstore.ErrNotFound) && p.lastHint != "" {
+	// Use a loop to keep retrying the prefetch as long as the key is not found
+	// This handles the case where the prefetch downloads a preimage, but it is then deleted unexpectedly
+	// before we get to read it.
+	for errors.Is(err, kvstore.ErrNotFound) && p.lastHint != "" {
 		hint := p.lastHint
-		p.lastHint = ""
 		if err := p.prefetch(ctx, hint); err != nil {
 			return nil, fmt.Errorf("prefetch failed: %w", err)
 		}
-		// Should now be available
-		return p.kvStore.Get(key)
+		pre, err = p.kvStore.Get(key)
+		if err != nil {
+			p.logger.Error("Fetched pre-images for last hint but did not find required key", "hint", hint, "key", key)
+		}
 	}
 	return pre, err
 }

op-program/host/prefetcher/prefetcher_test.go

@@ -306,6 +306,41 @@ func TestBadHints(t *testing.T) {
 	})
 }
 
+func TestRetryWhenNotAvailableAfterPrefetching(t *testing.T) {
+	rng := rand.New(rand.NewSource(123))
+	node := testutils.RandomData(rng, 30)
+	hash := crypto.Keccak256Hash(node)
+
+	_, l1Source, l2Cl, kv := createPrefetcher(t)
+	putsToIgnore := 2
+	kv = &unreliableKvStore{KV: kv, putsToIgnore: putsToIgnore}
+	prefetcher := NewPrefetcher(testlog.Logger(t, log.LvlInfo), l1Source, l2Cl, kv)
+
+	// Expect one call for each ignored put, plus one more request for when the put succeeds
+	for i := 0; i < putsToIgnore+1; i++ {
+		l2Cl.ExpectNodeByHash(hash, node, nil)
+	}
+	defer l2Cl.MockDebugClient.AssertExpectations(t)
+
+	oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+	result := oracle.NodeByHash(hash)
+	require.EqualValues(t, node, result)
+}
+
+type unreliableKvStore struct {
+	kvstore.KV
+	putsToIgnore int
+}
+
+func (s *unreliableKvStore) Put(k common.Hash, v []byte) error {
+	if s.putsToIgnore > 0 {
+		s.putsToIgnore--
+		return nil
+	}
+	println("storing")
+	return s.KV.Put(k, v)
+}
+
 type l2Client struct {
 	*testutils.MockL2Client
 	*testutils.MockDebugClient

op-ufm/cmd/ufm/main.go

@@ -28,7 +28,10 @@ func main() {
 		),
 	)
 
-	log.Info("initializing", "version", GitVersion, "commit", GitCommit, "date", GitDate)
+	log.Info("initializing",
+		"version", GitVersion,
+		"commit", GitCommit,
+		"date", GitDate)
 
 	if len(os.Args) < 2 {
 		log.Crit("must specify a config file on the command line")
@@ -42,7 +45,8 @@ func main() {
 	sig := make(chan os.Signal, 1)
 	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
 	recvSig := <-sig
-	log.Info("caught signal, shutting down", "signal", recvSig)
+	log.Info("caught signal, shutting down",
+		"signal", recvSig)
 
 	svc.Shutdown()
 }
@@ -50,7 +54,9 @@ func main() {
 func initConfig(cfgFile string) *config.Config {
 	cfg, err := config.New(cfgFile)
 	if err != nil {
-		log.Crit("error reading config file", "file", cfgFile, "err", err)
+		log.Crit("error reading config file",
+			"file", cfgFile,
+			"err", err)
 	}
 
 	// update log level from config
@@ -58,7 +64,8 @@ func initConfig(cfgFile string) *config.Config {
 	if err != nil {
 		logLevel = log.LvlInfo
 		if cfg.LogLevel != "" {
-			log.Warn("invalid server.log_level set: " + cfg.LogLevel)
+			log.Warn("invalid server.log_level",
+				"log_level", cfg.LogLevel)
 		}
 	}
 	log.Root().SetHandler(
@@ -74,7 +81,8 @@ func initConfig(cfgFile string) *config.Config {
 
 	err = cfg.Validate()
 	if err != nil {
-		log.Crit("invalid config", "err", err)
+		log.Crit("invalid config",
+			"err", err)
 	}
 
 	return cfg

op-ufm/pkg/metrics/metrics.go

@@ -84,8 +84,10 @@ var nonAlphanumericRegex = regexp.MustCompile(`[^a-zA-Z ]+`)
 
 func RecordError(provider string, errorLabel string) {
 	if Debug {
-		log.Debug("metric inc", "m", "errors_total",
-			"provider", provider, "error", errorLabel)
+		log.Debug("metric inc",
+			"m", "errors_total",
+			"provider", provider,
+			"error", errorLabel)
 	}
 	errorsTotal.WithLabelValues(provider, errorLabel).Inc()
 }
@@ -101,48 +103,64 @@ func RecordErrorDetails(provider string, label string, err error) {
 
 func RecordRPCLatency(provider string, client string, method string, latency time.Duration) {
 	if Debug {
-		log.Debug("metric set", "m", "rpc_latency",
-			"provider", provider, "client", client, "method", method, "latency", latency)
+		log.Debug("metric set",
+			"m", "rpc_latency",
+			"provider", provider,
+			"client", client,
+			"method", method,
+			"latency", latency)
 	}
 	rpcLatency.WithLabelValues(provider, client, method).Set(float64(latency.Milliseconds()))
 }
 
 func RecordRoundTripLatency(provider string, latency time.Duration) {
 	if Debug {
-		log.Debug("metric set", "m", "roundtrip_latency",
-			"provider", provider, "latency", latency)
+		log.Debug("metric set",
+			"m", "roundtrip_latency",
+			"provider", provider,
+			"latency", latency)
 	}
 	roundTripLatency.WithLabelValues(provider).Set(float64(latency.Milliseconds()))
 }
 
 func RecordGasUsed(provider string, val uint64) {
 	if Debug {
-		log.Debug("metric add", "m", "gas_used",
-			"provider", provider, "val", val)
+		log.Debug("metric add",
+			"m", "gas_used",
+			"provider", provider,
+			"val", val)
 	}
 	gasUsed.WithLabelValues(provider).Set(float64(val))
 }
 
 func RecordFirstSeenLatency(providerSource string, providerSeen string, latency time.Duration) {
 	if Debug {
-		log.Debug("metric set", "m", "first_seen_latency",
-			"provider_source", providerSource, "provider_seen", providerSeen, "latency", latency)
+		log.Debug("metric set",
+			"m", "first_seen_latency",
+			"provider_source", providerSource,
+			"provider_seen", providerSeen,
+			"latency", latency)
 	}
 	firstSeenLatency.WithLabelValues(providerSource, providerSeen).Set(float64(latency.Milliseconds()))
 }
 
 func RecordProviderToProviderLatency(providerSource string, providerSeen string, latency time.Duration) {
 	if Debug {
-		log.Debug("metric set", "m", "provider_to_provider_latency",
-			"provider_source", providerSource, "provider_seen", providerSeen, "latency", latency)
+		log.Debug("metric set",
+			"m", "provider_to_provider_latency",
+			"provider_source", providerSource,
+			"provider_seen", providerSeen,
+			"latency", latency)
 	}
 	providerToProviderLatency.WithLabelValues(providerSource, providerSeen).Set(float64(latency.Milliseconds()))
 }
 
 func RecordTransactionsInFlight(network string, count int) {
 	if Debug {
-		log.Debug("metric set", "m", "transactions_inflight",
-			"network", network, "count", count)
+		log.Debug("metric set",
+			"m", "transactions_inflight",
+			"network", network,
+			"count", count)
 	}
 	networkTransactionsInFlight.WithLabelValues(network).Set(float64(count))
 }

op-ufm/pkg/provider/heartbeat.go

@@ -14,7 +14,9 @@ import (
 
 // Heartbeat polls for expected in-flight transactions
 func (p *Provider) Heartbeat(ctx context.Context) {
-	log.Debug("heartbeat", "provider", p.name, "inflight", len(p.txPool.Transactions))
+	log.Debug("heartbeat",
+		"provider", p.name,
+		"count", len(p.txPool.Transactions))
 
 	metrics.RecordTransactionsInFlight(p.config.Network, len(p.txPool.Transactions))
 
@@ -33,26 +35,42 @@ func (p *Provider) Heartbeat(ctx context.Context) {
 	}
 
 	if len(expectedTransactions) == 0 {
-		log.Debug("no expected txs", "count", len(p.txPool.Transactions), "provider", p.name, "alreadySeen", alreadySeen)
+		log.Debug("no expected txs",
+			"count", len(p.txPool.Transactions),
+			"provider", p.name,
+			"alreadySeen", alreadySeen)
 		return
 	}
 
 	client, err := clients.Dial(p.name, p.config.URL)
 	if err != nil {
-		log.Error("cant dial to provider", "provider", p.name, "url", p.config.URL, "err", err)
+		log.Error("cant dial to provider",
+			"provider", p.name,
+			"url", p.config.URL,
+			"err", err)
 	}
 
-	log.Debug("checking in-flight tx", "count", len(p.txPool.Transactions), "provider", p.name, "alreadySeen", alreadySeen)
+	log.Debug("checking in-flight tx",
+		"count", len(p.txPool.Transactions),
+		"provider", p.name,
+		"alreadySeen", alreadySeen)
 	for _, st := range expectedTransactions {
 		hash := st.Hash.Hex()
 
 		_, isPending, err := client.TransactionByHash(ctx, st.Hash)
 		if err != nil && !errors.Is(err, ethereum.NotFound) {
-			log.Error("cant check transaction", "provider", p.name, "hash", hash, "url", p.config.URL, "err", err)
+			log.Error("cant check transaction",
+				"provider", p.name,
+				"hash", hash,
+				"url", p.config.URL,
+				"err", err)
 			continue
 		}
 
-		log.Debug("got transaction", "provider", p.name, "hash", hash, "isPending", isPending)
+		log.Debug("got transaction",
+			"provider", p.name,
+			"hash", hash,
+			"isPending", isPending)
 
 		// mark transaction as seen by this provider
 		st.M.Lock()
@@ -75,7 +93,10 @@ func (p *Provider) Heartbeat(ctx context.Context) {
 		// check if transaction have been seen by all providers
 		p.txPool.M.Lock()
 		if len(st.SeenBy) == p.txPool.Expected {
-			log.Debug("transaction seen by all", "hash", hash, "expected", p.txPool.Expected, "seenBy", len(st.SeenBy))
+			log.Debug("transaction seen by all",
+				"hash", hash,
+				"expected", p.txPool.Expected,
+				"seenBy", len(st.SeenBy))
 			delete(p.txPool.Transactions, st.Hash.Hex())
 		}
 		p.txPool.M.Unlock()

op-ufm/pkg/provider/roundtrip.go

@@ -6,11 +6,11 @@ import (
 
 	"github.com/ethereum-optimism/optimism/op-ufm/pkg/metrics"
 	iclients "github.com/ethereum-optimism/optimism/op-ufm/pkg/metrics/clients"
+	"github.com/ethereum/go-ethereum/core"
 
 	"github.com/ethereum-optimism/optimism/op-service/tls"
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/txpool"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/pkg/errors"
@@ -21,19 +21,35 @@ import (
 
 // RoundTrip send a new transaction to measure round trip latency
 func (p *Provider) RoundTrip(ctx context.Context) {
-	log.Debug("roundTripLatency", "provider", p.name)
+	log.Debug("roundTripLatency",
+		"provider", p.name)
 
 	client, err := iclients.Dial(p.name, p.config.URL)
 	if err != nil {
-		log.Error("cant dial to provider", "provider", p.name, "url", p.config.URL, "err", err)
+		log.Error("cant dial to provider",
+			"provider", p.name,
+			"url", p.config.URL,
+			"err", err)
 		return
 	}
 
-	nonce, err := client.PendingNonceAt(ctx, p.walletConfig.Address)
-	if err != nil {
-		log.Error("cant get nounce", "provider", p.name, "err", err)
-		return
+	var nonce uint64
+	p.txPool.M.Lock()
+	if p.txPool.Nonce == uint64(0) {
+		nonce, err = client.PendingNonceAt(ctx, p.walletConfig.Address)
+		if err != nil {
+			log.Error("cant get nounce",
+				"provider", p.name,
+				"err", err)
+			p.txPool.M.Unlock()
+			return
+		}
+		p.txPool.Nonce = nonce
+	} else {
+		p.txPool.Nonce++
+		nonce = p.txPool.Nonce
 	}
+	p.txPool.M.Unlock()
 
 	txHash := common.Hash{}
 	attempt := 0
@@ -47,7 +63,10 @@ func (p *Provider) RoundTrip(ctx context.Context) {
 
 		signedTx, err := p.sign(ctx, tx)
 		if err != nil {
-			log.Error("cant sign tx", "provider", p.name, "tx", tx, "err", err)
+			log.Error("cant sign tx",
+				"provider", p.name,
+				"tx", tx,
+				"err", err)
 			return
 		}
 
@@ -56,21 +75,40 @@ func (p *Provider) RoundTrip(ctx context.Context) {
 		roundTripStartedAt = time.Now()
 		err = client.SendTransaction(ctx, signedTx)
 		if err != nil {
-			if err.Error() == txpool.ErrAlreadyKnown.Error() || err.Error() == core.ErrNonceTooLow.Error() {
+			if err.Error() == txpool.ErrAlreadyKnown.Error() ||
+				err.Error() == txpool.ErrReplaceUnderpriced.Error() ||
+				err.Error() == core.ErrNonceTooLow.Error() {
 				if time.Since(firstAttemptAt) >= time.Duration(p.config.SendTransactionRetryTimeout) {
-					log.Error("send transaction timed out (known already)", "provider", p.name, "hash", txHash.Hex(), "elapsed", time.Since(firstAttemptAt), "attempt", attempt, "nonce", nonce)
+					log.Error("send transaction timed out (known already)",
+						"provider", p.name,
+						"hash", txHash.Hex(),
+						"elapsed", time.Since(firstAttemptAt),
+						"attempt", attempt,
+						"nonce", nonce)
 					metrics.RecordError(p.name, "ethclient.SendTransaction.nonce")
 					return
 				}
-				log.Warn("tx already known, incrementing nonce and trying again", "provider", p.name, "nonce", nonce)
+				log.Warn("tx already known, incrementing nonce and trying again",
+					"provider", p.name,
+					"nonce", nonce)
 				time.Sleep(time.Duration(p.config.SendTransactionRetryInterval))
-				nonce++
+
+				p.txPool.M.Lock()
+				p.txPool.Nonce++
+				nonce = p.txPool.Nonce
+				p.txPool.M.Unlock()
 				attempt++
 				if attempt%10 == 0 {
-					log.Debug("retrying send transaction...", "provider", p.name, "attempt", attempt, "nonce", nonce, "elapsed", time.Since(firstAttemptAt))
+					log.Debug("retrying send transaction...",
+						"provider", p.name,
+						"attempt", attempt,
+						"nonce", nonce,
+						"elapsed", time.Since(firstAttemptAt))
 				}
 			} else {
-				log.Error("cant send transaction", "provider", p.name, "err", err)
+				log.Error("cant send transaction",
+					"provider", p.name,
+					"err", err)
 				metrics.RecordErrorDetails(p.name, "ethclient.SendTransaction", err)
 				return
 			}
@@ -79,7 +117,10 @@ func (p *Provider) RoundTrip(ctx context.Context) {
 		}
 	}
 
-	log.Info("transaction sent", "provider", p.name, "hash", txHash.Hex(), "nonce", nonce)
+	log.Info("transaction sent",
+		"provider", p.name,
+		"hash", txHash.Hex(),
+		"nonce", nonce)
 
 	// add to pool
 	sentAt := time.Now()
@@ -96,16 +137,25 @@ func (p *Provider) RoundTrip(ctx context.Context) {
 	attempt = 0
 	for receipt == nil {
 		if time.Since(sentAt) >= time.Duration(p.config.ReceiptRetrievalTimeout) {
-			log.Error("receipt retrieval timed out", "provider", p.name, "hash", "elapsed", time.Since(sentAt))
+			log.Error("receipt retrieval timed out",
+				"provider", p.name,
+				"hash", txHash,
+				"elapsed", time.Since(sentAt))
 			return
 		}
 		time.Sleep(time.Duration(p.config.ReceiptRetrievalInterval))
 		if attempt%10 == 0 {
-			log.Debug("checking for receipt...", "provider", p.name, "attempt", attempt, "elapsed", time.Since(sentAt))
+			log.Debug("checking for receipt...",
+				"provider", p.name,
+				"attempt", attempt,
+				"elapsed", time.Since(sentAt))
 		}
 		receipt, err = client.TransactionReceipt(ctx, txHash)
 		if err != nil && !errors.Is(err, ethereum.NotFound) {
-			log.Error("cant get receipt for transaction", "provider", p.name, "hash", txHash.Hex(), "err", err)
+			log.Error("cant get receipt for transaction",
+				"provider", p.name,
+				"hash", txHash.Hex(),
+				"err", err)
 			return
 		}
 		attempt++
@@ -116,7 +166,8 @@ func (p *Provider) RoundTrip(ctx context.Context) {
 	metrics.RecordRoundTripLatency(p.name, roundTripLatency)
 	metrics.RecordGasUsed(p.name, receipt.GasUsed)
 
-	log.Info("got transaction receipt", "hash", txHash.Hex(),
+	log.Info("got transaction receipt",
+		"hash", txHash.Hex(),
 		"roundTripLatency", roundTripLatency,
 		"provider", p.name,
 		"blockNumber", receipt.BlockNumber,
@@ -155,7 +206,9 @@ func (p *Provider) sign(ctx context.Context, tx *types.Transaction) (*types.Tran
 			TLSKey:    p.signerConfig.TLSKey,
 		}
 		client, err := iclients.NewSignerClient(p.name, log.Root(), p.signerConfig.URL, tlsConfig)
-		log.Debug("signerclient", "client", client, "err", err)
+		log.Debug("signerclient",
+			"client", client,
+			"err", err)
 		if err != nil {
 			return nil, err
 		}

op-ufm/pkg/provider/tx_pool.go

@@ -15,6 +15,7 @@ type NetworkTransactionPool struct {
 	M            sync.Mutex
 	Transactions map[string]*TransactionState
 	Expected     int
+	Nonce        uint64
 }
 
 type TransactionState struct {

op-ufm/pkg/service/service.go

@@ -32,10 +32,12 @@ func (s *Service) Start(ctx context.Context) {
 	log.Info("service starting")
 	if s.Config.Healthz.Enabled {
 		addr := net.JoinHostPort(s.Config.Healthz.Host, s.Config.Healthz.Port)
-		log.Info("starting healthz server", "addr", addr)
+		log.Info("starting healthz server",
+			"addr", addr)
 		go func() {
 			if err := s.Healthz.Start(ctx, addr); err != nil {
-				log.Error("error starting healthz server", "err", err)
+				log.Error("error starting healthz server",
+					"err", err)
 			}
 		}()
 	}
@@ -43,10 +45,12 @@ func (s *Service) Start(ctx context.Context) {
 	metrics.Debug = s.Config.Metrics.Debug
 	if s.Config.Metrics.Enabled {
 		addr := net.JoinHostPort(s.Config.Metrics.Host, s.Config.Metrics.Port)
-		log.Info("starting metrics server", "addr", addr)
+		log.Info("starting metrics server",
+			"addr", addr)
 		go func() {
 			if err := s.Metrics.Start(ctx, addr); err != nil {
-				log.Error("error starting metrics server", "err", err)
+				log.Error("error starting metrics server",
+					"err", err)
 			}
 		}()
 	}
@@ -60,7 +64,8 @@ func (s *Service) Start(ctx context.Context) {
 	txpool := &provider.TransactionPool{}
 	for name, providers := range networks {
 		if len(providers) == 1 {
-			log.Warn("can't measure first seen for network, please another provider", "network", name)
+			log.Warn("can't measure first seen for network, please another provider",
+				"network", name)
 		}
 		(*txpool)[name] = &provider.NetworkTransactionPool{}
 		(*txpool)[name].Transactions = make(map[string]*provider.TransactionState)
@@ -76,7 +81,8 @@ func (s *Service) Start(ctx context.Context) {
 			s.Config.Wallets[providerConfig.Wallet],
 			(*txpool)[providerConfig.Network])
 		s.Providers[name].Start(ctx)
-		log.Info("provider started", "provider", name)
+		log.Info("provider started",
+			"provider", name)
 	}
 
 	log.Info("service started")
@@ -94,7 +100,8 @@ func (s *Service) Shutdown() {
 	}
 	for name, provider := range s.Providers {
 		provider.Shutdown()
-		log.Info("provider stopped", "provider", name)
+		log.Info("provider stopped",
+			"provider", name)
 	}
 	log.Info("service stopped")
 }

specs/challenger.md

@@ -73,7 +73,7 @@ to each of the different game types. For specification of dispute game types, se
   created by the `DisputeGameFactory` is equal to the output root of their `rollup-node` at the game's `l2BlockNumber`.
   - If it is, the Challenger should sign the [EIP-712 typeHash](./dispute-game.md) of the struct containing the
     `AttestationDisputeGame`'s `rootClaim` and `l2BlockNumber`. The Challenger should then submit the abi-encoded
-    signature to the `AttetationDisputeGame`'s `challenge` function.
+    signature to the `AttestationDisputeGame`'s `challenge` function.
   - If it is not, the Challenger should do nothing in support of this dispute game.
   ![Attestation `DisputeGameCreated` Diagram](./assets/challenger_attestation_dispute_game_created.png)