Merge branch 'develop' into indexer.healthz

op-chain-ops/safe/batch_helpers.go

@@ -141,9 +141,8 @@ func createContractInput(input abi.Argument, inputs []ContractInput) ([]Contract
 		return nil, err
 	}
 
-	// TODO: could probably do better than string comparison?
 	internalType := input.Type.String()
-	if inputType == "tuple" {
+	if input.Type.T == abi.TupleTy {
 		internalType = input.Type.TupleRawName
 	}
 

op-challenger/README.md

@@ -7,16 +7,167 @@ games, and validity games. To learn more about dispute games, visit the
 
 ## Quickstart
 
-First, clone this repo. Then, run `make`, which will build all required targets.
-Alternatively, run `make devnet` to bring up the [devnet](../ops-bedrock/devnet-up.sh)
-which deploys the [mock dispute game contracts](./contracts) as well as an
-`op-challenger` instance.
+First, clone this repo. Then run:
 
-Alternatively, you can build the `op-challenger` binary locally using the pre-configured
-[Makefile](./Makefile) target by running `make build`, and then running `./op-challenger --help`
+```shell
+cd op-challenger
+make alphabet
+```
+
+This creates a local devnet, starts a dispute game using the simple alphabet trace type and runs two op-challenger
+instances with differing views of the correct alphabet to play the game.
+
+Alternatively, you can build the `op-challenger` binary directly using the pre-configured
+[Makefile](./Makefile) target by running `make build`, and then running `./bin/op-challenger --help`
 to see a list of available options.
 
 ## Usage
 
 `op-challenger` is configurable via command line flags and environment variables. The help menu
 shows the available config options and can be accessed by running `./op-challenger --help`.
+
+### Running with Cannon on Local Devnet
+
+To run `op-challenger` against the local devnet, first ensure the required components are built and the devnet is running.
+From the top level of the repository run:
+
+```shell
+make devnet-clean
+make cannon-prestate op-challenger
+make devnet-up
+```
+
+Then start `op-challenger` with:
+```shell
+DISPUTE_GAME_FACTORY=$(jq -r .DisputeGameFactoryProxy .devnet/addresses.json)
+./op-challenger/bin/op-challenger \
+  --trace-type cannon \
+  --l1-eth-rpc http://localhost:8545 \
+  --game-factory-address $DISPUTE_GAME_FACTORY \
+  --agree-with-proposed-output=true \
+  --datadir temp/challenger-data \
+  --cannon-rollup-config .devnet/rollup.json  \
+  --cannon-l2-genesis .devnet/genesis-l2.json \
+  --cannon-bin ./cannon/bin/cannon \
+  --cannon-server ./op-program/bin/op-program \
+  --cannon-prestate ./op-program/bin/prestate.json \
+  --cannon-l2 http://localhost:9545 \
+  --mnemonic "test test test test test test test test test test test junk" \
+  --hd-path "m/44'/60'/0'/0/8" \
+  --num-confirmations 1
+```
+
+The mnemonic and hd-path above is a prefunded address on the devnet. The challenger respond to any created games by
+posting the correct trace as the counter-claim. The scripts below can then be used to create and interact with games.
+
+## Scripts
+
+The [scripts](scripts) directory contains a collection of scripts to assist with manually creating and playing games.
+This are not intended to be used in production, only to support manual testing and to aid with understanding how
+dispute games work. They also serve as examples of how to use `cast` to manually interact with the dispute game
+contracts.
+
+### Understanding Revert Reasons
+
+When actions performed by these scripts fails, they typically print a message that includes the
+abi encoded revert reason provided by the contract. e.g.
+
+```
+Error:
+(code: 3, message: execution reverted, data: Some(String("0x67fe1950")))
+```
+
+The `cast 4byte` command can be used to decode these revert reasons. e.g.
+
+```shell
+$ cast 4byte 0x67fe1950
+GameNotInProgress()
+```
+
+### Dependencies
+
+These scripts assume that the following tools are installed and available on the current `PATH`:
+
+* `cast` (https://book.getfoundry.sh/cast/)
+* `jq` (https://jqlang.github.io/jq/)
+* `bash`
+
+### [create_game.sh](scripts/create_game.sh)
+
+```shell
+./scripts/create_game.sh <RPC_URL> <GAME_FACTORY_ADDRESS> <ROOT_CLAIM> <SIGNER_ARGS>...
+```
+
+Starts a new fault dispute game that disputes the latest output proposal in the L2 output oracle.
+
+* `RPC_URL` - the RPC endpoint of the L1 endpoint to use (e.g. `http://localhost:8545`).
+* `GAME_FACTORY_ADDRESS` - the address of the dispute game factory contract on L1.
+* `ROOT_CLAIM` a hex encoded 32 byte hash to use as the root claim for the created game.
+* `SIGNER_ARGS` the remaining args are past as arguments to `cast` when sending transactions.
+  These arguments must specify a way for `cast` to sign the transactions.
+  See `cast send --help` for supported options.
+
+Creating a dispute game requires sending two transactions. The first transaction creates a
+checkpoint in the `BlockOracle` that records the L1 block that will be used as the L1 head
+when generating the cannon execution trace. The second transaction then creates the actual
+dispute game, specifying the disputed L2 block number and previously checkpointed L1 head block.
+
+### [move.sh](scripts/move.sh)
+
+```shell
+./scripts/move.sh <RPC_URL> <GAME_ADDRESS> (attack|defend) <PARENT_INDEX> <CLAIM> <SIGNER_ARGS>...
+```
+
+Performs a move to either attack or defend the latest claim in the specified game.
+
+* `RPC_URL` - the RPC endpoint of the L1 endpoint to use (e.g. `http://localhost:8545`).
+* `GAME_ADDRESS` - the address of the dispute game to perform the move in.
+* `(attack|defend)` - the type of move to make.
+  * `attack` indicates that the state hash in your local cannon trace differs to the state
+    hash included in the latest claim.
+  * `defend` indicates that the state hash in your local cannon trace matches the state hash
+    included in the latest claim.
+* `PARENT_INDEX` - the index of the parent claim that will be countered by this new claim.
+  The special value of `latest` will counter the latest claim added to the game.
+* `CLAIM` - the state hash to include in the counter-claim you are posting.
+* `SIGNER_ARGS` the remaining args are past as arguments to `cast` when sending transactions.
+  These arguments must specify a way for `cast` to sign the transactions.
+  See `cast send --help` for supported options.
+
+### [resolve.sh](scripts/resolve.sh)
+
+```shell
+./scripts/resolve.sh <RPC_URL> <GAME_ADDRESS> <SIGNER_ARGS>...
+```
+
+Resolves a dispute game. Note that this will fail if the dispute game has already been resolved
+or if the clocks have not yet expired and further moves are possible.
+If the game is resolved successfully, the result is printed.
+
+* `RPC_URL` - the RPC endpoint of the L1 endpoint to use (e.g. `http://localhost:8545`).
+* `GAME_ADDRESS` - the address of the dispute game to resolve.
+* `SIGNER_ARGS` the remaining args are past as arguments to `cast` when sending transactions.
+  These arguments must specify a way for `cast` to sign the transactions.
+  See `cast send --help` for supported options.
+
+### [list_games.sh](scripts/list_games.sh)
+
+```shell
+./scripts/list_games.sh <RPC> <GAME_FACTORY_ADDRESS>
+```
+
+Prints the games created by the game factory along with their current status.
+
+* `RPC_URL` - the RPC endpoint of the L1 endpoint to use (e.g. `http://localhost:8545`).
+* `GAME_FACTORY_ADDRESS` - the address of the dispute game factory contract on L1.
+
+### [list_claims.sh](scripts/list_claims.sh)
+
+```shell
+./scripts/list_claims.sh <RPC> <GAME_ADDR>
+```
+
+Prints the list of current claims in a dispute game.
+
+* `RPC_URL` - the RPC endpoint of the L1 endpoint to use (e.g. `http://localhost:8545`).
+* `GAME_ADDRESS` - the address of the dispute game to list the move in.

op-challenger/cmd/main_test.go

@@ -254,6 +254,28 @@ func TestCannonSnapshotFreq(t *testing.T) {
 		cfg := configForArgs(t, addRequiredArgs(config.TraceTypeCannon, "--cannon-snapshot-freq=1234"))
 		require.Equal(t, uint(1234), cfg.CannonSnapshotFreq)
 	})
+
+	t.Run("Invalid", func(t *testing.T) {
+		verifyArgsInvalid(t, "invalid value \"abc\" for flag -cannon-snapshot-freq",
+			addRequiredArgs(config.TraceTypeCannon, "--cannon-snapshot-freq=abc"))
+	})
+}
+
+func TestCannonInfoFreq(t *testing.T) {
+	t.Run("UsesDefault", func(t *testing.T) {
+		cfg := configForArgs(t, addRequiredArgs(config.TraceTypeCannon))
+		require.Equal(t, config.DefaultCannonInfoFreq, cfg.CannonInfoFreq)
+	})
+
+	t.Run("Valid", func(t *testing.T) {
+		cfg := configForArgs(t, addRequiredArgs(config.TraceTypeCannon, "--cannon-info-freq=1234"))
+		require.Equal(t, uint(1234), cfg.CannonInfoFreq)
+	})
+
+	t.Run("Invalid", func(t *testing.T) {
+		verifyArgsInvalid(t, "invalid value \"abc\" for flag -cannon-info-freq",
+			addRequiredArgs(config.TraceTypeCannon, "--cannon-info-freq=abc"))
+	})
 }
 
 func TestGameWindow(t *testing.T) {

op-challenger/config/config.go

@@ -26,6 +26,7 @@ var (
 	ErrMissingL1EthRPC               = errors.New("missing l1 eth rpc url")
 	ErrMissingGameFactoryAddress     = errors.New("missing game factory address")
 	ErrMissingCannonSnapshotFreq     = errors.New("missing cannon snapshot freq")
+	ErrMissingCannonInfoFreq         = errors.New("missing cannon info freq")
 	ErrMissingCannonRollupConfig     = errors.New("missing cannon network or rollup config path")
 	ErrMissingCannonL2Genesis        = errors.New("missing cannon network or l2 genesis path")
 	ErrCannonNetworkAndRollupConfig  = errors.New("only specify one of network or rollup config path")
@@ -78,6 +79,7 @@ func ValidTraceType(value TraceType) bool {
 
 const (
 	DefaultCannonSnapshotFreq = uint(1_000_000_000)
+	DefaultCannonInfoFreq     = uint(10_000_000)
 	// DefaultGameWindow is the default maximum time duration in the past
 	// that the challenger will look for games to progress.
 	// The default value is 11 days, which is a 4 day resolution buffer
@@ -111,6 +113,7 @@ type Config struct {
 	CannonL2GenesisPath    string
 	CannonL2               string // L2 RPC Url
 	CannonSnapshotFreq     uint   // Frequency of snapshots to create when executing cannon (in VM instructions)
+	CannonInfoFreq         uint   // Frequency of cannon progress log messages (in VM instructions)
 
 	TxMgrConfig   txmgr.CLIConfig
 	MetricsConfig opmetrics.CLIConfig
@@ -140,6 +143,7 @@ func NewConfig(
 		Datadir: datadir,
 
 		CannonSnapshotFreq: DefaultCannonSnapshotFreq,
+		CannonInfoFreq:     DefaultCannonInfoFreq,
 		GameWindow:         DefaultGameWindow,
 	}
 }
@@ -194,6 +198,9 @@ func (c Config) Check() error {
 		if c.CannonSnapshotFreq == 0 {
 			return ErrMissingCannonSnapshotFreq
 		}
+		if c.CannonInfoFreq == 0 {
+			return ErrMissingCannonInfoFreq
+		}
 	}
 	if c.TraceType == TraceTypeAlphabet && c.AlphabetTrace == "" {
 		return ErrMissingAlphabetTrace

op-challenger/config/config_test.go

@@ -132,6 +132,14 @@ func TestCannonSnapshotFreq(t *testing.T) {
 	})
 }
 
+func TestCannonInfoFreq(t *testing.T) {
+	t.Run("MustNotBeZero", func(t *testing.T) {
+		cfg := validConfig(TraceTypeCannon)
+		cfg.CannonInfoFreq = 0
+		require.ErrorIs(t, cfg.Check(), ErrMissingCannonInfoFreq)
+	})
+}
+
 func TestCannonNetworkOrRollupConfigRequired(t *testing.T) {
 	cfg := validConfig(TraceTypeCannon)
 	cfg.CannonNetwork = ""

op-challenger/flags/flags.go

@@ -116,6 +116,12 @@ var (
 		EnvVars: prefixEnvVars("CANNON_SNAPSHOT_FREQ"),
 		Value:   config.DefaultCannonSnapshotFreq,
 	}
+	CannonInfoFreqFlag = &cli.UintFlag{
+		Name:    "cannon-info-freq",
+		Usage:   "Frequency of cannon info log messages to generate in VM steps (cannon trace type only)",
+		EnvVars: prefixEnvVars("CANNON_INFO_FREQ"),
+		Value:   config.DefaultCannonInfoFreq,
+	}
 	GameWindowFlag = &cli.DurationFlag{
 		Name:    "game-window",
 		Usage:   "The time window which the challenger will look for games to progress.",
@@ -146,6 +152,7 @@ var optionalFlags = []cli.Flag{
 	CannonPreStateFlag,
 	CannonL2Flag,
 	CannonSnapshotFreqFlag,
+	CannonInfoFreqFlag,
 	GameWindowFlag,
 }
 
@@ -250,6 +257,7 @@ func NewConfigFromCLI(ctx *cli.Context) (*config.Config, error) {
 		Datadir:                 ctx.String(DatadirFlag.Name),
 		CannonL2:                ctx.String(CannonL2Flag.Name),
 		CannonSnapshotFreq:      ctx.Uint(CannonSnapshotFreqFlag.Name),
+		CannonInfoFreq:          ctx.Uint(CannonInfoFreqFlag.Name),
 		AgreeWithProposedOutput: ctx.Bool(AgreeWithProposedOutputFlag.Name),
 		TxMgrConfig:             txMgrConfig,
 		MetricsConfig:           metricsConfig,

op-challenger/game/fault/agent.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/solver"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum/go-ethereum/log"
 )
 
@@ -24,6 +25,7 @@ type ClaimLoader interface {
 }
 
 type Agent struct {
+	metrics                 metrics.Metricer
 	solver                  *solver.Solver
 	loader                  ClaimLoader
 	responder               Responder
@@ -33,8 +35,9 @@ type Agent struct {
 	log                     log.Logger
 }
 
-func NewAgent(loader ClaimLoader, maxDepth int, trace types.TraceProvider, responder Responder, updater types.OracleUpdater, agreeWithProposedOutput bool, log log.Logger) *Agent {
+func NewAgent(m metrics.Metricer, loader ClaimLoader, maxDepth int, trace types.TraceProvider, responder Responder, updater types.OracleUpdater, agreeWithProposedOutput bool, log log.Logger) *Agent {
 	return &Agent{
+		metrics:                 m,
 		solver:                  solver.NewSolver(maxDepth, trace),
 		loader:                  loader,
 		responder:               responder,
@@ -71,7 +74,7 @@ func (a *Agent) Act(ctx context.Context) error {
 
 // shouldResolve returns true if the agent should resolve the game.
 // This method will return false if the game is still in progress.
-func (a *Agent) shouldResolve(ctx context.Context, status types.GameStatus) bool {
+func (a *Agent) shouldResolve(status types.GameStatus) bool {
 	expected := types.GameStatusDefenderWon
 	if a.agreeWithProposedOutput {
 		expected = types.GameStatusChallengerWon
@@ -82,20 +85,19 @@ func (a *Agent) shouldResolve(ctx context.Context, status types.GameStatus) bool
 	return expected == status
 }
 
-// tryResolve resolves the game if it is in a terminal state
-// and returns true if the game resolves successfully.
+// tryResolve resolves the game if it is in a winning state
+// Returns true if the game is resolvable (regardless of whether it was actually resolved)
 func (a *Agent) tryResolve(ctx context.Context) bool {
 	status, err := a.responder.CallResolve(ctx)
-	if err != nil {
+	if err != nil || status == types.GameStatusInProgress {
 		return false
 	}
-	if !a.shouldResolve(ctx, status) {
-		return false
+	if !a.shouldResolve(status) {
+		return true
 	}
 	a.log.Info("Resolving game")
 	if err := a.responder.Resolve(ctx); err != nil {
 		a.log.Error("Failed to resolve the game", "err", err)
-		return false
 	}
 	return true
 }
@@ -134,6 +136,7 @@ func (a *Agent) move(ctx context.Context, claim types.Claim, game types.Game) er
 		log.Debug("Skipping duplicate move")
 		return nil
 	}
+	a.metrics.RecordGameMove()
 	log.Info("Performing move")
 	return a.responder.Respond(ctx, move)
 }
@@ -170,6 +173,7 @@ func (a *Agent) step(ctx context.Context, claim types.Claim, game types.Game) er
 
 	a.log.Info("Performing step", "is_attack", step.IsAttack,
 		"depth", step.LeafClaim.Depth(), "index_at_depth", step.LeafClaim.IndexAtDepth(), "value", step.LeafClaim.Value)
+	a.metrics.RecordGameStep()
 	callData := types.StepCallData{
 		ClaimIndex: uint64(step.LeafClaim.ContractIndex),
 		IsAttack:   step.IsAttack,

op-challenger/game/fault/agent_test.go

@@ -2,9 +2,13 @@ package fault
 
 import (
 	"context"
+	"errors"
 	"testing"
 
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/test"
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/alphabet"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/stretchr/testify/require"
 
@@ -13,19 +17,143 @@ import (
 
 // TestShouldResolve tests the resolution logic.
 func TestShouldResolve(t *testing.T) {
-	log := testlog.Logger(t, log.LvlCrit)
-
 	t.Run("AgreeWithProposedOutput", func(t *testing.T) {
-		agent := NewAgent(nil, 0, nil, nil, nil, true, log)
-		require.False(t, agent.shouldResolve(context.Background(), types.GameStatusDefenderWon))
-		require.True(t, agent.shouldResolve(context.Background(), types.GameStatusChallengerWon))
-		require.False(t, agent.shouldResolve(context.Background(), types.GameStatusInProgress))
+		agent, _, _ := setupTestAgent(t, true)
+		require.False(t, agent.shouldResolve(types.GameStatusDefenderWon))
+		require.True(t, agent.shouldResolve(types.GameStatusChallengerWon))
+		require.False(t, agent.shouldResolve(types.GameStatusInProgress))
 	})
 
 	t.Run("DisagreeWithProposedOutput", func(t *testing.T) {
-		agent := NewAgent(nil, 0, nil, nil, nil, false, log)
-		require.True(t, agent.shouldResolve(context.Background(), types.GameStatusDefenderWon))
-		require.False(t, agent.shouldResolve(context.Background(), types.GameStatusChallengerWon))
-		require.False(t, agent.shouldResolve(context.Background(), types.GameStatusInProgress))
+		agent, _, _ := setupTestAgent(t, false)
+		require.True(t, agent.shouldResolve(types.GameStatusDefenderWon))
+		require.False(t, agent.shouldResolve(types.GameStatusChallengerWon))
+		require.False(t, agent.shouldResolve(types.GameStatusInProgress))
 	})
 }
+
+func TestDoNotMakeMovesWhenGameIsResolvable(t *testing.T) {
+	ctx := context.Background()
+
+	tests := []struct {
+		name                    string
+		agreeWithProposedOutput bool
+		callResolveStatus       types.GameStatus
+		shouldResolve           bool
+	}{
+		{
+			name:                    "Agree_Losing",
+			agreeWithProposedOutput: true,
+			callResolveStatus:       types.GameStatusDefenderWon,
+			shouldResolve:           false,
+		},
+		{
+			name:                    "Agree_Winning",
+			agreeWithProposedOutput: true,
+			callResolveStatus:       types.GameStatusChallengerWon,
+			shouldResolve:           true,
+		},
+		{
+			name:                    "Disagree_Losing",
+			agreeWithProposedOutput: false,
+			callResolveStatus:       types.GameStatusChallengerWon,
+			shouldResolve:           false,
+		},
+		{
+			name:                    "Disagree_Winning",
+			agreeWithProposedOutput: false,
+			callResolveStatus:       types.GameStatusDefenderWon,
+			shouldResolve:           true,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			agent, claimLoader, responder := setupTestAgent(t, test.agreeWithProposedOutput)
+			responder.callResolveStatus = test.callResolveStatus
+
+			require.NoError(t, agent.Act(ctx))
+
+			require.Equal(t, 1, responder.callResolveCount, "should check if game is resolvable")
+			require.Zero(t, claimLoader.callCount, "should not fetch claims for resolvable game")
+
+			if test.shouldResolve {
+				require.EqualValues(t, 1, responder.resolveCount, "should resolve winning game")
+			} else {
+				require.Zero(t, responder.resolveCount, "should not resolve losing game")
+			}
+		})
+	}
+}
+
+func TestLoadClaimsWhenGameNotResolvable(t *testing.T) {
+	// Checks that if the game isn't resolvable, that the agent continues on to start checking claims
+	agent, claimLoader, responder := setupTestAgent(t, false)
+	responder.callResolveErr = errors.New("game is not resolvable")
+	depth := 4
+	claimBuilder := test.NewClaimBuilder(t, depth, alphabet.NewTraceProvider("abcdefg", uint64(depth)))
+
+	claimLoader.claims = []types.Claim{
+		claimBuilder.CreateRootClaim(true),
+	}
+
+	require.NoError(t, agent.Act(context.Background()))
+
+	require.EqualValues(t, 1, claimLoader.callCount, "should load claims for unresolvable game")
+}
+
+func setupTestAgent(t *testing.T, agreeWithProposedOutput bool) (*Agent, *stubClaimLoader, *stubResponder) {
+	logger := testlog.Logger(t, log.LvlInfo)
+	claimLoader := &stubClaimLoader{}
+	depth := 4
+	trace := alphabet.NewTraceProvider("abcd", uint64(depth))
+	responder := &stubResponder{}
+	updater := &stubUpdater{}
+	agent := NewAgent(metrics.NoopMetrics, claimLoader, depth, trace, responder, updater, agreeWithProposedOutput, logger)
+	return agent, claimLoader, responder
+}
+
+type stubClaimLoader struct {
+	callCount int
+	claims    []types.Claim
+}
+
+func (s *stubClaimLoader) FetchClaims(ctx context.Context) ([]types.Claim, error) {
+	s.callCount++
+	return s.claims, nil
+}
+
+type stubResponder struct {
+	callResolveCount  int
+	callResolveStatus types.GameStatus
+	callResolveErr    error
+
+	resolveCount int
+	resolveErr   error
+}
+
+func (s *stubResponder) CallResolve(ctx context.Context) (types.GameStatus, error) {
+	s.callResolveCount++
+	return s.callResolveStatus, s.callResolveErr
+}
+
+func (s *stubResponder) Resolve(ctx context.Context) error {
+	s.resolveCount++
+	return s.resolveErr
+}
+
+func (s *stubResponder) Respond(ctx context.Context, response types.Claim) error {
+	panic("Not implemented")
+}
+
+func (s *stubResponder) Step(ctx context.Context, stepData types.StepCallData) error {
+	panic("Not implemented")
+}
+
+type stubUpdater struct {
+}
+
+func (s *stubUpdater) UpdateOracle(ctx context.Context, data *types.PreimageOracleData) error {
+	panic("Not implemented")
+}

op-challenger/game/fault/player.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/alphabet"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/cannon"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum-optimism/optimism/op-service/txmgr"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
@@ -37,6 +38,7 @@ type GamePlayer struct {
 func NewGamePlayer(
 	ctx context.Context,
 	logger log.Logger,
+	m metrics.Metricer,
 	cfg *config.Config,
 	dir string,
 	addr common.Address,
@@ -79,7 +81,7 @@ func NewGamePlayer(
 	var updater types.OracleUpdater
 	switch cfg.TraceType {
 	case config.TraceTypeCannon:
-		cannonProvider, err := cannon.NewTraceProvider(ctx, logger, cfg, client, dir, addr)
+		cannonProvider, err := cannon.NewTraceProvider(ctx, logger, m, cfg, client, dir, addr)
 		if err != nil {
 			return nil, fmt.Errorf("create cannon trace provider: %w", err)
 		}
@@ -105,7 +107,7 @@ func NewGamePlayer(
 	}
 
 	return &GamePlayer{
-		act:                     NewAgent(loader, int(gameDepth), provider, responder, updater, cfg.AgreeWithProposedOutput, logger).Act,
+		act:                     NewAgent(m, loader, int(gameDepth), provider, responder, updater, cfg.AgreeWithProposedOutput, logger).Act,
 		agreeWithProposedOutput: cfg.AgreeWithProposedOutput,
 		loader:                  loader,
 		logger:                  logger,

op-challenger/game/fault/trace/cannon/executor.go

@@ -11,6 +11,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/ethereum-optimism/optimism/op-challenger/config"
 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
@@ -30,6 +31,7 @@ type cmdExecutor func(ctx context.Context, l log.Logger, binary string, args ...
 
 type Executor struct {
 	logger           log.Logger
+	metrics          CannonMetricer
 	l1               string
 	l2               string
 	inputs           LocalGameInputs
@@ -40,13 +42,15 @@ type Executor struct {
 	l2Genesis        string
 	absolutePreState string
 	snapshotFreq     uint
+	infoFreq         uint
 	selectSnapshot   snapshotSelect
 	cmdExecutor      cmdExecutor
 }
 
-func NewExecutor(logger log.Logger, cfg *config.Config, inputs LocalGameInputs) *Executor {
+func NewExecutor(logger log.Logger, m CannonMetricer, cfg *config.Config, inputs LocalGameInputs) *Executor {
 	return &Executor{
 		logger:           logger,
+		metrics:          m,
 		l1:               cfg.L1EthRpc,
 		l2:               cfg.CannonL2,
 		inputs:           inputs,
@@ -57,6 +61,7 @@ func NewExecutor(logger log.Logger, cfg *config.Config, inputs LocalGameInputs)
 		l2Genesis:        cfg.CannonL2GenesisPath,
 		absolutePreState: cfg.CannonAbsolutePreState,
 		snapshotFreq:     cfg.CannonSnapshotFreq,
+		infoFreq:         cfg.CannonInfoFreq,
 		selectSnapshot:   findStartingSnapshot,
 		cmdExecutor:      runCmd,
 	}
@@ -76,6 +81,7 @@ func (e *Executor) GenerateProof(ctx context.Context, dir string, i uint64) erro
 		"--input", start,
 		"--output", lastGeneratedState,
 		"--meta", "",
+		"--info-at", "%" + strconv.FormatUint(uint64(e.infoFreq), 10),
 		"--proof-at", "=" + strconv.FormatUint(i, 10),
 		"--proof-fmt", filepath.Join(proofDir, "%d.json.gz"),
 		"--snapshot-at", "%" + strconv.FormatUint(uint64(e.snapshotFreq), 10),
@@ -116,7 +122,13 @@ func (e *Executor) GenerateProof(ctx context.Context, dir string, i uint64) erro
 		return fmt.Errorf("could not create proofs directory %v: %w", proofDir, err)
 	}
 	e.logger.Info("Generating trace", "proof", i, "cmd", e.cannon, "args", strings.Join(args, ", "))
-	return e.cmdExecutor(ctx, e.logger.New("proof", i), e.cannon, args...)
+	execStart := time.Now()
+	err = e.cmdExecutor(ctx, e.logger.New("proof", i), e.cannon, args...)
+	if err != nil {
+		execDuration := time.Since(execStart).Seconds()
+		e.metrics.RecordCannonExecutionTime(execDuration)
+	}
+	return err
 }
 
 func runCmd(ctx context.Context, l log.Logger, binary string, args ...string) error {

op-challenger/game/fault/trace/cannon/executor_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/ethereum-optimism/optimism/op-challenger/config"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum-optimism/optimism/op-node/testlog"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
@@ -29,6 +30,7 @@ func TestGenerateProof(t *testing.T) {
 	cfg.CannonServer = "./bin/op-program"
 	cfg.CannonL2 = "http://localhost:9999"
 	cfg.CannonSnapshotFreq = 500
+	cfg.CannonInfoFreq = 900
 
 	inputs := LocalGameInputs{
 		L1Head:        common.Hash{0x11},
@@ -38,7 +40,7 @@ func TestGenerateProof(t *testing.T) {
 		L2BlockNumber: big.NewInt(3333),
 	}
 	captureExec := func(t *testing.T, cfg config.Config, proofAt uint64) (string, string, map[string]string) {
-		executor := NewExecutor(testlog.Logger(t, log.LvlInfo), &cfg, inputs)
+		executor := NewExecutor(testlog.Logger(t, log.LvlInfo), metrics.NoopMetrics, &cfg, inputs)
 		executor.selectSnapshot = func(logger log.Logger, dir string, absolutePreState string, i uint64) (string, error) {
 			return input, nil
 		}
@@ -81,6 +83,7 @@ func TestGenerateProof(t *testing.T) {
 		require.Equal(t, "=150000000", args["--proof-at"])
 		require.Equal(t, "=150000001", args["--stop-at"])
 		require.Equal(t, "%500", args["--snapshot-at"])
+		require.Equal(t, "%900", args["--info-at"])
 		// Slight quirk of how we pair off args
 		// The server binary winds up as the key and the first arg --server as the value which has no value
 		// Then everything else pairs off correctly again

op-challenger/game/fault/trace/cannon/provider.go

@@ -33,6 +33,10 @@ type proofData struct {
 	OracleOffset uint32        `json:"oracle-offset,omitempty"`
 }
 
+type CannonMetricer interface {
+	RecordCannonExecutionTime(t float64)
+}
+
 type ProofGenerator interface {
 	// GenerateProof executes cannon to generate a proof at the specified trace index in dataDir.
 	GenerateProof(ctx context.Context, dataDir string, proofAt uint64) error
@@ -51,7 +55,7 @@ type CannonTraceProvider struct {
 	lastProof *proofData
 }
 
-func NewTraceProvider(ctx context.Context, logger log.Logger, cfg *config.Config, l1Client bind.ContractCaller, dir string, gameAddr common.Address) (*CannonTraceProvider, error) {
+func NewTraceProvider(ctx context.Context, logger log.Logger, m CannonMetricer, cfg *config.Config, l1Client bind.ContractCaller, dir string, gameAddr common.Address) (*CannonTraceProvider, error) {
 	l2Client, err := ethclient.DialContext(ctx, cfg.CannonL2)
 	if err != nil {
 		return nil, fmt.Errorf("dial l2 client %v: %w", cfg.CannonL2, err)
@@ -65,15 +69,15 @@ func NewTraceProvider(ctx context.Context, logger log.Logger, cfg *config.Config
 	if err != nil {
 		return nil, fmt.Errorf("fetch local game inputs: %w", err)
 	}
-	return NewTraceProviderFromInputs(logger, cfg, localInputs, dir), nil
+	return NewTraceProviderFromInputs(logger, m, cfg, localInputs, dir), nil
 }
 
-func NewTraceProviderFromInputs(logger log.Logger, cfg *config.Config, localInputs LocalGameInputs, dir string) *CannonTraceProvider {
+func NewTraceProviderFromInputs(logger log.Logger, m CannonMetricer, cfg *config.Config, localInputs LocalGameInputs, dir string) *CannonTraceProvider {
 	return &CannonTraceProvider{
 		logger:    logger,
 		dir:       dir,
 		prestate:  cfg.CannonAbsolutePreState,
-		generator: NewExecutor(logger, cfg, localInputs),
+		generator: NewExecutor(logger, m, cfg, localInputs),
 	}
 }
 

op-challenger/game/monitor.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/ethereum-optimism/optimism/op-challenger/game/scheduler"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum-optimism/optimism/op-service/clock"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
@@ -26,6 +27,7 @@ type gameScheduler interface {
 
 type gameMonitor struct {
 	logger           log.Logger
+	metrics          metrics.Metricer
 	clock            clock.Clock
 	source           gameSource
 	scheduler        gameScheduler
@@ -36,6 +38,7 @@ type gameMonitor struct {
 
 func newGameMonitor(
 	logger log.Logger,
+	m metrics.Metricer,
 	cl clock.Clock,
 	source gameSource,
 	scheduler gameScheduler,
@@ -45,6 +48,7 @@ func newGameMonitor(
 ) *gameMonitor {
 	return &gameMonitor{
 		logger:           logger,
+		metrics:          m,
 		clock:            cl,
 		scheduler:        scheduler,
 		source:           source,

op-challenger/game/monitor_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum-optimism/optimism/op-node/testlog"
 	"github.com/ethereum-optimism/optimism/op-service/clock"
 	"github.com/ethereum/go-ethereum/common"
@@ -100,7 +101,7 @@ func setupMonitorTest(t *testing.T, allowedGames []common.Address) (*gameMonitor
 		return i, nil
 	}
 	sched := &stubScheduler{}
-	monitor := newGameMonitor(logger, clock.SystemClock, source, sched, time.Duration(0), fetchBlockNum, allowedGames)
+	monitor := newGameMonitor(logger, metrics.NoopMetrics, clock.SystemClock, source, sched, time.Duration(0), fetchBlockNum, allowedGames)
 	return monitor, source, sched
 }
 

op-challenger/game/service.go

@@ -72,10 +72,10 @@ func NewService(ctx context.Context, logger log.Logger, cfg *config.Config) (*Se
 		disk,
 		cfg.MaxConcurrency,
 		func(addr common.Address, dir string) (scheduler.GamePlayer, error) {
-			return fault.NewGamePlayer(ctx, logger, cfg, dir, addr, txMgr, client)
+			return fault.NewGamePlayer(ctx, logger, m, cfg, dir, addr, txMgr, client)
 		})
 
-	monitor := newGameMonitor(logger, cl, loader, sched, cfg.GameWindow, client.BlockNumber, cfg.GameAllowlist)
+	monitor := newGameMonitor(logger, m, cl, loader, sched, cfg.GameWindow, client.BlockNumber, cfg.GameAllowlist)
 
 	m.RecordInfo(version.SimpleWithMeta)
 	m.RecordUp()

op-challenger/metrics/metrics.go

@@ -20,6 +20,10 @@ type Metricer interface {
 
 	// Record Tx metrics
 	txmetrics.TxMetricer
+
+	RecordGameStep()
+	RecordGameMove()
+	RecordCannonExecutionTime(t float64)
 }
 
 type Metrics struct {
@@ -31,6 +35,10 @@ type Metrics struct {
 
 	info prometheus.GaugeVec
 	up   prometheus.Gauge
+
+	moves               prometheus.Counter
+	steps               prometheus.Counter
+	cannonExecutionTime prometheus.Histogram
 }
 
 var _ Metricer = (*Metrics)(nil)
@@ -58,6 +66,22 @@ func NewMetrics() *Metrics {
 			Name:      "up",
 			Help:      "1 if the op-challenger has finished starting up",
 		}),
+		moves: factory.NewCounter(prometheus.CounterOpts{
+			Namespace: Namespace,
+			Name:      "moves",
+			Help:      "Number of game moves made by the challenge agent",
+		}),
+		steps: factory.NewCounter(prometheus.CounterOpts{
+			Namespace: Namespace,
+			Name:      "steps",
+			Help:      "Number of game steps made by the challenge agent",
+		}),
+		cannonExecutionTime: factory.NewHistogram(prometheus.HistogramOpts{
+			Namespace: Namespace,
+			Name:      "cannon_execution_time",
+			Help:      "Time (in seconds) to execute cannon",
+			Buckets:   append([]float64{1.0, 10.0}, prometheus.ExponentialBuckets(30.0, 2.0, 14)...),
+		}),
 	}
 }
 
@@ -84,3 +108,15 @@ func (m *Metrics) RecordUp() {
 func (m *Metrics) Document() []opmetrics.DocumentedMetric {
 	return m.factory.Document()
 }
+
+func (m *Metrics) RecordGameMove() {
+	m.moves.Add(1)
+}
+
+func (m *Metrics) RecordGameStep() {
+	m.steps.Add(1)
+}
+
+func (m *Metrics) RecordCannonExecutionTime(t float64) {
+	m.cannonExecutionTime.Observe(t)
+}

op-challenger/metrics/noop.go

@@ -10,5 +10,8 @@ type noopMetrics struct {
 
 var NoopMetrics Metricer = new(noopMetrics)
 
-func (*noopMetrics) RecordInfo(version string) {}
-func (*noopMetrics) RecordUp()                 {}
+func (*noopMetrics) RecordInfo(version string)           {}
+func (*noopMetrics) RecordUp()                           {}
+func (*noopMetrics) RecordGameMove()                     {}
+func (*noopMetrics) RecordGameStep()                     {}
+func (*noopMetrics) RecordCannonExecutionTime(t float64) {}

op-challenger/scripts/alphabet/init_game.sh

@@ -15,7 +15,7 @@ make cannon-prestate
 make devnet-up
 
 DEVNET_SPONSOR="ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
-DISPUTE_GAME_PROXY=$(jq -r .DisputeGameFactoryProxy $MONOREPO_DIR/.devnet/addresses.json)
+DISPUTE_GAME_FACTORY=$(jq -r .DisputeGameFactoryProxy $MONOREPO_DIR/.devnet/addresses.json)
 
 echo "----------------------------------------------------------------"
 echo " Dispute Game Factory at $DISPUTE_GAME_PROXY"
@@ -59,43 +59,8 @@ do
   sleep 2
 done
 
-# Fetch the latest block number
-L2_BLOCK_NUMBER=$(cast call $L2_OUTPUT_ORACLE_PROXY "latestBlockNumber()")
-echo "Using the latest L2OO block number: $L2_BLOCK_NUMBER"
-
-# We will use the l2 block number of 1 for the dispute game.
-# We need to check that the block oracle contains the corresponding l1 block number.
-echo "Checkpointing the block oracle..."
-L1_CHECKPOINT=$(cast send --private-key $DEVNET_SPONSOR $BLOCK_ORACLE_PROXY "checkpoint()" --json | jq -r .blockNumber | cast to-dec)
-((L1_CHECKPOINT=L1_CHECKPOINT-1))
-echo "L1 Checkpoint: $L1_CHECKPOINT"
-
-INDEX=$(cast call $L2_OUTPUT_ORACLE_PROXY "getL2OutputIndexAfter(uint256)" $L2_BLOCK_NUMBER | cast to-dec)
-((PRIOR_INDEX=INDEX-1))
-echo "Getting the l2 output at index $PRIOR_INDEX"
-cast call $L2_OUTPUT_ORACLE_PROXY "getL2Output(uint256)" $PRIOR_INDEX
-
-echo "Getting the l2 output at index $INDEX"
-cast call $L2_OUTPUT_ORACLE_PROXY "getL2Output(uint256)" $INDEX
-
-# (Alphabet) Fault game type = 255
-GAME_TYPE=255
-
 # Root claim commits to the entire trace.
 # Alphabet game claim construction: keccak256(abi.encode(trace_index, trace[trace_index]))
 ROOT_CLAIM=$(cast keccak $(cast abi-encode "f(uint256,uint256)" 15 122))
 
-# Fault dispute game extra data is calculated as follows.
-# abi.encode(uint256(l2_block_number), uint256(l1 checkpoint))
-EXTRA_DATA=$(cast abi-encode "f(uint256,uint256)" $L2_BLOCK_NUMBER $L1_CHECKPOINT)
-
-echo "Initializing the game"
-FAULT_GAME_ADDRESS=$(cast call --private-key $MALLORY_KEY $DISPUTE_GAME_PROXY "create(uint8,bytes32,bytes)" $GAME_TYPE $ROOT_CLAIM $EXTRA_DATA)
-
-echo "Creating game at address $FAULT_GAME_ADDRESS"
-cast send --private-key $MALLORY_KEY $DISPUTE_GAME_PROXY "create(uint8,bytes32,bytes)" $GAME_TYPE $ROOT_CLAIM $EXTRA_DATA
-
-FORMATTED_ADDRESS=$(cast parse-bytes32-address $FAULT_GAME_ADDRESS)
-echo "Formatted Address: $FORMATTED_ADDRESS"
-
-echo $FORMATTED_ADDRESS > $CHALLENGER_DIR/.fault-game-address
+GAME_TYPE=255 ${SOURCE_DIR}/../create_game.sh http://localhost:8545 "${DISPUTE_GAME_FACTORY}" "${ROOT_CLAIM}" --private-key "${DEVNET_SPONSOR}"

op-challenger/scripts/create_game.sh

+#!/usr/bin/env bash
+set -euo pipefail
+
+SOURCE_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
+CHALLENGER_DIR=$(echo ${SOURCE_DIR%/*})
+MONOREPO_DIR=$(echo ${CHALLENGER_DIR%/*})
+
+# ./create_game.sh <rpc-addr> <dispute-game-factory-addr> <cast signing args>
+RPC=${1:?Must specify RPC address}
+FACTORY_ADDR=${2:?Must specify factory address}
+ROOT_CLAIM=${3:?Must specify root claim}
+SIGNER_ARGS="${@:4}"
+
+# Default to Cannon Fault game type
+GAME_TYPE=${GAME_TYPE:-0}
+
+# Get the fault dispute game implementation addr
+GAME_IMPL_ADDR=$(cast call --rpc-url "${RPC}" "${FACTORY_ADDR}" 'gameImpls(uint8) returns(address)' "${GAME_TYPE}")
+echo "Fault dispute game impl: ${GAME_IMPL_ADDR}"
+
+# Get the L2 output oracle address
+L2OO_ADDR=$(cast call --rpc-url "${RPC}" "${GAME_IMPL_ADDR}" 'L2_OUTPUT_ORACLE() returns(address)')
+echo "L2OO: ${L2OO_ADDR}"
+
+# Get the block oracle address
+BLOCK_ORACLE_ADDR=$(cast call --rpc-url "${RPC}" "${GAME_IMPL_ADDR}" 'BLOCK_ORACLE() returns(address)')
+echo "Block Oracle: ${BLOCK_ORACLE_ADDR}"
+
+# Get the L2 block number of the latest output proposal. This is the proposal that will be disputed by the created game.
+L2_BLOCK_NUM=$(cast call --rpc-url "${RPC}" "${L2OO_ADDR}" 'latestBlockNumber() public view returns (uint256)')
+echo "L2 Block Number: ${L2_BLOCK_NUM}"
+
+# Create a checkpoint in the block oracle to commit to the current L1 head.
+# This defines the L1 head that will be used in the dispute game.
+echo "Checkpointing the block oracle..."
+L1_CHECKPOINT=$(cast send --rpc-url "${RPC}" ${SIGNER_ARGS} "${BLOCK_ORACLE_ADDR}" "checkpoint()" --json | jq -r '.logs[0].topics[1]' | cast to-dec)
+echo "L1 Checkpoint: $L1_CHECKPOINT"
+
+# Fault dispute game extra data is calculated as follows.
+# abi.encode(uint256(l2_block_number), uint256(l1 checkpoint))
+EXTRA_DATA=$(cast abi-encode "f(uint256,uint256)" "${L2_BLOCK_NUM}" "${L1_CHECKPOINT}")
+
+echo "Initializing the game"
+FAULT_GAME_DATA=$(cast send --rpc-url "${RPC}" ${SIGNER_ARGS} "${FACTORY_ADDR}" "create(uint8,bytes32,bytes) returns(address)" "${GAME_TYPE}" "${ROOT_CLAIM}" "${EXTRA_DATA}" --json)
+
+# Extract the address of the newly created game from the receipt logs.
+FAULT_GAME_ADDRESS=$(echo "${FAULT_GAME_DATA}" | jq -r '.logs[0].topics[1]' | cast parse-bytes32-address)
+echo "Fault game address: ${FAULT_GAME_ADDRESS}"
+echo "${FAULT_GAME_ADDRESS}" > $CHALLENGER_DIR/.fault-game-address

op-challenger/scripts/list_claims.sh

+#!/usr/bin/env bash
+
+set -euo pipefail
+
+RPC=${1:?Must specify RPC address}
+GAME_ADDR=${2:?Must specify fault dispute game address}
+
+COUNT=$(cast call --rpc-url "${RPC}" "${GAME_ADDR}" 'claimDataLen() returns(uint256)')
+echo "Claim count: ${COUNT}"
+((COUNT=COUNT-1))
+for i in $(seq 0 "${COUNT}")
+do
+  CLAIM=$(cast call --rpc-url "${RPC}" "${GAME_ADDR}" 'claimData(uint256) returns(uint32 parentIndex, bool countered, bytes32 claim, uint128 position, uint128 clock)' "${i}")
+  SAVEIFS=$IFS   # Save current IFS (Internal Field Separator)
+  IFS=$'\n'      # Change IFS to newline char
+  CLAIM=($CLAIM) # split the string into an array by the same name
+  IFS=$SAVEIFS   # Restore original IFS
+
+  echo "${i}  Parent: ${CLAIM[0]} Countered: ${CLAIM[1]} Claim: ${CLAIM[2]} Position: ${CLAIM[3]} Clock ${CLAIM[4]}"
+done

op-challenger/scripts/list_games.sh

+#!/usr/bin/env bash
+
+set -euo pipefail
+
+RPC=${1:?Must specify RPC address}
+FACTORY_ADDR=${2:?Must specify dispute game factory address}
+
+COUNT=$(cast call --rpc-url "${RPC}" "${FACTORY_ADDR}" 'gameCount() returns(uint256)')
+echo "Game count: ${COUNT}"
+if [[ "${COUNT}" == "0" ]]
+then
+  exit
+fi
+((COUNT=COUNT-1))
+for i in $(seq 0 "${COUNT}")
+do
+  GAME=$(cast call --rpc-url "${RPC}" "${FACTORY_ADDR}" 'gameAtIndex(uint256) returns(uint8, uint64, address)' "${i}")
+
+  SAVEIFS=$IFS   # Save current IFS (Internal Field Separator)
+  IFS=$'\n'      # Change IFS to newline char
+  GAME=($GAME) # split the string into an array by the same name
+  IFS=$SAVEIFS   # Restore original IFS
+
+  GAME_ADDR="${GAME[2]}"
+  STATUS=$(cast call --rpc-url "${RPC}" "${GAME_ADDR}" "status() return(uint8)" | cast to-dec)
+  if [[ "${STATUS}" == "0" ]]
+  then
+    STATUS="In Progress"
+  elif [[ "${STATUS}" == "1" ]]
+  then
+    STATUS="Challenger Wins"
+  elif [[ "${STATUS}" == "2" ]]
+  then
+    STATUS="Defender Wins"
+  fi
+  echo "${i}  Game: ${GAME_ADDR} Type: ${GAME[0]} Created: ${GAME[1]} Status: ${STATUS}"
+done

op-challenger/scripts/move.sh

+#!/bin/bash
+set -euo pipefail
+
+RPC=${1:?Must specify RPC URL}
+GAME_ADDR=${2:?Must specify game address}
+ACTION=${3:?Must specify attack or defend}
+PARENT_INDEX=${4:?Must specify parent index. Use latest to counter the latest claim added to the game.}
+CLAIM=${5:?Must specify claim hash}
+SIGNER_ARGS="${@:6}"
+
+if [[ "${ACTION}" != "attack" && "${ACTION}" != "defend" ]]
+then
+  echo "Action must be either attack or defend"
+  exit 1
+fi
+
+if [[ "${PARENT_INDEX}" == "latest" ]]
+then
+  # Fetch the index of the most recent claim made.
+  PARENT_INDEX=$(cast call --rpc-url "${RPC}" "${GAME_ADDR}" 'claimDataLen() returns(uint256)')
+  ((PARENT_INDEX=PARENT_INDEX-1))
+fi
+
+# Perform the move.
+cast send --rpc-url "${RPC}" ${SIGNER_ARGS} "${GAME_ADDR}" "$ACTION(uint256,bytes32)" "${PARENT_INDEX}" "${CLAIM}"

op-challenger/scripts/resolve.sh

+#!/bin/bash
+set -euo pipefail
+
+RPC=${1:?Must specify RPC URL}
+GAME_ADDR=${2:?Must specify game address}
+SIGNER_ARGS="${@:3}"
+
+# Perform the move.
+RESULT_DATA=$(cast send --rpc-url "${RPC}" ${SIGNER_ARGS} "${GAME_ADDR}" "resolve()" --json)
+RESULT=$(echo "${RESULT_DATA}" | jq -r '.logs[0].topics[1]' | cast to-dec)
+
+if [[ "${RESULT}" == "0" ]]
+then
+  RESULT="In Progress"
+elif [[ "${RESULT}" == "1" ]]
+then
+  RESULT="Challenger Wins"
+elif [[ "${RESULT}" == "2" ]]
+then
+  RESULT="Defender Wins"
+fi
+
+echo "Result: $RESULT"

op-challenger/scripts/visualize.sh

@@ -2,23 +2,8 @@
 
 set -euo pipefail
 
-if [ $# -eq 0 ]
-  then
-    echo "Missing Fault Dispute Game address argument"
-fi
-
-echo ""
-echo "Visualize the fault dispute game at https://dispute.clab.by/game?addr=$1"
-echo ""
-
-DISPUTE_GAME_PROXY=$(jq .DisputeGameFactoryProxy .devnet/addresses.json)
-DISPUTE_GAME_PROXY=$(echo $DISPUTE_GAME_PROXY | tr -d '"')
-
-echo "----------------------------------------------------------------"
-echo " Dispute Game Factory at $DISPUTE_GAME_PROXY"
-echo "----------------------------------------------------------------"
-
-FAULT_GAME_ADDRESS=$1
+RPC="${1:?Must specify RPC address}"
+FAULT_GAME_ADDRESS="${2:?Must specify game address}"
 
 DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
 DIR=$(echo ${DIR%/*/*})
@@ -26,6 +11,6 @@ cd $DIR/packages/contracts-bedrock
 
 forge script scripts/FaultDisputeGameViz.s.sol \
   --sig "remote(address)" $FAULT_GAME_ADDRESS \
-  --fork-url http://localhost:8545
+  --fork-url "$RPC"
 
 mv dispute_game.svg "$dir"

op-e2e/Makefile

@@ -13,7 +13,7 @@ test: pre-test test-ws
 
 test-external-%: pre-test
 	make -C ./external_$*/
-	$(go_test) $(go_test_flags) --externalL2 ./external_$*/shim
+	$(go_test) $(go_test_flags) --externalL2 ./external_$*/
 
 test-ws: pre-test
 	$(go_test) $(go_test_flags) ./...

op-e2e/config/init.go

 package config
 
 import (
+	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"os"
@@ -9,6 +11,7 @@ import (
 	"time"
 
 	"github.com/ethereum-optimism/optimism/op-chain-ops/genesis"
+	"github.com/ethereum-optimism/optimism/op-e2e/external"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/state"
 )
@@ -26,21 +29,18 @@ var (
 	L1Deployments *genesis.L1Deployments
 	// DeployConfig represents the deploy config used by the system.
 	DeployConfig *genesis.DeployConfig
-	// ExternalL2Nodes is the shim to use if external ethereum client testing is
+	// ExternalL2Shim is the shim to use if external ethereum client testing is
 	// enabled
-	ExternalL2Nodes string
+	ExternalL2Shim string
+	// ExternalL2TestParms is additional metadata for executing external L2
+	// tests.
+	ExternalL2TestParms external.TestParms
 	// EthNodeVerbosity is the level of verbosity to output
 	EthNodeVerbosity int
 )
 
-// Init testing to enable test flags
-var _ = func() bool {
-	testing.Init()
-	return true
-}()
-
 func init() {
-	var l1AllocsPath, l1DeploymentsPath, deployConfigPath string
+	var l1AllocsPath, l1DeploymentsPath, deployConfigPath, externalL2 string
 
 	cwd, err := os.Getwd()
 	if err != nil {
@@ -58,8 +58,9 @@ func init() {
 	flag.StringVar(&l1AllocsPath, "l1-allocs", defaultL1AllocsPath, "")
 	flag.StringVar(&l1DeploymentsPath, "l1-deployments", defaultL1DeploymentsPath, "")
 	flag.StringVar(&deployConfigPath, "deploy-config", defaultDeployConfigPath, "")
-	flag.StringVar(&ExternalL2Nodes, "externalL2", "", "Enable tests with external L2")
+	flag.StringVar(&externalL2, "externalL2", "", "Enable tests with external L2")
 	flag.IntVar(&EthNodeVerbosity, "ethLogVerbosity", 3, "The level of verbosity to use for the eth node logs")
+	testing.Init() // Register test flags before parsing
 	flag.Parse()
 
 	if err := allExist(l1AllocsPath, l1DeploymentsPath, deployConfigPath); err != nil {
@@ -92,6 +93,40 @@ func init() {
 	if L1Deployments != nil {
 		DeployConfig.SetDeployments(L1Deployments)
 	}
+
+	if externalL2 != "" {
+		if err := initExternalL2(externalL2); err != nil {
+			panic(fmt.Errorf("could not initialize external L2: %w", err))
+		}
+	}
+}
+
+func initExternalL2(externalL2 string) error {
+	var err error
+	ExternalL2Shim, err = filepath.Abs(filepath.Join(externalL2, "shim"))
+	if err != nil {
+		return fmt.Errorf("could not compute abs of externalL2Nodes shim: %w", err)
+	}
+
+	_, err = os.Stat(ExternalL2Shim)
+	if err != nil {
+		return fmt.Errorf("failed to stat externalL2Nodes path: %w", err)
+	}
+
+	file, err := os.Open(filepath.Join(externalL2, "test_parms.json"))
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil
+		}
+		return fmt.Errorf("could not open external L2 test parms: %w", err)
+	}
+	defer file.Close()
+
+	if err := json.NewDecoder(file).Decode(&ExternalL2TestParms); err != nil {
+		return fmt.Errorf("could not decode external L2 test parms: %w", err)
+	}
+
+	return nil
 }
 
 func allExist(filenames ...string) error {

op-e2e/e2eutils/disputegame/cannon_helper.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/cannon"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/challenger"
 	"github.com/ethereum-optimism/optimism/op-node/rollup"
 	"github.com/ethereum-optimism/optimism/op-node/testlog"
@@ -40,7 +41,7 @@ func (g *CannonGameHelper) CreateHonestActor(ctx context.Context, rollupCfg *rol
 	opts = append(opts, options...)
 	cfg := challenger.NewChallengerConfig(g.t, l1Endpoint, opts...)
 	logger := testlog.Logger(g.t, log.LvlInfo).New("role", "CorrectTrace")
-	provider, err := cannon.NewTraceProvider(ctx, logger, cfg, l1Client, filepath.Join(cfg.Datadir, "honest"), g.addr)
+	provider, err := cannon.NewTraceProvider(ctx, logger, metrics.NoopMetrics, cfg, l1Client, filepath.Join(cfg.Datadir, "honest"), g.addr)
 	g.require.NoError(err, "create cannon trace provider")
 
 	return &HonestHelper{

op-e2e/e2eutils/disputegame/helper.go

@@ -14,6 +14,7 @@ import (
 	"github.com/ethereum-optimism/optimism/op-chain-ops/genesis"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/alphabet"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/cannon"
+	"github.com/ethereum-optimism/optimism/op-challenger/metrics"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/challenger"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/transactions"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait"
@@ -175,7 +176,7 @@ func (h *FactoryHelper) StartCannonGameWithCorrectRoot(ctx context.Context, roll
 		L2Claim:       challengedOutput.OutputRoot,
 		L2BlockNumber: challengedOutput.L2BlockNumber,
 	}
-	provider := cannon.NewTraceProviderFromInputs(testlog.Logger(h.t, log.LvlInfo).New("role", "CorrectTrace"), cfg, inputs, cfg.Datadir)
+	provider := cannon.NewTraceProviderFromInputs(testlog.Logger(h.t, log.LvlInfo).New("role", "CorrectTrace"), metrics.NoopMetrics, cfg, inputs, cfg.Datadir)
 	rootClaim, err := provider.Get(ctx, math.MaxUint64)
 	h.require.NoError(err, "Compute correct root hash")
 

op-e2e/external/config.go

 package external
 
 import (
+	"bytes"
 	"encoding/json"
 	"os"
+	"strings"
+	"testing"
 )
 
 type Config struct {
@@ -40,3 +43,26 @@ type Endpoints struct {
 	HTTPAuthEndpoint string `json:"http_auth_endpoint"`
 	WSAuthEndpoint   string `json:"ws_auth_endpoint"`
 }
+
+type TestParms struct {
+	// SkipTests is a map from test name to skip message.  The skip message may
+	// be arbitrary, but the test name should match the skipped test (either
+	// base, or a sub-test) exactly.  Precisely, the skip name must match rune for
+	// rune starting with the first rune.  If the skip name does not match all
+	// runes, the first mismatched rune must be a '/'.
+	SkipTests map[string]string `json:"skip_tests"`
+}
+
+func (tp TestParms) SkipIfNecessary(t *testing.T) {
+	if len(tp.SkipTests) == 0 {
+		return
+	}
+	var base bytes.Buffer
+	for _, name := range strings.Split(t.Name(), "/") {
+		base.WriteString(name)
+		if msg, ok := tp.SkipTests[base.String()]; ok {
+			t.Skip(msg)
+		}
+		base.WriteRune('/')
+	}
+}

op-e2e/external_geth/README.md

@@ -41,6 +41,16 @@ process and looks for the lines indicating that the HTTP server and Auth HTTP
 server have started up.  It then reads the ports which were allocated (because
 the requested ports were passed in as ephemeral via the CLI arguments).
 
+## Skipping tests
+
+Although ideally, all tests would be structured such that they may execute
+either with an in-process op-geth or with an extra-process ethereum client,
+this is not always the case.  You may optionally create a `test_parms.json`
+file in the `external_<your-client>` directory, as there is in the
+`external_geth` directory which specifies a map of tests to skip, and
+accompanying skip text.  See the `op-e2e/external/config.go` file for more
+details.
+
 ## Generalization
 
 This shim is included to help document an demonstrate the usage of the

op-e2e/external_geth/test_parms.json

+{
+  "skip_tests":{
+    "TestPendingGasLimit":"This test requires directly modifying go structures and cannot be implemented with flags"
+  }
+}

op-e2e/setup.go

@@ -78,6 +78,8 @@ func newTxMgrConfig(l1Addr string, privKey *ecdsa.PrivateKey) txmgr.CLIConfig {
 }
 
 func DefaultSystemConfig(t *testing.T) SystemConfig {
+	config.ExternalL2TestParms.SkipIfNecessary(t)
+
 	secrets, err := e2eutils.DefaultMnemonicConfig.Secrets()
 	require.NoError(t, err)
 	deployConfig := config.DeployConfig.Copy()
@@ -139,7 +141,7 @@ func DefaultSystemConfig(t *testing.T) SystemConfig {
 		GethOptions:                map[string][]GethOption{},
 		P2PTopology:                nil, // no P2P connectivity by default
 		NonFinalizedProposals:      false,
-		ExternalL2Nodes:            config.ExternalL2Nodes,
+		ExternalL2Shim:             config.ExternalL2Shim,
 		BatcherTargetL1TxSizeBytes: 100_000,
 	}
 }
@@ -175,7 +177,7 @@ type SystemConfig struct {
 	ProposerLogger log.Logger
 	BatcherLogger  log.Logger
 
-	ExternalL2Nodes string
+	ExternalL2Shim string
 
 	// map of outbound connections to other nodes. Node names prefixed with "~" are unconnected but linked.
 	// A nil map disables P2P completely.
@@ -438,7 +440,7 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste
 
 	for name := range cfg.Nodes {
 		var ethClient EthInstance
-		if cfg.ExternalL2Nodes == "" {
+		if cfg.ExternalL2Shim == "" {
 			node, backend, err := initL2Geth(name, big.NewInt(int64(cfg.DeployConfig.L2ChainID)), l2Genesis, cfg.JWTFilePath, cfg.GethOptions[name]...)
 			if err != nil {
 				return nil, err
@@ -459,7 +461,7 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste
 			}
 			ethClient = (&ExternalRunner{
 				Name:    name,
-				BinPath: cfg.ExternalL2Nodes,
+				BinPath: cfg.ExternalL2Shim,
 				Genesis: l2Genesis,
 				JWTPath: cfg.JWTFilePath,
 			}).Run(t)

op-e2e/system_test.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"math/big"
 	"os"
-	"path/filepath"
 	"runtime"
 	"testing"
 	"time"
@@ -45,23 +44,8 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	if config.ExternalL2Nodes != "" {
-		fmt.Println("Running tests with external L2 process adapter at ", config.ExternalL2Nodes)
-		shimPath, err := filepath.Abs(config.ExternalL2Nodes)
-		if err != nil {
-			fmt.Printf("Could not compute abs of externalL2Nodes shim: %s\n", err)
-			os.Exit(2)
-		}
-		// We convert the passed in path to an absolute path, as it simplifies
-		// the path handling logic for the rest of the testing
-		config.ExternalL2Nodes = shimPath
-
-		_, err = os.Stat(config.ExternalL2Nodes)
-		if err != nil {
-			fmt.Printf("Failed to stat externalL2Nodes path: %s\n", err)
-			os.Exit(3)
-		}
-
+	if config.ExternalL2Shim != "" {
+		fmt.Println("Running tests with external L2 process adapter at ", config.ExternalL2Shim)
 		// As these are integration tests which launch many other processes, the
 		// default parallelism makes the tests flaky.  This change aims to
 		// reduce the flakiness of these tests.
@@ -273,13 +257,6 @@ func TestPendingGasLimit(t *testing.T) {
 	InitParallel(t)
 
 	cfg := DefaultSystemConfig(t)
-	if cfg.ExternalL2Nodes != "" {
-		// Some eth clients such as Erigon don't currently build blocks until
-		// they receive the engine call which includes the gas limit.  After we
-		// provide a mechanism for external clients to advertise test support we
-		// should enable for those which support it.
-		t.Skip()
-	}
 
 	// configure the L2 gas limit to be high, and the pending gas limits to be lower for resource saving.
 	cfg.DeployConfig.L2GenesisBlockGasLimit = 30_000_000

op-program/host/prefetcher/prefetcher.go

@@ -58,14 +58,18 @@ func (p *Prefetcher) Hint(hint string) error {
 func (p *Prefetcher) GetPreimage(ctx context.Context, key common.Hash) ([]byte, error) {
 	p.logger.Trace("Pre-image requested", "key", key)
 	pre, err := p.kvStore.Get(key)
-	if errors.Is(err, kvstore.ErrNotFound) && p.lastHint != "" {
+	// Use a loop to keep retrying the prefetch as long as the key is not found
+	// This handles the case where the prefetch downloads a preimage, but it is then deleted unexpectedly
+	// before we get to read it.
+	for errors.Is(err, kvstore.ErrNotFound) && p.lastHint != "" {
 		hint := p.lastHint
-		p.lastHint = ""
 		if err := p.prefetch(ctx, hint); err != nil {
 			return nil, fmt.Errorf("prefetch failed: %w", err)
 		}
-		// Should now be available
-		return p.kvStore.Get(key)
+		pre, err = p.kvStore.Get(key)
+		if err != nil {
+			p.logger.Error("Fetched pre-images for last hint but did not find required key", "hint", hint, "key", key)
+		}
 	}
 	return pre, err
 }

op-program/host/prefetcher/prefetcher_test.go

@@ -306,6 +306,41 @@ func TestBadHints(t *testing.T) {
 	})
 }
 
+func TestRetryWhenNotAvailableAfterPrefetching(t *testing.T) {
+	rng := rand.New(rand.NewSource(123))
+	node := testutils.RandomData(rng, 30)
+	hash := crypto.Keccak256Hash(node)
+
+	_, l1Source, l2Cl, kv := createPrefetcher(t)
+	putsToIgnore := 2
+	kv = &unreliableKvStore{KV: kv, putsToIgnore: putsToIgnore}
+	prefetcher := NewPrefetcher(testlog.Logger(t, log.LvlInfo), l1Source, l2Cl, kv)
+
+	// Expect one call for each ignored put, plus one more request for when the put succeeds
+	for i := 0; i < putsToIgnore+1; i++ {
+		l2Cl.ExpectNodeByHash(hash, node, nil)
+	}
+	defer l2Cl.MockDebugClient.AssertExpectations(t)
+
+	oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+	result := oracle.NodeByHash(hash)
+	require.EqualValues(t, node, result)
+}
+
+type unreliableKvStore struct {
+	kvstore.KV
+	putsToIgnore int
+}
+
+func (s *unreliableKvStore) Put(k common.Hash, v []byte) error {
+	if s.putsToIgnore > 0 {
+		s.putsToIgnore--
+		return nil
+	}
+	println("storing")
+	return s.KV.Put(k, v)
+}
+
 type l2Client struct {
 	*testutils.MockL2Client
 	*testutils.MockDebugClient

op-ufm/cmd/ufm/main.go

@@ -28,7 +28,10 @@ func main() {
 		),
 	)
 
-	log.Info("initializing", "version", GitVersion, "commit", GitCommit, "date", GitDate)
+	log.Info("initializing",
+		"version", GitVersion,
+		"commit", GitCommit,
+		"date", GitDate)
 
 	if len(os.Args) < 2 {
 		log.Crit("must specify a config file on the command line")
@@ -42,7 +45,8 @@ func main() {
 	sig := make(chan os.Signal, 1)
 	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
 	recvSig := <-sig
-	log.Info("caught signal, shutting down", "signal", recvSig)
+	log.Info("caught signal, shutting down",
+		"signal", recvSig)
 
 	svc.Shutdown()
 }
@@ -50,7 +54,9 @@ func main() {
 func initConfig(cfgFile string) *config.Config {
 	cfg, err := config.New(cfgFile)
 	if err != nil {
-		log.Crit("error reading config file", "file", cfgFile, "err", err)
+		log.Crit("error reading config file",
+			"file", cfgFile,
+			"err", err)
 	}
 
 	// update log level from config
@@ -58,7 +64,8 @@ func initConfig(cfgFile string) *config.Config {
 	if err != nil {
 		logLevel = log.LvlInfo
 		if cfg.LogLevel != "" {
-			log.Warn("invalid server.log_level set: " + cfg.LogLevel)
+			log.Warn("invalid server.log_level",
+				"log_level", cfg.LogLevel)
 		}
 	}
 	log.Root().SetHandler(
@@ -74,7 +81,8 @@ func initConfig(cfgFile string) *config.Config {
 
 	err = cfg.Validate()
 	if err != nil {
-		log.Crit("invalid config", "err", err)
+		log.Crit("invalid config",
+			"err", err)
 	}
 
 	return cfg

op-ufm/pkg/metrics/metrics.go

@@ -84,8 +84,10 @@ var nonAlphanumericRegex = regexp.MustCompile(`[^a-zA-Z ]+`)
 
 func RecordError(provider string, errorLabel string) {
 	if Debug {
-		log.Debug("metric inc", "m", "errors_total",
-			"provider", provider, "error", errorLabel)
+		log.Debug("metric inc",
+			"m", "errors_total",
+			"provider", provider,
+			"error", errorLabel)
 	}
 	errorsTotal.WithLabelValues(provider, errorLabel).Inc()
 }
@@ -101,48 +103,64 @@ func RecordErrorDetails(provider string, label string, err error) {
 
 func RecordRPCLatency(provider string, client string, method string, latency time.Duration) {
 	if Debug {
-		log.Debug("metric set", "m", "rpc_latency",
-			"provider", provider, "client", client, "method", method, "latency", latency)
+		log.Debug("metric set",
+			"m", "rpc_latency",
+			"provider", provider,
+			"client", client,
+			"method", method,
+			"latency", latency)
 	}
 	rpcLatency.WithLabelValues(provider, client, method).Set(float64(latency.Milliseconds()))
 }
 
 func RecordRoundTripLatency(provider string, latency time.Duration) {
 	if Debug {
-		log.Debug("metric set", "m", "roundtrip_latency",
-			"provider", provider, "latency", latency)
+		log.Debug("metric set",
+			"m", "roundtrip_latency",
+			"provider", provider,
+			"latency", latency)
 	}
 	roundTripLatency.WithLabelValues(provider).Set(float64(latency.Milliseconds()))
 }
 
 func RecordGasUsed(provider string, val uint64) {
 	if Debug {
-		log.Debug("metric add", "m", "gas_used",
-			"provider", provider, "val", val)
+		log.Debug("metric add",
+			"m", "gas_used",
+			"provider", provider,
+			"val", val)
 	}
 	gasUsed.WithLabelValues(provider).Set(float64(val))
 }
 
 func RecordFirstSeenLatency(providerSource string, providerSeen string, latency time.Duration) {
 	if Debug {
-		log.Debug("metric set", "m", "first_seen_latency",
-			"provider_source", providerSource, "provider_seen", providerSeen, "latency", latency)
+		log.Debug("metric set",
+			"m", "first_seen_latency",
+			"provider_source", providerSource,
+			"provider_seen", providerSeen,
+			"latency", latency)
 	}
 	firstSeenLatency.WithLabelValues(providerSource, providerSeen).Set(float64(latency.Milliseconds()))
 }
 
 func RecordProviderToProviderLatency(providerSource string, providerSeen string, latency time.Duration) {
 	if Debug {
-		log.Debug("metric set", "m", "provider_to_provider_latency",
-			"provider_source", providerSource, "provider_seen", providerSeen, "latency", latency)
+		log.Debug("metric set",
+			"m", "provider_to_provider_latency",
+			"provider_source", providerSource,
+			"provider_seen", providerSeen,
+			"latency", latency)
 	}
 	providerToProviderLatency.WithLabelValues(providerSource, providerSeen).Set(float64(latency.Milliseconds()))
 }
 
 func RecordTransactionsInFlight(network string, count int) {
 	if Debug {
-		log.Debug("metric set", "m", "transactions_inflight",
-			"network", network, "count", count)
+		log.Debug("metric set",
+			"m", "transactions_inflight",
+			"network", network,
+			"count", count)
 	}
 	networkTransactionsInFlight.WithLabelValues(network).Set(float64(count))
 }

op-ufm/pkg/provider/heartbeat.go

@@ -14,7 +14,9 @@ import (
 
 // Heartbeat polls for expected in-flight transactions
 func (p *Provider) Heartbeat(ctx context.Context) {
-	log.Debug("heartbeat", "provider", p.name, "inflight", len(p.txPool.Transactions))
+	log.Debug("heartbeat",
+		"provider", p.name,
+		"count", len(p.txPool.Transactions))
 
 	metrics.RecordTransactionsInFlight(p.config.Network, len(p.txPool.Transactions))
 
@@ -33,26 +35,42 @@ func (p *Provider) Heartbeat(ctx context.Context) {
 	}
 
 	if len(expectedTransactions) == 0 {
-		log.Debug("no expected txs", "count", len(p.txPool.Transactions), "provider", p.name, "alreadySeen", alreadySeen)
+		log.Debug("no expected txs",
+			"count", len(p.txPool.Transactions),
+			"provider", p.name,
+			"alreadySeen", alreadySeen)
 		return
 	}
 
 	client, err := clients.Dial(p.name, p.config.URL)
 	if err != nil {
-		log.Error("cant dial to provider", "provider", p.name, "url", p.config.URL, "err", err)
+		log.Error("cant dial to provider",
+			"provider", p.name,
+			"url", p.config.URL,
+			"err", err)
 	}
 
-	log.Debug("checking in-flight tx", "count", len(p.txPool.Transactions), "provider", p.name, "alreadySeen", alreadySeen)
+	log.Debug("checking in-flight tx",
+		"count", len(p.txPool.Transactions),
+		"provider", p.name,
+		"alreadySeen", alreadySeen)
 	for _, st := range expectedTransactions {
 		hash := st.Hash.Hex()
 
 		_, isPending, err := client.TransactionByHash(ctx, st.Hash)
 		if err != nil && !errors.Is(err, ethereum.NotFound) {
-			log.Error("cant check transaction", "provider", p.name, "hash", hash, "url", p.config.URL, "err", err)
+			log.Error("cant check transaction",
+				"provider", p.name,
+				"hash", hash,
+				"url", p.config.URL,
+				"err", err)
 			continue
 		}
 
-		log.Debug("got transaction", "provider", p.name, "hash", hash, "isPending", isPending)
+		log.Debug("got transaction",
+			"provider", p.name,
+			"hash", hash,
+			"isPending", isPending)
 
 		// mark transaction as seen by this provider
 		st.M.Lock()
@@ -75,7 +93,10 @@ func (p *Provider) Heartbeat(ctx context.Context) {
 		// check if transaction have been seen by all providers
 		p.txPool.M.Lock()
 		if len(st.SeenBy) == p.txPool.Expected {
-			log.Debug("transaction seen by all", "hash", hash, "expected", p.txPool.Expected, "seenBy", len(st.SeenBy))
+			log.Debug("transaction seen by all",
+				"hash", hash,
+				"expected", p.txPool.Expected,
+				"seenBy", len(st.SeenBy))
 			delete(p.txPool.Transactions, st.Hash.Hex())
 		}
 		p.txPool.M.Unlock()

op-ufm/pkg/provider/roundtrip.go

@@ -6,11 +6,11 @@ import (
 
 	"github.com/ethereum-optimism/optimism/op-ufm/pkg/metrics"
 	iclients "github.com/ethereum-optimism/optimism/op-ufm/pkg/metrics/clients"
+	"github.com/ethereum/go-ethereum/core"
 
 	"github.com/ethereum-optimism/optimism/op-service/tls"
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/txpool"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/pkg/errors"
@@ -21,19 +21,35 @@ import (
 
 // RoundTrip send a new transaction to measure round trip latency
 func (p *Provider) RoundTrip(ctx context.Context) {
-	log.Debug("roundTripLatency", "provider", p.name)
+	log.Debug("roundTripLatency",
+		"provider", p.name)
 
 	client, err := iclients.Dial(p.name, p.config.URL)
 	if err != nil {
-		log.Error("cant dial to provider", "provider", p.name, "url", p.config.URL, "err", err)
+		log.Error("cant dial to provider",
+			"provider", p.name,
+			"url", p.config.URL,
+			"err", err)
 		return
 	}
 
-	nonce, err := client.PendingNonceAt(ctx, p.walletConfig.Address)
-	if err != nil {
-		log.Error("cant get nounce", "provider", p.name, "err", err)
-		return
+	var nonce uint64
+	p.txPool.M.Lock()
+	if p.txPool.Nonce == uint64(0) {
+		nonce, err = client.PendingNonceAt(ctx, p.walletConfig.Address)
+		if err != nil {
+			log.Error("cant get nounce",
+				"provider", p.name,
+				"err", err)
+			p.txPool.M.Unlock()
+			return
+		}
+		p.txPool.Nonce = nonce
+	} else {
+		p.txPool.Nonce++
+		nonce = p.txPool.Nonce
 	}
+	p.txPool.M.Unlock()
 
 	txHash := common.Hash{}
 	attempt := 0
@@ -47,7 +63,10 @@ func (p *Provider) RoundTrip(ctx context.Context) {
 
 		signedTx, err := p.sign(ctx, tx)
 		if err != nil {
-			log.Error("cant sign tx", "provider", p.name, "tx", tx, "err", err)
+			log.Error("cant sign tx",
+				"provider", p.name,
+				"tx", tx,
+				"err", err)
 			return
 		}
 
@@ -56,21 +75,40 @@ func (p *Provider) RoundTrip(ctx context.Context) {
 		roundTripStartedAt = time.Now()
 		err = client.SendTransaction(ctx, signedTx)
 		if err != nil {
-			if err.Error() == txpool.ErrAlreadyKnown.Error() || err.Error() == core.ErrNonceTooLow.Error() {
+			if err.Error() == txpool.ErrAlreadyKnown.Error() ||
+				err.Error() == txpool.ErrReplaceUnderpriced.Error() ||
+				err.Error() == core.ErrNonceTooLow.Error() {
 				if time.Since(firstAttemptAt) >= time.Duration(p.config.SendTransactionRetryTimeout) {
-					log.Error("send transaction timed out (known already)", "provider", p.name, "hash", txHash.Hex(), "elapsed", time.Since(firstAttemptAt), "attempt", attempt, "nonce", nonce)
+					log.Error("send transaction timed out (known already)",
+						"provider", p.name,
+						"hash", txHash.Hex(),
+						"elapsed", time.Since(firstAttemptAt),
+						"attempt", attempt,
+						"nonce", nonce)
 					metrics.RecordError(p.name, "ethclient.SendTransaction.nonce")
 					return
 				}
-				log.Warn("tx already known, incrementing nonce and trying again", "provider", p.name, "nonce", nonce)
+				log.Warn("tx already known, incrementing nonce and trying again",
+					"provider", p.name,
+					"nonce", nonce)
 				time.Sleep(time.Duration(p.config.SendTransactionRetryInterval))
-				nonce++
+
+				p.txPool.M.Lock()
+				p.txPool.Nonce++
+				nonce = p.txPool.Nonce
+				p.txPool.M.Unlock()
 				attempt++
 				if attempt%10 == 0 {
-					log.Debug("retrying send transaction...", "provider", p.name, "attempt", attempt, "nonce", nonce, "elapsed", time.Since(firstAttemptAt))
+					log.Debug("retrying send transaction...",
+						"provider", p.name,
+						"attempt", attempt,
+						"nonce", nonce,
+						"elapsed", time.Since(firstAttemptAt))
 				}
 			} else {
-				log.Error("cant send transaction", "provider", p.name, "err", err)
+				log.Error("cant send transaction",
+					"provider", p.name,
+					"err", err)
 				metrics.RecordErrorDetails(p.name, "ethclient.SendTransaction", err)
 				return
 			}
@@ -79,7 +117,10 @@ func (p *Provider) RoundTrip(ctx context.Context) {
 		}
 	}
 
-	log.Info("transaction sent", "provider", p.name, "hash", txHash.Hex(), "nonce", nonce)
+	log.Info("transaction sent",
+		"provider", p.name,
+		"hash", txHash.Hex(),
+		"nonce", nonce)
 
 	// add to pool
 	sentAt := time.Now()
@@ -96,16 +137,25 @@ func (p *Provider) RoundTrip(ctx context.Context) {
 	attempt = 0
 	for receipt == nil {
 		if time.Since(sentAt) >= time.Duration(p.config.ReceiptRetrievalTimeout) {
-			log.Error("receipt retrieval timed out", "provider", p.name, "hash", "elapsed", time.Since(sentAt))
+			log.Error("receipt retrieval timed out",
+				"provider", p.name,
+				"hash", txHash,
+				"elapsed", time.Since(sentAt))
 			return
 		}
 		time.Sleep(time.Duration(p.config.ReceiptRetrievalInterval))
 		if attempt%10 == 0 {
-			log.Debug("checking for receipt...", "provider", p.name, "attempt", attempt, "elapsed", time.Since(sentAt))
+			log.Debug("checking for receipt...",
+				"provider", p.name,
+				"attempt", attempt,
+				"elapsed", time.Since(sentAt))
 		}
 		receipt, err = client.TransactionReceipt(ctx, txHash)
 		if err != nil && !errors.Is(err, ethereum.NotFound) {
-			log.Error("cant get receipt for transaction", "provider", p.name, "hash", txHash.Hex(), "err", err)
+			log.Error("cant get receipt for transaction",
+				"provider", p.name,
+				"hash", txHash.Hex(),
+				"err", err)
 			return
 		}
 		attempt++
@@ -116,7 +166,8 @@ func (p *Provider) RoundTrip(ctx context.Context) {
 	metrics.RecordRoundTripLatency(p.name, roundTripLatency)
 	metrics.RecordGasUsed(p.name, receipt.GasUsed)
 
-	log.Info("got transaction receipt", "hash", txHash.Hex(),
+	log.Info("got transaction receipt",
+		"hash", txHash.Hex(),
 		"roundTripLatency", roundTripLatency,
 		"provider", p.name,
 		"blockNumber", receipt.BlockNumber,
@@ -155,7 +206,9 @@ func (p *Provider) sign(ctx context.Context, tx *types.Transaction) (*types.Tran
 			TLSKey:    p.signerConfig.TLSKey,
 		}
 		client, err := iclients.NewSignerClient(p.name, log.Root(), p.signerConfig.URL, tlsConfig)
-		log.Debug("signerclient", "client", client, "err", err)
+		log.Debug("signerclient",
+			"client", client,
+			"err", err)
 		if err != nil {
 			return nil, err
 		}

op-ufm/pkg/provider/tx_pool.go

@@ -15,6 +15,7 @@ type NetworkTransactionPool struct {
 	M            sync.Mutex
 	Transactions map[string]*TransactionState
 	Expected     int
+	Nonce        uint64
 }
 
 type TransactionState struct {

op-ufm/pkg/service/service.go

@@ -32,10 +32,12 @@ func (s *Service) Start(ctx context.Context) {
 	log.Info("service starting")
 	if s.Config.Healthz.Enabled {
 		addr := net.JoinHostPort(s.Config.Healthz.Host, s.Config.Healthz.Port)
-		log.Info("starting healthz server", "addr", addr)
+		log.Info("starting healthz server",
+			"addr", addr)
 		go func() {
 			if err := s.Healthz.Start(ctx, addr); err != nil {
-				log.Error("error starting healthz server", "err", err)
+				log.Error("error starting healthz server",
+					"err", err)
 			}
 		}()
 	}
@@ -43,10 +45,12 @@ func (s *Service) Start(ctx context.Context) {
 	metrics.Debug = s.Config.Metrics.Debug
 	if s.Config.Metrics.Enabled {
 		addr := net.JoinHostPort(s.Config.Metrics.Host, s.Config.Metrics.Port)
-		log.Info("starting metrics server", "addr", addr)
+		log.Info("starting metrics server",
+			"addr", addr)
 		go func() {
 			if err := s.Metrics.Start(ctx, addr); err != nil {
-				log.Error("error starting metrics server", "err", err)
+				log.Error("error starting metrics server",
+					"err", err)
 			}
 		}()
 	}
@@ -60,7 +64,8 @@ func (s *Service) Start(ctx context.Context) {
 	txpool := &provider.TransactionPool{}
 	for name, providers := range networks {
 		if len(providers) == 1 {
-			log.Warn("can't measure first seen for network, please another provider", "network", name)
+			log.Warn("can't measure first seen for network, please another provider",
+				"network", name)
 		}
 		(*txpool)[name] = &provider.NetworkTransactionPool{}
 		(*txpool)[name].Transactions = make(map[string]*provider.TransactionState)
@@ -76,7 +81,8 @@ func (s *Service) Start(ctx context.Context) {
 			s.Config.Wallets[providerConfig.Wallet],
 			(*txpool)[providerConfig.Network])
 		s.Providers[name].Start(ctx)
-		log.Info("provider started", "provider", name)
+		log.Info("provider started",
+			"provider", name)
 	}
 
 	log.Info("service started")
@@ -94,7 +100,8 @@ func (s *Service) Shutdown() {
 	}
 	for name, provider := range s.Providers {
 		provider.Shutdown()
-		log.Info("provider stopped", "provider", name)
+		log.Info("provider stopped",
+			"provider", name)
 	}
 	log.Info("service stopped")
 }