op-service, op-batcher, op-proposer: Active sequencer follow mode (#8585)

* op-service: Add ActiveL2EndpointProvider.

* Fix bug in initialization, and handle case where no ethUrls are provided.

* Split active L2 provider into active rollup and active L2 provider.

* Re-duplicate some code until tests are passing.

* op-proposer: Add ability to enable active provider.

* op-batcher: Add ability to enable active provider.

* Add an empty test skeleton.

* Add an empty test skeleton.

* op-service: add, but do not yet use, RollupClientInterface and EthClientInterface.

* op-service: update mocks and interfaces for endpoint provider testing.

* op-service - WIP on Active L2 Providers: unit tests pass, design and impl contains TODOs.

* op-service: restore design in Active Endpoint Providers that only keeps one client open at a time.

* op-service: when dialing a new sequencer, close() the old connection.

* op-service: obey coderabbit suggestion around safer handling of p.currentIndex in Active L2 Providers.

* op-service, op-batcher, op-proposer: address review comments in PR#8585.

* op-service: Active L2 Provider - add test case for a sequencer returning an error.

* op-service: Active L2/Rollup Providers: improve unit testing and logging.

* op-service, op-batcher: address review comments in 8585 regarding first-startup behavior and testing.

* op-service: address review comments through adding more tests, and moving "nil client" behavior from client getter to constructor.

* op-service: minor error message change in active endpoint providers.

* Update op-service/dial/active_l2_provider.go
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* op-service: obey linter in rabbit-provided error message change.

* Update op-service/dial/active_l2_provider.go
Co-authored-by: Sebastian Stammler <seb@oplabs.co>

* op-service active L2 provider tests: assertAllExpectations after most tests.

* op-service: more elegantly handle startup in active l2 providers, and improve testing.

* Change remaining longDurationTests to be able to use ept.assertAllExpectations.

* use new error errSeqUnset.

* Add test for scenario where many sequencers are inactive, and only the last is active.

* Readability change: move the on-creation initialization to its own function.

* Move extra one-time dial to constructor.

* Update op-service/dial/active_rollup_provider.go
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* Add nil check to active l2 provider.

* Update op-service/dial/active_rollup_provider.go
Co-authored-by: Sebastian Stammler <seb@oplabs.co>

* Address review comment: change many-inactive tests to many-undialable tests.

* Add test that reproduces internal state corruption.

* op-service: Improve active seq provider

- Preserve the invariant that the index and current rollup/eth
  client match.
- Dial at the start of the loop instead of at the end.

* Fix some tests.

* Move usage of ExpectClose to MaybeClose, we don't want to enforce a particular close behavior in these tests.

* add a missing call to assertAllExpectations.

* Test even the case where the active providers are managing a list of 1 element.

* Revert experimental hunk in active_l2_provider.

---------
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Co-authored-by: Sebastian Stammler <seb@oplabs.co>

op-batcher/batcher/config.go

@@ -3,6 +3,7 @@ package batcher
 import (
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/urfave/cli/v2"
@@ -20,10 +21,10 @@ type CLIConfig struct {
 	// L1EthRpc is the HTTP provider URL for L1.
 	L1EthRpc string
 
-	// L2EthRpc is the HTTP provider URL for the L2 execution engine.
+	// L2EthRpc is the HTTP provider URL for the L2 execution engine. A comma-separated list enables the active L2 provider. Such a list needs to match the number of RollupRpcs provided.
 	L2EthRpc string
 
-	// RollupRpc is the HTTP provider URL for the L2 rollup node.
+	// RollupRpc is the HTTP provider URL for the L2 rollup node. A comma-separated list enables the active L2 provider. Such a list needs to match the number of L2EthRpcs provided.
 	RollupRpc string
 
 	// MaxChannelDuration is the maximum duration (in #L1-blocks) to keep a
@@ -74,6 +75,9 @@ func (c *CLIConfig) Check() error {
 	if c.RollupRpc == "" {
 		return errors.New("empty rollup RPC URL")
 	}
+	if strings.Count(c.RollupRpc, ",") != strings.Count(c.L2EthRpc, ",") {
+		return errors.New("number of rollup and eth URLs must match")
+	}
 	if c.PollInterval == 0 {
 		return errors.New("must set PollInterval")
 	}

op-batcher/batcher/service.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	_ "net/http/pprof"
 	"strconv"
+	"strings"
 	"sync/atomic"
 	"time"
 
@@ -125,9 +126,16 @@ func (bs *BatcherService) initRPCClients(ctx context.Context, cfg *CLIConfig) er
 	}
 	bs.L1Client = l1Client
 
-	endpointProvider, err := dial.NewStaticL2EndpointProvider(ctx, bs.Log, cfg.L2EthRpc, cfg.RollupRpc)
+	var endpointProvider dial.L2EndpointProvider
+	if strings.Contains(cfg.RollupRpc, ",") && strings.Contains(cfg.L2EthRpc, ",") {
+		rollupUrls := strings.Split(cfg.RollupRpc, ",")
+		ethUrls := strings.Split(cfg.L2EthRpc, ",")
+		endpointProvider, err = dial.NewActiveL2EndpointProvider(ctx, ethUrls, rollupUrls, dial.DefaultActiveSequencerFollowerCheckDuration, dial.DefaultDialTimeout, bs.Log)
+	} else {
+		endpointProvider, err = dial.NewStaticL2EndpointProvider(ctx, bs.Log, cfg.L2EthRpc, cfg.RollupRpc)
+	}
 	if err != nil {
-		return fmt.Errorf("failed to create L2 endpoint provider: %w", err)
+		return fmt.Errorf("failed to build L2 endpoint provider: %w", err)
 	}
 	bs.EndpointProvider = endpointProvider
 

op-batcher/flags/flags.go

@@ -30,12 +30,12 @@ var (
 	}
 	L2EthRpcFlag = &cli.StringFlag{
 		Name:    "l2-eth-rpc",
-		Usage:   "HTTP provider URL for L2 execution engine",
+		Usage:   "HTTP provider URL for L2 execution engine. A comma-separated list enables the active L2 endpoint provider. Such a list needs to match the number of rollup-rpcs provided.",
 		EnvVars: prefixEnvVars("L2_ETH_RPC"),
 	}
 	RollupRpcFlag = &cli.StringFlag{
 		Name:    "rollup-rpc",
-		Usage:   "HTTP provider URL for Rollup node",
+		Usage:   "HTTP provider URL for Rollup node. A comma-separated list enables the active L2 endpoint provider. Such a list needs to match the number of l2-eth-rpcs provided.",
 		EnvVars: prefixEnvVars("ROLLUP_RPC"),
 	}
 	// Optional flags

op-proposer/flags/flags.go

@@ -29,7 +29,7 @@ var (
 	}
 	RollupRpcFlag = &cli.StringFlag{
 		Name:    "rollup-rpc",
-		Usage:   "HTTP provider URL for the rollup node",
+		Usage:   "HTTP provider URL for the rollup node. A comma-separated list enables the active rollup provider.",
 		EnvVars: prefixEnvVars("ROLLUP_RPC"),
 	}
 	L2OOAddressFlag = &cli.StringFlag{

op-proposer/proposer/config.go

@@ -22,7 +22,7 @@ type CLIConfig struct {
 	// L1EthRpc is the HTTP provider URL for L1.
 	L1EthRpc string
 
-	// RollupRpc is the HTTP provider URL for the rollup node.
+	// RollupRpc is the HTTP provider URL for the rollup node. A comma-separated list enables the active rollup provider.
 	RollupRpc string
 
 	// L2OOAddress is the L2OutputOracle contract address.

op-proposer/proposer/service.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net"
 	"strconv"
+	"strings"
 	"sync/atomic"
 	"time"
 
@@ -121,7 +122,13 @@ func (ps *ProposerService) initRPCClients(ctx context.Context, cfg *CLIConfig) e
 	}
 	ps.L1Client = l1Client
 
-	rollupProvider, err := dial.NewStaticL2RollupProvider(ctx, ps.Log, cfg.RollupRpc)
+	var rollupProvider dial.RollupProvider
+	if strings.Contains(cfg.RollupRpc, ",") {
+		rollupUrls := strings.Split(cfg.RollupRpc, ",")
+		rollupProvider, err = dial.NewActiveL2RollupProvider(ctx, rollupUrls, dial.DefaultActiveSequencerFollowerCheckDuration, dial.DefaultDialTimeout, ps.Log)
+	} else {
+		rollupProvider, err = dial.NewStaticL2RollupProvider(ctx, ps.Log, cfg.RollupRpc)
+	}
 	if err != nil {
 		return fmt.Errorf("failed to build L2 endpoint provider: %w", err)
 	}

op-service/dial/active_l2_provider.go

+package dial
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+const DefaultActiveSequencerFollowerCheckDuration = 2 * DefaultDialTimeout
+
+type ethDialer func(ctx context.Context, timeout time.Duration, log log.Logger, url string) (EthClientInterface, error)
+
+// ActiveL2EndpointProvider is an interface for providing a RollupClient and l2 eth client
+// It manages the lifecycle of the RollupClient and eth client for callers
+// It does this by failing over down the list of rollupUrls if the current one is inactive or broken
+type ActiveL2EndpointProvider struct {
+	ActiveL2RollupProvider
+	currentEthClient EthClientInterface
+	ethClientIndex   int
+	ethDialer        ethDialer
+	ethUrls          []string
+}
+
+// NewActiveL2EndpointProvider creates a new ActiveL2EndpointProvider
+// the checkDuration is the duration between checks to see if the current rollup client is active
+// provide a checkDuration of 0 to check every time
+func NewActiveL2EndpointProvider(ctx context.Context,
+	ethUrls, rollupUrls []string,
+	checkDuration time.Duration,
+	networkTimeout time.Duration,
+	logger log.Logger,
+) (*ActiveL2EndpointProvider, error) {
+	ethDialer := func(ctx context.Context, timeout time.Duration,
+		log log.Logger, url string,
+	) (EthClientInterface, error) {
+		return DialEthClientWithTimeout(ctx, timeout, log, url)
+	}
+	rollupDialer := func(ctx context.Context, timeout time.Duration,
+		log log.Logger, url string,
+	) (RollupClientInterface, error) {
+		return DialRollupClientWithTimeout(ctx, timeout, log, url)
+	}
+	return newActiveL2EndpointProvider(ctx, ethUrls, rollupUrls, checkDuration, networkTimeout, logger, ethDialer, rollupDialer)
+}
+
+func newActiveL2EndpointProvider(
+	ctx context.Context,
+	ethUrls, rollupUrls []string,
+	checkDuration time.Duration,
+	networkTimeout time.Duration,
+	logger log.Logger,
+	ethDialer ethDialer,
+	rollupDialer rollupDialer,
+) (*ActiveL2EndpointProvider, error) {
+	if len(rollupUrls) == 0 {
+		return nil, errors.New("empty rollup urls list, expected at least one URL")
+	}
+	if len(ethUrls) != len(rollupUrls) {
+		return nil, fmt.Errorf("number of eth urls (%d) and rollup urls (%d) mismatch", len(ethUrls), len(rollupUrls))
+	}
+
+	rollupProvider, err := newActiveL2RollupProvider(ctx, rollupUrls, checkDuration, networkTimeout, logger, rollupDialer)
+	if err != nil {
+		return nil, err
+	}
+	p := &ActiveL2EndpointProvider{
+		ActiveL2RollupProvider: *rollupProvider,
+		ethDialer:              ethDialer,
+		ethUrls:                ethUrls,
+	}
+	cctx, cancel := context.WithTimeout(ctx, networkTimeout)
+	defer cancel()
+	if _, err = p.EthClient(cctx); err != nil {
+		return nil, fmt.Errorf("setting provider eth client: %w", err)
+	}
+	return p, nil
+}
+
+func (p *ActiveL2EndpointProvider) EthClient(ctx context.Context) (EthClientInterface, error) {
+	p.clientLock.Lock()
+	defer p.clientLock.Unlock()
+	err := p.ensureActiveEndpoint(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if p.ethClientIndex != p.rollupIndex || p.currentEthClient == nil {
+		// we changed sequencers, dial a new EthClient
+		cctx, cancel := context.WithTimeout(ctx, p.networkTimeout)
+		defer cancel()
+		idx := p.rollupIndex
+		ep := p.ethUrls[idx]
+		log.Info("sequencer changed (or ethClient was nil due to startup), dialing new eth client", "new_index", idx, "new_url", ep)
+		ethClient, err := p.ethDialer(cctx, p.networkTimeout, p.log, ep)
+		if err != nil {
+			return nil, fmt.Errorf("dialing eth client: %w", err)
+		}
+		if p.currentEthClient != nil {
+			p.currentEthClient.Close()
+		}
+		p.ethClientIndex = idx
+		p.currentEthClient = ethClient
+	}
+	return p.currentEthClient, nil
+}
+
+func (p *ActiveL2EndpointProvider) Close() {
+	if p.currentEthClient != nil {
+		p.currentEthClient.Close()
+	}
+	p.ActiveL2RollupProvider.Close()
+}

op-service/dial/active_rollup_provider.go

+package dial
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+type rollupDialer func(ctx context.Context, timeout time.Duration, log log.Logger, url string) (RollupClientInterface, error)
+
+// ActiveL2EndpointProvider is an interface for providing a RollupClient
+// It manages the lifecycle of the RollupClient for callers
+// It does this by failing over down the list of rollupUrls if the current one is inactive or broken
+type ActiveL2RollupProvider struct {
+	checkDuration  time.Duration
+	networkTimeout time.Duration
+	log            log.Logger
+
+	activeTimeout time.Time
+
+	rollupUrls          []string
+	rollupDialer        rollupDialer
+	currentRollupClient RollupClientInterface
+	rollupIndex         int
+	clientLock          *sync.Mutex
+}
+
+// NewActiveL2RollupProvider creates a new ActiveL2RollupProvider
+// the checkDuration is the duration between checks to see if the current rollup client is active
+// provide a checkDuration of 0 to check every time
+func NewActiveL2RollupProvider(
+	ctx context.Context,
+	rollupUrls []string,
+	checkDuration time.Duration,
+	networkTimeout time.Duration,
+	logger log.Logger,
+) (*ActiveL2RollupProvider, error) {
+	rollupDialer := func(ctx context.Context, timeout time.Duration,
+		log log.Logger, url string,
+	) (RollupClientInterface, error) {
+		return DialRollupClientWithTimeout(ctx, timeout, log, url)
+	}
+	return newActiveL2RollupProvider(ctx, rollupUrls, checkDuration, networkTimeout, logger, rollupDialer)
+}
+
+func newActiveL2RollupProvider(
+	ctx context.Context,
+	rollupUrls []string,
+	checkDuration time.Duration,
+	networkTimeout time.Duration,
+	logger log.Logger,
+	dialer rollupDialer,
+) (*ActiveL2RollupProvider, error) {
+	if len(rollupUrls) == 0 {
+		return nil, errors.New("empty rollup urls list")
+	}
+	p := &ActiveL2RollupProvider{
+		checkDuration:  checkDuration,
+		networkTimeout: networkTimeout,
+		log:            logger,
+		rollupUrls:     rollupUrls,
+		rollupDialer:   dialer,
+		clientLock:     &sync.Mutex{},
+	}
+	cctx, cancel := context.WithTimeout(ctx, networkTimeout)
+	defer cancel()
+
+	if _, err := p.RollupClient(cctx); err != nil {
+		return nil, fmt.Errorf("setting provider rollup client: %w", err)
+	}
+	return p, nil
+}
+
+func (p *ActiveL2RollupProvider) RollupClient(ctx context.Context) (RollupClientInterface, error) {
+	p.clientLock.Lock()
+	defer p.clientLock.Unlock()
+	err := p.ensureActiveEndpoint(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return p.currentRollupClient, nil
+}
+
+func (p *ActiveL2RollupProvider) ensureActiveEndpoint(ctx context.Context) error {
+	if !p.shouldCheck() {
+		return nil
+	}
+	if err := p.findActiveEndpoints(ctx); err != nil {
+		return err
+	}
+	p.activeTimeout = time.Now().Add(p.checkDuration)
+	return nil
+}
+
+func (p *ActiveL2RollupProvider) shouldCheck() bool {
+	return time.Now().After(p.activeTimeout)
+}
+
+func (p *ActiveL2RollupProvider) findActiveEndpoints(ctx context.Context) error {
+	startIdx := p.rollupIndex
+	var errs error
+	for offset := range p.rollupUrls {
+		idx := (startIdx + offset) % p.numEndpoints()
+		if offset != 0 || p.currentRollupClient == nil {
+			if err := p.dialSequencer(ctx, idx); err != nil {
+				errs = errors.Join(errs, err)
+				p.log.Warn("Error dialing next sequencer.", "err", err, "index", p.rollupIndex)
+				continue
+			}
+		}
+
+		ep := p.rollupUrls[idx]
+		if active, err := p.checkCurrentSequencer(ctx); err != nil {
+			errs = errors.Join(errs, err)
+			p.log.Warn("Error querying active sequencer, trying next.", "err", err, "index", idx, "url", ep)
+		} else if active {
+			if offset == 0 {
+				p.log.Debug("Current sequencer active.", "index", idx, "url", ep)
+			} else {
+				p.log.Info("Found new active sequencer.", "index", idx, "url", ep)
+			}
+			return nil
+		} else {
+			p.log.Info("Sequencer inactive, trying next.", "index", idx, "url", ep)
+		}
+	}
+	return fmt.Errorf("failed to find an active sequencer, tried following urls: %v; errs: %w", p.rollupUrls, errs)
+}
+
+func (p *ActiveL2RollupProvider) checkCurrentSequencer(ctx context.Context) (bool, error) {
+	cctx, cancel := context.WithTimeout(ctx, p.networkTimeout)
+	defer cancel()
+	return p.currentRollupClient.SequencerActive(cctx)
+}
+
+func (p *ActiveL2RollupProvider) numEndpoints() int {
+	return len(p.rollupUrls)
+}
+
+// dialSequencer dials the sequencer for the url at the given index.
+// If successful, the currentRollupClient and rollupIndex are updated and the
+// old rollup client is closed.
+func (p *ActiveL2RollupProvider) dialSequencer(ctx context.Context, idx int) error {
+	cctx, cancel := context.WithTimeout(ctx, p.networkTimeout)
+	defer cancel()
+
+	ep := p.rollupUrls[idx]
+	p.log.Info("Dialing next sequencer.", "index", idx, "url", ep)
+	rollupClient, err := p.rollupDialer(cctx, p.networkTimeout, p.log, ep)
+	if err != nil {
+		return fmt.Errorf("dialing rollup client: %w", err)
+	}
+	if p.currentRollupClient != nil {
+		p.currentRollupClient.Close()
+	}
+	p.rollupIndex = idx
+	p.currentRollupClient = rollupClient
+	return nil
+}
+
+func (p *ActiveL2RollupProvider) Close() {
+	if p.currentRollupClient != nil {
+		p.currentRollupClient.Close()
+	}
+}

op-service/dial/ethclient_interface.go

+package dial
+
+import (
+	"context"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+// EthClientInterface is an interface for providing an ethclient.Client
+// It does not describe all of the functions an ethclient.Client has, only the ones used by callers of the L2 Providers
+type EthClientInterface interface {
+	BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error)
+
+	Close()
+}

op-service/dial/rollupclient_interface.go

+package dial
+
+import (
+	"context"
+
+	"github.com/ethereum-optimism/optimism/op-node/rollup"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// RollupClientInterface is an interface for providing a RollupClient
+// It does not describe all of the functions a RollupClient has, only the ones used by the L2 Providers and their callers
+type RollupClientInterface interface {
+	OutputAtBlock(ctx context.Context, blockNum uint64) (*eth.OutputResponse, error)
+	SyncStatus(ctx context.Context) (*eth.SyncStatus, error)
+	RollupConfig(ctx context.Context) (*rollup.Config, error)
+	StartSequencer(ctx context.Context, unsafeHead common.Hash) error
+	SequencerActive(ctx context.Context) (bool, error)
+	Close()
+}

op-service/dial/static_l2_provider.go

@@ -13,7 +13,7 @@ import (
 type L2EndpointProvider interface {
 	RollupProvider
 	// EthClient(ctx) returns the underlying ethclient pointing to the L2 execution node
-	EthClient(ctx context.Context) (*ethclient.Client, error)
+	EthClient(ctx context.Context) (EthClientInterface, error)
 }
 
 // StaticL2EndpointProvider is a L2EndpointProvider that always returns the same static RollupClient and eth client
@@ -38,7 +38,7 @@ func NewStaticL2EndpointProvider(ctx context.Context, log log.Logger, ethClientU
 	}, nil
 }
 
-func (p *StaticL2EndpointProvider) EthClient(context.Context) (*ethclient.Client, error) {
+func (p *StaticL2EndpointProvider) EthClient(context.Context) (EthClientInterface, error) {
 	return p.ethClient, nil
 }
 

op-service/dial/static_rollup_provider.go

@@ -11,7 +11,7 @@ import (
 // It manages the lifecycle of the RollupClient for callers
 type RollupProvider interface {
 	// RollupClient(ctx) returns the underlying sources.RollupClient pointing to the L2 rollup consensus node
-	RollupClient(ctx context.Context) (*sources.RollupClient, error)
+	RollupClient(ctx context.Context) (RollupClientInterface, error)
 	// Close() closes the underlying client or clients
 	Close()
 }
@@ -39,7 +39,7 @@ func NewStaticL2RollupProviderFromExistingRollup(rollupCl *sources.RollupClient)
 	}, nil
 }
 
-func (p *StaticL2RollupProvider) RollupClient(context.Context) (*sources.RollupClient, error) {
+func (p *StaticL2RollupProvider) RollupClient(context.Context) (RollupClientInterface, error) {
 	return p.rollupClient, nil
 }
 

op-service/testutils/mock_eth_client.go

@@ -2,6 +2,7 @@ package testutils
 
 import (
 	"context"
+	"math/big"
 
 	"github.com/stretchr/testify/mock"
 
@@ -131,3 +132,24 @@ func (m *MockEthClient) ReadStorageAt(ctx context.Context, address common.Addres
 func (m *MockEthClient) ExpectReadStorageAt(ctx context.Context, address common.Address, storageSlot common.Hash, blockHash common.Hash, result common.Hash, err error) {
 	m.Mock.On("ReadStorageAt", address, storageSlot, blockHash).Once().Return(result, err)
 }
+
+func (m *MockEthClient) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) {
+	out := m.Mock.Called(number)
+	return out.Get(0).(*types.Block), out.Error(1)
+}
+
+func (m *MockEthClient) ExpectBlockByNumber(number *big.Int, block *types.Block, err error) {
+	m.Mock.On("BlockByNumber", number).Once().Return(block, err)
+}
+
+func (m *MockEthClient) ExpectClose() {
+	m.Mock.On("Close").Once()
+}
+
+func (m *MockEthClient) MaybeClose() {
+	m.Mock.On("Close").Maybe()
+}
+
+func (m *MockEthClient) Close() {
+	m.Mock.Called()
+}

op-service/testutils/mock_rollup_client.go

+package testutils
+
+import (
+	"context"
+
+	"github.com/ethereum-optimism/optimism/op-node/rollup"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/stretchr/testify/mock"
+)
+
+type MockRollupClient struct {
+	mock.Mock
+}
+
+func (m *MockRollupClient) OutputAtBlock(ctx context.Context, blockNum uint64) (*eth.OutputResponse, error) {
+	out := m.Mock.Called(blockNum)
+	return out.Get(0).(*eth.OutputResponse), out.Error(1)
+}
+
+func (m *MockRollupClient) ExpectOutputAtBlock(blockNum uint64, response *eth.OutputResponse, err error) {
+	m.Mock.On("OutputAtBlock", blockNum).Once().Return(response, err)
+}
+
+func (m *MockRollupClient) SyncStatus(ctx context.Context) (*eth.SyncStatus, error) {
+	out := m.Mock.Called()
+	return out.Get(0).(*eth.SyncStatus), out.Error(1)
+}
+
+func (m *MockRollupClient) ExpectSyncStatus(status *eth.SyncStatus, err error) {
+	m.Mock.On("SyncStatus").Once().Return(status, err)
+}
+
+func (m *MockRollupClient) RollupConfig(ctx context.Context) (*rollup.Config, error) {
+	out := m.Mock.Called()
+	return out.Get(0).(*rollup.Config), out.Error(1)
+}
+
+func (m *MockRollupClient) ExpectRollupConfig(config *rollup.Config, err error) {
+	m.Mock.On("RollupConfig").Once().Return(config, err)
+}
+
+func (m *MockRollupClient) StartSequencer(ctx context.Context, unsafeHead common.Hash) error {
+	out := m.Mock.Called(unsafeHead)
+	return out.Error(0)
+}
+
+func (m *MockRollupClient) ExpectStartSequencer(unsafeHead common.Hash, err error) {
+	m.Mock.On("StartSequencer", unsafeHead).Once().Return(err)
+}
+
+func (m *MockRollupClient) SequencerActive(ctx context.Context) (bool, error) {
+	out := m.Mock.Called()
+	return out.Bool(0), out.Error(1)
+}
+
+func (m *MockRollupClient) ExpectSequencerActive(active bool, err error) {
+	m.Mock.On("SequencerActive").Once().Return(active, err)
+}
+
+func (m *MockRollupClient) ExpectClose() {
+	m.Mock.On("Close").Once()
+}
+
+func (m *MockRollupClient) MaybeClose() {
+	m.Mock.On("Close").Maybe()
+}
+
+func (m *MockRollupClient) Close() {
+	m.Mock.Called()
+}