Interop: SuperSystem for E2E Tests (#11850)

* op-e2e: interop test setup (work in progress)

* op-e2e: interop test setup

* organization and comment updates

* refactor creation code into WIP system2

* save secrets per L2

* Add SuperSystem Interface ; Add Users and Transactions

* Further Refactoring ; Fix Test

* Add Supervisor

* Add Supervisor Client

* Comment out Proposer

* Add AddL2RPC to Supervisor Client

* Fully link Supervisor and OP Node in E2E Test

* correct RPC call supervisor_checkBlock

* Make EOF acceptable for backend check

* final structure names

* Change unused functions to _ for linter

* fix import order

* Add Github Issue Numbers to TODOs

* tynes comments: add World Resource as configurable

---------
Co-authored-by: protolambda <proto@protolambda.com>

op-e2e/e2eutils/setuputils/utils.go

+package setuputils
+
+import (
+	"crypto/ecdsa"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils"
+	"github.com/ethereum-optimism/optimism/op-service/endpoint"
+	"github.com/ethereum-optimism/optimism/op-service/txmgr"
+)
+
+func hexPriv(in *ecdsa.PrivateKey) string {
+	b := e2eutils.EncodePrivKey(in)
+	return hexutil.Encode(b)
+}
+
+func NewTxMgrConfig(l1Addr endpoint.RPC, privKey *ecdsa.PrivateKey) txmgr.CLIConfig {
+	return txmgr.CLIConfig{
+		L1RPCURL:                  l1Addr.RPC(),
+		PrivateKey:                hexPriv(privKey),
+		NumConfirmations:          1,
+		SafeAbortNonceTooLowCount: 3,
+		FeeLimitMultiplier:        5,
+		ResubmissionTimeout:       3 * time.Second,
+		ReceiptQueryInterval:      50 * time.Millisecond,
+		NetworkTimeout:            2 * time.Second,
+		TxNotInMempoolTimeout:     2 * time.Minute,
+	}
+}

op-e2e/interop/interop_test.go

+package interop
+
+import (
+	"context"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-chain-ops/interopgen"
+	op_e2e "github.com/ethereum-optimism/optimism/op-e2e"
+	"github.com/stretchr/testify/require"
+)
+
+// TestInteropTrivial tests a simple interop scenario
+// Chains A and B exist, but no messages are sent between them
+// and in fact no event-logs are emitted by either chain at all.
+// A transaction is sent from Alice to Bob on Chain A.
+// The balance of Bob on Chain A is checked before and after the tx.
+// The balance of Bob on Chain B is checked after the tx.
+func TestInteropTrivial(t *testing.T) {
+	recipe := interopgen.InteropDevRecipe{
+		L1ChainID:        900100,
+		L2ChainIDs:       []uint64{900200, 900201},
+		GenesisTimestamp: uint64(time.Now().Unix() + 3), // start chain 3 seconds from now
+	}
+	worldResources := worldResourcePaths{
+		foundryArtifacts: "../../packages/contracts-bedrock/forge-artifacts",
+		sourceMap:        "../../packages/contracts-bedrock",
+	}
+
+	// create a super system from the recipe
+	// and get the L2 IDs for use in the test
+	s2 := NewSuperSystem(t, &recipe, worldResources)
+	ids := s2.L2IDs()
+
+	// chainA is the first L2 chain
+	chainA := ids[0]
+	// chainB is the second L2 chain
+	chainB := ids[1]
+
+	// create two users on all L2 chains
+	s2.AddUser("Alice")
+	s2.AddUser("Bob")
+
+	bobAddr := s2.Address(chainA, "Bob")
+
+	// check the balance of Bob
+	clientA := s2.L2GethClient(chainA)
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
+	bobBalance, err := clientA.BalanceAt(ctx, bobAddr, nil)
+	require.NoError(t, err)
+	expectedBalance, _ := big.NewInt(0).SetString("10000000000000000000000000", 10)
+	require.Equal(t, expectedBalance, bobBalance)
+
+	// send a tx from Alice to Bob
+	s2.SendL2Tx(
+		chainA,
+		"Alice",
+		func(l2Opts *op_e2e.TxOpts) {
+			l2Opts.ToAddr = &bobAddr
+			l2Opts.Value = big.NewInt(1000000)
+			l2Opts.GasFeeCap = big.NewInt(1_000_000_000)
+			l2Opts.GasTipCap = big.NewInt(1_000_000_000)
+		},
+	)
+
+	// check the balance of Bob after the tx
+	ctx, cancel = context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
+	bobBalance, err = clientA.BalanceAt(ctx, bobAddr, nil)
+	require.NoError(t, err)
+	expectedBalance, _ = big.NewInt(0).SetString("10000000000000000001000000", 10)
+	require.Equal(t, expectedBalance, bobBalance)
+
+	// check that the balance of Bob on ChainB hasn't changed
+	bobAddrB := s2.Address(chainB, "Bob")
+	clientB := s2.L2GethClient(chainB)
+	ctx, cancel = context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
+	bobBalance, err = clientB.BalanceAt(ctx, bobAddrB, nil)
+	require.NoError(t, err)
+	expectedBalance, _ = big.NewInt(0).SetString("10000000000000000000000000", 10)
+	require.Equal(t, expectedBalance, bobBalance)
+}

op-e2e/sequencer_failover_setup.go

@@ -18,6 +18,7 @@ import (
 	con "github.com/ethereum-optimism/optimism/op-conductor/conductor"
 	"github.com/ethereum-optimism/optimism/op-conductor/consensus"
 	conrpc "github.com/ethereum-optimism/optimism/op-conductor/rpc"
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/setuputils"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait"
 	rollupNode "github.com/ethereum-optimism/optimism/op-node/node"
 	"github.com/ethereum-optimism/optimism/op-node/rollup"
@@ -291,7 +292,7 @@ func setupBatcher(t *testing.T, sys *System, conductors map[string]*conductor) {
 		ApproxComprRatio:       0.4,
 		SubSafetyMargin:        4,
 		PollInterval:           1 * time.Second,
-		TxMgrConfig:            newTxMgrConfig(sys.EthInstances["l1"].UserRPC().RPC(), sys.Cfg.Secrets.Batcher),
+		TxMgrConfig:            setuputils.NewTxMgrConfig(sys.EthInstances["l1"].UserRPC(), sys.Cfg.Secrets.Batcher),
 		LogConfig: oplog.CLIConfig{
 			Level:  log.LevelDebug,
 			Format: oplog.FormatText,

op-e2e/setup.go

@@ -50,6 +50,7 @@ import (
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/geth"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/opnode"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/services"
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/setuputils"
 	"github.com/ethereum-optimism/optimism/op-node/chaincfg"
 	rollupNode "github.com/ethereum-optimism/optimism/op-node/node"
 	"github.com/ethereum-optimism/optimism/op-node/p2p"
@@ -68,7 +69,6 @@ import (
 	"github.com/ethereum-optimism/optimism/op-service/predeploys"
 	"github.com/ethereum-optimism/optimism/op-service/sources"
 	"github.com/ethereum-optimism/optimism/op-service/testlog"
-	"github.com/ethereum-optimism/optimism/op-service/txmgr"
 )
 
 const (
@@ -82,20 +82,6 @@ var (
 	genesisTime      = hexutil.Uint64(0)
 )
 
-func newTxMgrConfig(l1Addr string, privKey *ecdsa.PrivateKey) txmgr.CLIConfig {
-	return txmgr.CLIConfig{
-		L1RPCURL:                  l1Addr,
-		PrivateKey:                hexPriv(privKey),
-		NumConfirmations:          1,
-		SafeAbortNonceTooLowCount: 3,
-		FeeLimitMultiplier:        5,
-		ResubmissionTimeout:       3 * time.Second,
-		ReceiptQueryInterval:      50 * time.Millisecond,
-		NetworkTimeout:            2 * time.Second,
-		TxNotInMempoolTimeout:     2 * time.Minute,
-	}
-}
-
 func DefaultSystemConfig(t testing.TB) SystemConfig {
 	config.ExternalL2TestParms.SkipIfNecessary(t)
 
@@ -789,7 +775,7 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste
 			ProposalInterval:  6 * time.Second,
 			DisputeGameType:   254, // Fast game type
 			PollInterval:      500 * time.Millisecond,
-			TxMgrConfig:       newTxMgrConfig(sys.EthInstances[RoleL1].UserRPC().RPC(), cfg.Secrets.Proposer),
+			TxMgrConfig:       setuputils.NewTxMgrConfig(sys.EthInstances[RoleL1].UserRPC(), cfg.Secrets.Proposer),
 			AllowNonFinalized: cfg.NonFinalizedProposals,
 			LogConfig: oplog.CLIConfig{
 				Level:  log.LvlInfo,
@@ -802,7 +788,7 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste
 			RollupRpc:         sys.RollupNodes[RoleSeq].UserRPC().RPC(),
 			L2OOAddress:       config.L1Deployments.L2OutputOracleProxy.Hex(),
 			PollInterval:      500 * time.Millisecond,
-			TxMgrConfig:       newTxMgrConfig(sys.EthInstances[RoleL1].UserRPC().RPC(), cfg.Secrets.Proposer),
+			TxMgrConfig:       setuputils.NewTxMgrConfig(sys.EthInstances[RoleL1].UserRPC(), cfg.Secrets.Proposer),
 			AllowNonFinalized: cfg.NonFinalizedProposals,
 			LogConfig: oplog.CLIConfig{
 				Level:  log.LvlInfo,
@@ -865,7 +851,7 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste
 		ApproxComprRatio:         0.4,
 		SubSafetyMargin:          4,
 		PollInterval:             50 * time.Millisecond,
-		TxMgrConfig:              newTxMgrConfig(sys.EthInstances[RoleL1].UserRPC().RPC(), cfg.Secrets.Batcher),
+		TxMgrConfig:              setuputils.NewTxMgrConfig(sys.EthInstances[RoleL1].UserRPC(), cfg.Secrets.Batcher),
 		LogConfig: oplog.CLIConfig{
 			Level:  log.LevelInfo,
 			Format: oplog.FormatText,
@@ -984,11 +970,6 @@ func (cfg SystemConfig) L2ChainIDBig() *big.Int {
 	return new(big.Int).SetUint64(cfg.DeployConfig.L2ChainID)
 }
 
-func hexPriv(in *ecdsa.PrivateKey) string {
-	b := e2eutils.EncodePrivKey(in)
-	return hexutil.Encode(b)
-}
-
 func (sys *System) RollupClient(name string) *sources.RollupClient {
 	rollupClient, ok := sys.rollupClients[name]
 	if ok {

op-e2e/tx_helper.go

@@ -84,11 +84,11 @@ func defaultDepositTxOpts(opts *bind.TransactOpts) *DepositTxOpts {
 // The supplied privKey is used to specify the account to send from and the transaction is sent to the supplied l2Client
 // Transaction options and expected status can be configured in the applyTxOpts function by modifying the supplied TxOpts
 // Will verify that the transaction is included with the expected status on l2Client and any clients added to TxOpts.VerifyClients
-func SendL2Tx(t *testing.T, cfg SystemConfig, l2Client *ethclient.Client, privKey *ecdsa.PrivateKey, applyTxOpts TxOptsFn) *types.Receipt {
+func SendL2TxWithID(t *testing.T, chainID *big.Int, l2Client *ethclient.Client, privKey *ecdsa.PrivateKey, applyTxOpts TxOptsFn) *types.Receipt {
 	opts := defaultTxOpts()
 	applyTxOpts(opts)
-	tx := types.MustSignNewTx(privKey, types.LatestSignerForChainID(cfg.L2ChainIDBig()), &types.DynamicFeeTx{
-		ChainID:   cfg.L2ChainIDBig(),
+	tx := types.MustSignNewTx(privKey, types.LatestSignerForChainID(chainID), &types.DynamicFeeTx{
+		ChainID:   chainID,
 		Nonce:     opts.Nonce, // Already have deposit
 		To:        opts.ToAddr,
 		Value:     opts.Value,
@@ -115,6 +115,10 @@ func SendL2Tx(t *testing.T, cfg SystemConfig, l2Client *ethclient.Client, privKe
 	return receipt
 }
 
+func SendL2Tx(t *testing.T, cfg SystemConfig, l2Client *ethclient.Client, privKey *ecdsa.PrivateKey, applyTxOpts TxOptsFn) *types.Receipt {
+	return SendL2TxWithID(t, cfg.L2ChainIDBig(), l2Client, privKey, applyTxOpts)
+}
+
 type TxOptsFn func(opts *TxOpts)
 
 type TxOpts struct {

op-service/sources/supervisor_client.go

@@ -21,10 +21,29 @@ func NewSupervisorClient(client client.RPC) *SupervisorClient {
 	}
 }
 
+func (cl *SupervisorClient) AddL2RPC(
+	ctx context.Context,
+	rpc string,
+) error {
+	var result error
+	err := cl.client.CallContext(
+		ctx,
+		&result,
+		"admin_addL2RPC",
+		rpc)
+	if err != nil {
+		return fmt.Errorf("failed to Add L2 to Supervisor (rpc: %s): %w", rpc, err)
+	}
+	return result
+}
+
 func (cl *SupervisorClient) CheckBlock(ctx context.Context,
 	chainID types.ChainID, blockHash common.Hash, blockNumber uint64) (types.SafetyLevel, error) {
 	var result types.SafetyLevel
-	err := cl.client.CallContext(ctx, &result, "interop_checkBlock",
+	err := cl.client.CallContext(
+		ctx,
+		&result,
+		"supervisor_checkBlock",
 		(*hexutil.U256)(&chainID), blockHash, hexutil.Uint64(blockNumber))
 	if err != nil {
 		return types.Unsafe, fmt.Errorf("failed to check Block %s:%d (chain %s): %w", blockHash, blockNumber, chainID, err)

op-supervisor/supervisor/backend/backend.go

@@ -25,7 +25,6 @@ import (
 )
 
 type SupervisorBackend struct {
-	ctx     context.Context
 	started atomic.Bool
 	logger  log.Logger
 	m       Metrics
@@ -82,7 +81,7 @@ func NewSupervisorBackend(ctx context.Context, logger log.Logger, m Metrics, cfg
 // it does not expect to be called after the backend has been started
 func (su *SupervisorBackend) addFromRPC(ctx context.Context, logger log.Logger, rpc string) error {
 	// create the rpc client, which yields the chain id
-	rpcClient, chainID, err := createRpcClient(su.ctx, logger, rpc)
+	rpcClient, chainID, err := createRpcClient(ctx, logger, rpc)
 	if err != nil {
 		return err
 	}
@@ -173,9 +172,11 @@ func (su *SupervisorBackend) AddL2RPC(ctx context.Context, rpc string) error {
 	if err := su.Stop(ctx); err != nil {
 		return fmt.Errorf("failed to stop backend: %w", err)
 	}
+	su.logger.Info("temporarily stopped the backend to add a new L2 RPC", "rpc", rpc)
 	if err := su.addFromRPC(ctx, su.logger, rpc); err != nil {
 		return fmt.Errorf("failed to add chain monitor: %w", err)
 	}
+	su.logger.Info("added the new L2 RPC, starting supervisor again", "rpc", rpc)
 	return su.Start(ctx)
 }
 
@@ -231,7 +232,10 @@ func (su *SupervisorBackend) CheckBlock(chainID *hexutil.U256, blockHash common.
 	safest := types.CrossUnsafe
 	// find the last log index in the block
 	i, err := su.db.LastLogInBlock(types.ChainID(*chainID), uint64(blockNumber))
-	if err != nil {
+	// TODO(#11836) checking for EOF as a non-error case is a bit of a code smell
+	// and could potentially be incorrect if the given block number is intentionally far in the future
+	if err != nil && !errors.Is(err, io.EOF) {
+		su.logger.Error("failed to scan block", "err", err)
 		return types.Invalid, fmt.Errorf("failed to scan block: %w", err)
 	}
 	// at this point we have the extent of the block, and we can check if it is safe by various criteria

op-supervisor/supervisor/backend/source/chain_processor.go

@@ -49,10 +49,13 @@ func NewChainProcessor(log log.Logger, client BlockByNumberSource, chain types.C
 }
 
 func (s *ChainProcessor) OnNewHead(ctx context.Context, head eth.L1BlockRef) {
+	s.log.Debug("Processing chain", "chain", s.chain, "head", head)
 	if head.Number <= s.lastBlock.Number {
+		s.log.Info("head is not newer than last processed block", "head", head, "lastBlock", s.lastBlock)
 		return
 	}
 	for s.lastBlock.Number+1 < head.Number {
+		s.log.Debug("Filling in skipped block", "chain", s.chain, "lastBlock", s.lastBlock, "head", head)
 		blockNum := s.lastBlock.Number + 1
 		nextBlock, err := s.client.L1BlockRefByNumber(ctx, blockNum)
 		if err != nil {

op-supervisor/supervisor/service.go

@@ -164,6 +164,7 @@ func (su *SupervisorService) Start(ctx context.Context) error {
 	}
 
 	su.metrics.RecordUp()
+	su.log.Info("JSON-RPC Server started", "endpoint", su.rpcServer.Endpoint())
 	return nil
 }
 
@@ -171,7 +172,7 @@ func (su *SupervisorService) Stop(ctx context.Context) error {
 	if !su.closing.CompareAndSwap(false, true) {
 		return nil // already closing
 	}
-
+	su.log.Info("Stopping JSON-RPC server")
 	var result error
 	if su.rpcServer != nil {
 		if err := su.rpcServer.Stop(); err != nil {
@@ -193,9 +194,16 @@ func (su *SupervisorService) Stop(ctx context.Context) error {
 			result = errors.Join(result, fmt.Errorf("failed to stop metrics server: %w", err))
 		}
 	}
+	su.log.Info("JSON-RPC server stopped")
 	return result
 }
 
 func (su *SupervisorService) Stopped() bool {
 	return su.closing.Load()
 }
+
+func (su *SupervisorService) RPC() string {
+	// the RPC endpoint is assumed to be HTTP
+	// TODO(#11032): make this flexible for ws if the server supports it
+	return "http://" + su.rpcServer.Endpoint()
+}