Merge pull request #5155 from ethereum-optimism/action-test-full-chain

op-e2e: test high-demand large L2 blocks case

op-e2e/actions/l1_miner.go

@@ -10,6 +10,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/stretchr/testify/require"
 )
 
 // L1Miner wraps a L1Replica with instrumented block building ability.
@@ -100,26 +101,33 @@ func (s *L1Miner) ActL1IncludeTx(from common.Address) Action {
 			t.Fatalf("no pending txs from %s, and have %d unprocessable queued txs from this account", from, len(q))
 		}
 		tx := txs[i]
-		if tx.Gas() > s.l1BuildingHeader.GasLimit {
-			t.Fatalf("tx consumes %d gas, more than available in L1 block %d", tx.Gas(), s.l1BuildingHeader.GasLimit)
-		}
-		if tx.Gas() > uint64(*s.l1GasPool) {
-			t.InvalidAction("action takes too much gas: %d, only have %d", tx.Gas(), uint64(*s.l1GasPool))
-			return
-		}
+		s.IncludeTx(t, tx)
 		s.pendingIndices[from] = i + 1 // won't retry the tx
-		s.l1BuildingState.SetTxContext(tx.Hash(), len(s.l1Transactions))
-		receipt, err := core.ApplyTransaction(s.l1Cfg.Config, s.l1Chain, &s.l1BuildingHeader.Coinbase,
-			s.l1GasPool, s.l1BuildingState, s.l1BuildingHeader, tx, &s.l1BuildingHeader.GasUsed, *s.l1Chain.GetVMConfig())
-		if err != nil {
-			s.l1TxFailed = append(s.l1TxFailed, tx)
-			t.Fatalf("failed to apply transaction to L1 block (tx %d): %w", len(s.l1Transactions), err)
-		}
-		s.l1Receipts = append(s.l1Receipts, receipt)
-		s.l1Transactions = append(s.l1Transactions, tx)
 	}
 }
 
+func (s *L1Miner) IncludeTx(t Testing, tx *types.Transaction) {
+	from, err := s.l1Signer.Sender(tx)
+	require.NoError(t, err)
+	s.log.Info("including tx", "nonce", tx.Nonce(), "from", from)
+	if tx.Gas() > s.l1BuildingHeader.GasLimit {
+		t.Fatalf("tx consumes %d gas, more than available in L1 block %d", tx.Gas(), s.l1BuildingHeader.GasLimit)
+	}
+	if tx.Gas() > uint64(*s.l1GasPool) {
+		t.InvalidAction("action takes too much gas: %d, only have %d", tx.Gas(), uint64(*s.l1GasPool))
+		return
+	}
+	s.l1BuildingState.SetTxContext(tx.Hash(), len(s.l1Transactions))
+	receipt, err := core.ApplyTransaction(s.l1Cfg.Config, s.l1Chain, &s.l1BuildingHeader.Coinbase,
+		s.l1GasPool, s.l1BuildingState, s.l1BuildingHeader, tx, &s.l1BuildingHeader.GasUsed, *s.l1Chain.GetVMConfig())
+	if err != nil {
+		s.l1TxFailed = append(s.l1TxFailed, tx)
+		t.Fatalf("failed to apply transaction to L1 block (tx %d): %v", len(s.l1Transactions), err)
+	}
+	s.l1Receipts = append(s.l1Receipts, receipt)
+	s.l1Transactions = append(s.l1Transactions, tx)
+}
+
 func (s *L1Miner) ActL1SetFeeRecipient(coinbase common.Address) {
 	s.prefCoinbase = coinbase
 	if s.l1Building {

op-e2e/actions/l2_batcher.go

@@ -91,6 +91,10 @@ func (s *L2Batcher) SubmittingData() bool {
 // ActL2BatchBuffer adds the next L2 block to the batch buffer.
 // If the buffer is being submitted, the buffer is wiped.
 func (s *L2Batcher) ActL2BatchBuffer(t Testing) {
+	require.NoError(t, s.Buffer(t), "failed to add block to channel")
+}
+
+func (s *L2Batcher) Buffer(t Testing) error {
 	if s.l2Submitting { // break ongoing submitting work if necessary
 		s.l2ChannelOut = nil
 		s.l2Submitting = false
@@ -120,7 +124,7 @@ func (s *L2Batcher) ActL2BatchBuffer(t Testing) {
 			s.l2ChannelOut = nil
 		} else {
 			s.log.Info("nothing left to submit")
-			return
+			return nil
 		}
 	}
 	// Create channel if we don't have one yet
@@ -143,9 +147,10 @@ func (s *L2Batcher) ActL2BatchBuffer(t Testing) {
 		s.l2ChannelOut = nil
 	}
 	if _, err := s.l2ChannelOut.AddBlock(block); err != nil { // should always succeed
-		t.Fatalf("failed to add block to channel: %v", err)
+		return err
 	}
 	s.l2BufferedBlock = eth.ToBlockID(block)
+	return nil
 }
 
 func (s *L2Batcher) ActL2ChannelClose(t Testing) {
@@ -158,7 +163,7 @@ func (s *L2Batcher) ActL2ChannelClose(t Testing) {
 }
 
 // ActL2BatchSubmit constructs a batch tx from previous buffered L2 blocks, and submits it to L1
-func (s *L2Batcher) ActL2BatchSubmit(t Testing) {
+func (s *L2Batcher) ActL2BatchSubmit(t Testing, txOpts ...func(tx *types.DynamicFeeTx)) {
 	// Don't run this action if there's no data to submit
 	if s.l2ChannelOut == nil {
 		t.InvalidAction("need to buffer data first, cannot batch submit with empty buffer")
@@ -192,6 +197,9 @@ func (s *L2Batcher) ActL2BatchSubmit(t Testing) {
 		GasFeeCap: gasFeeCap,
 		Data:      data.Bytes(),
 	}
+	for _, opt := range txOpts {
+		opt(rawTx)
+	}
 	gas, err := core.IntrinsicGas(rawTx.Data, nil, false, true, true, false)
 	require.NoError(t, err, "need to compute intrinsic gas")
 	rawTx.Gas = gas

op-e2e/actions/l2_batcher_test.go

 package actions
 
 import (
+	"crypto/rand"
+	"errors"
 	"math/big"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
@@ -12,6 +15,7 @@ import (
 
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils"
 	"github.com/ethereum-optimism/optimism/op-node/eth"
+	"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
 	"github.com/ethereum-optimism/optimism/op-node/testlog"
 )
 
@@ -378,3 +382,131 @@ func TestExtendedTimeWithoutL1Batches(gt *testing.T) {
 	sequencer.ActL2PipelineFull(t)
 	require.Equal(t, sequencer.L2Unsafe(), sequencer.L2Safe(), "same for sequencer")
 }
+
+// TestBigL2Txs tests a high-throughput case with constrained batcher:
+//   - Fill 100 L2 blocks to near max-capacity, with txs of 120 KB each
+//   - Buffer the L2 blocks into channels together as much as possible, submit data-txs only when necessary
+//     (just before crossing the max RLP channel size)
+//   - Limit the data-tx size to 40 KB, to force data to be split across multiple datat-txs
+//   - Defer all data-tx inclusion till the end
+//   - Fill L1 blocks with data-txs until we have processed them all
+//   - Run the verifier, and check if it derives the same L2 chain as was created by the sequencer.
+//
+// The goal of this test is to quickly run through an otherwise very slow process of submitting and including lots of data.
+// This does not test the batcher code, but is really focused at testing the batcher utils
+// and channel-decoding verifier code in the derive package.
+func TestBigL2Txs(gt *testing.T) {
+	t := NewDefaultTesting(gt)
+	p := &e2eutils.TestParams{
+		MaxSequencerDrift:   100,
+		SequencerWindowSize: 1000,
+		ChannelTimeout:      200, // give enough space to buffer large amounts of data before submitting it
+	}
+	dp := e2eutils.MakeDeployParams(t, p)
+	sd := e2eutils.Setup(t, dp, defaultAlloc)
+	log := testlog.Logger(t, log.LvlInfo)
+	miner, engine, sequencer := setupSequencerTest(t, sd, log)
+
+	_, verifier := setupVerifier(t, sd, log, miner.L1Client(t, sd.RollupCfg))
+
+	batcher := NewL2Batcher(log, sd.RollupCfg, &BatcherCfg{
+		MinL1TxSize: 0,
+		MaxL1TxSize: 40_000, // try a small batch size, to force the data to be split between more frames
+		BatcherKey:  dp.Secrets.Batcher,
+	}, sequencer.RollupClient(), miner.EthClient(), engine.EthClient())
+
+	sequencer.ActL2PipelineFull(t)
+
+	verifier.ActL2PipelineFull(t)
+	cl := engine.EthClient()
+
+	batcherNonce := uint64(0) // manually track batcher nonce. the "pending nonce" value in tx-pool is incorrect after we fill the pending-block gas limit and keep adding txs to the pool.
+	batcherTxOpts := func(tx *types.DynamicFeeTx) {
+		tx.Nonce = batcherNonce
+		batcherNonce++
+		tx.GasFeeCap = e2eutils.Ether(1) // be very generous with basefee, since we're spamming L1
+	}
+
+	// build many L2 blocks filled to the brim with large txs of random data
+	for i := 0; i < 100; i++ {
+		aliceNonce, err := cl.PendingNonceAt(t.Ctx(), dp.Addresses.Alice)
+		status := sequencer.SyncStatus()
+		// build empty L1 blocks as necessary, so the L2 sequencer can continue to include txs while not drifting too far out
+		if status.UnsafeL2.Time >= status.HeadL1.Time+12 {
+			miner.ActEmptyBlock(t)
+		}
+		sequencer.ActL1HeadSignal(t)
+		sequencer.ActL2StartBlock(t)
+		baseFee := engine.l2Chain.CurrentBlock().BaseFee() // this will go quite high, since so many consecutive blocks are filled at capacity.
+		// fill the block with large L2 txs from alice
+		for n := aliceNonce; ; n++ {
+			require.NoError(t, err)
+			signer := types.LatestSigner(sd.L2Cfg.Config)
+			data := make([]byte, 120_000) // very large L2 txs, as large as the tx-pool will accept
+			_, err := rand.Read(data[:])  // fill with random bytes, to make compression ineffective
+			require.NoError(t, err)
+			gas, err := core.IntrinsicGas(data, nil, false, true, true, false)
+			require.NoError(t, err)
+			if gas > engine.l2GasPool.Gas() {
+				break
+			}
+			tx := types.MustSignNewTx(dp.Secrets.Alice, signer, &types.DynamicFeeTx{
+				ChainID:   sd.L2Cfg.Config.ChainID,
+				Nonce:     n,
+				GasTipCap: big.NewInt(2 * params.GWei),
+				GasFeeCap: new(big.Int).Add(new(big.Int).Mul(baseFee, big.NewInt(2)), big.NewInt(2*params.GWei)),
+				Gas:       gas,
+				To:        &dp.Addresses.Bob,
+				Value:     big.NewInt(0),
+				Data:      data,
+			})
+			require.NoError(gt, cl.SendTransaction(t.Ctx(), tx))
+			engine.ActL2IncludeTx(dp.Addresses.Alice)(t)
+		}
+		sequencer.ActL2EndBlock(t)
+		for batcher.l2BufferedBlock.Number < sequencer.SyncStatus().UnsafeL2.Number {
+			// if we run out of space, close the channel and submit all the txs
+			if err := batcher.Buffer(t); errors.Is(err, derive.ErrTooManyRLPBytes) {
+				log.Info("flushing filled channel to batch txs", "id", batcher.l2ChannelOut.ID())
+				batcher.ActL2ChannelClose(t)
+				for batcher.l2ChannelOut != nil {
+					batcher.ActL2BatchSubmit(t, batcherTxOpts)
+				}
+			}
+		}
+	}
+
+	// if anything is left in the channel, submit it
+	if batcher.l2ChannelOut != nil {
+		log.Info("flushing trailing channel to batch txs", "id", batcher.l2ChannelOut.ID())
+		batcher.ActL2ChannelClose(t)
+		for batcher.l2ChannelOut != nil {
+			batcher.ActL2BatchSubmit(t, batcherTxOpts)
+		}
+	}
+
+	// build L1 blocks until we're out of txs
+	txs, _ := miner.eth.TxPool().ContentFrom(dp.Addresses.Batcher)
+	for {
+		if len(txs) == 0 {
+			break
+		}
+		miner.ActL1StartBlock(12)(t)
+		for range txs {
+			if len(txs) == 0 {
+				break
+			}
+			tx := txs[0]
+			if miner.l1GasPool.Gas() < tx.Gas() { // fill the L1 block with batcher txs until we run out of gas
+				break
+			}
+			log.Info("including batcher tx", "nonce", tx)
+			miner.IncludeTx(t, tx)
+			txs = txs[1:]
+		}
+		miner.ActL1EndBlock(t)
+	}
+	verifier.ActL1HeadSignal(t)
+	verifier.ActL2PipelineFull(t)
+	require.Equal(t, sequencer.SyncStatus().UnsafeL2, verifier.SyncStatus().SafeL2, "verifier synced sequencer data even though of huge tx in block")
+}