go-batch-submitter: enable typed batch support

Allow the go batch submitter to submit typed batches.
The env var `BATCH_SUBMITTER_SEQUENCER_BATCH_TYPE=zlib`
can be set to enable the submission of zlib compressed
batches. If that env var is not set, then legacy batches
will be submitted instead.

.changeset/ten-rats-build.md

+---
+'@eth-optimism/batch-submitter-service': patch
+---
+
+Enable the usage of typed batches and type 0 zlib compressed batches

go/batch-submitter/batch_submitter.go

@@ -125,6 +125,7 @@ func Main(gitVersion string) func(ctx *cli.Context) error {
 				CTCAddr:     ctcAddress,
 				ChainID:     chainID,
 				PrivKey:     sequencerPrivKey,
+				BatchType:   sequencer.BatchTypeFromString(cfg.SequencerBatchType),
 			})
 			if err != nil {
 				return err

go/batch-submitter/config.go

@@ -33,6 +33,11 @@ var (
 	ErrSameSequencerAndProposerPrivKey = errors.New("sequencer-priv-key and " +
 		"proposer-priv-key must be distinct")
 
+	// ErrInvalidBatchType  signals that an unsupported batch type is being
+	// configured. The default is "legacy" and the options are "legacy" or
+	// "zlib"
+	ErrInvalidBatchType = errors.New("invalid batch type")
+
 	// ErrSentryDSNNotSet signals that not Data Source Name was provided
 	// with which to configure Sentry logging.
 	ErrSentryDSNNotSet = errors.New("sentry-dsn must be set if use-sentry " +
@@ -164,6 +169,9 @@ type Config struct {
 	// the proposer transactions.
 	ProposerHDPath string
 
+	// SequencerBatchType represents the type of batch the sequencer submits.
+	SequencerBatchType string
+
 	// MetricsServerEnable if true, will create a metrics client and log to
 	// Prometheus.
 	MetricsServerEnable bool
@@ -212,6 +220,7 @@ func NewConfig(ctx *cli.Context) (Config, error) {
 		Mnemonic:            ctx.GlobalString(flags.MnemonicFlag.Name),
 		SequencerHDPath:     ctx.GlobalString(flags.SequencerHDPathFlag.Name),
 		ProposerHDPath:      ctx.GlobalString(flags.ProposerHDPathFlag.Name),
+		SequencerBatchType:  ctx.GlobalString(flags.SequencerBatchType.Name),
 		MetricsServerEnable: ctx.GlobalBool(flags.MetricsServerEnableFlag.Name),
 		MetricsHostname:     ctx.GlobalString(flags.MetricsHostnameFlag.Name),
 		MetricsPort:         ctx.GlobalUint64(flags.MetricsPortFlag.Name),
@@ -265,6 +274,12 @@ func ValidateConfig(cfg *Config) error {
 		return ErrSameSequencerAndProposerPrivKey
 	}
 
+	usingTypedBatches := cfg.SequencerBatchType != ""
+	validBatchType := cfg.SequencerBatchType == "legacy" || cfg.SequencerBatchType == "zlib"
+	if usingTypedBatches && !validBatchType {
+		return ErrInvalidBatchType
+	}
+
 	// Ensure the Sentry Data Source Name is set when using Sentry.
 	if cfg.SentryEnable && cfg.SentryDsn == "" {
 		return ErrSentryDSNNotSet

go/batch-submitter/drivers/sequencer/batch.go

@@ -78,6 +78,7 @@ func GenSequencerBatchParams(
 	shouldStartAtElement uint64,
 	blockOffset uint64,
 	batch []BatchElement,
+	batchType BatchType,
 ) (*AppendSequencerBatchParams, error) {
 
 	var (
@@ -188,5 +189,6 @@ func GenSequencerBatchParams(
 		TotalElementsToAppend: uint64(len(batch)),
 		Contexts:              contexts,
 		Txs:                   txs,
+		Type:                  batchType,
 	}, nil
 }

go/batch-submitter/drivers/sequencer/driver.go

@@ -36,6 +36,7 @@ type Config struct {
 	CTCAddr     common.Address
 	ChainID     *big.Int
 	PrivKey     *ecdsa.PrivateKey
+	BatchType   BatchType
 }
 
 type Driver struct {
@@ -160,7 +161,7 @@ func (d *Driver) CraftBatchTx(
 	name := d.cfg.Name
 
 	log.Info(name+" crafting batch tx", "start", start, "end", end,
-		"nonce", nonce)
+		"nonce", nonce, "type", d.cfg.BatchType.String())
 
 	var (
 		batchElements []BatchElement
@@ -195,7 +196,7 @@ func (d *Driver) CraftBatchTx(
 	var pruneCount int
 	for {
 		batchParams, err := GenSequencerBatchParams(
-			shouldStartAt, d.cfg.BlockOffset, batchElements,
+			shouldStartAt, d.cfg.BlockOffset, batchElements, d.cfg.BatchType,
 		)
 		if err != nil {
 			return nil, err

go/batch-submitter/drivers/sequencer/encoding.go

 package sequencer
 
 import (
+	"bufio"
 	"bytes"
+	"compress/zlib"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"io"
 	"math"
@@ -17,7 +20,13 @@ const (
 	TxLenSize = 3
 )
 
-var byteOrder = binary.BigEndian
+var (
+	// byteOrder represents the endiannes used for batch serialization
+	byteOrder = binary.BigEndian
+	// ErrMalformedBatch represents a batch that is not well formed
+	// according to the protocol specification
+	ErrMalformedBatch = errors.New("malformed batch")
+)
 
 // BatchContext denotes a range of transactions that belong the same batch. It
 // is used to compress shared fields that would otherwise be repeated for each
@@ -44,11 +53,14 @@ type BatchContext struct {
 //  - num_subsequent_queue_txs: 3 bytes
 //  - timestamp:                5 bytes
 //  - block_number:             5 bytes
+//
+// Note that writing to a bytes.Buffer cannot
+// error, so errors are ignored here
 func (c *BatchContext) Write(w *bytes.Buffer) {
-	writeUint64(w, c.NumSequencedTxs, 3)
-	writeUint64(w, c.NumSubsequentQueueTxs, 3)
-	writeUint64(w, c.Timestamp, 5)
-	writeUint64(w, c.BlockNumber, 5)
+	_ = writeUint64(w, c.NumSequencedTxs, 3)
+	_ = writeUint64(w, c.NumSubsequentQueueTxs, 3)
+	_ = writeUint64(w, c.Timestamp, 5)
+	_ = writeUint64(w, c.BlockNumber, 5)
 }
 
 // Read decodes the BatchContext from the passed reader. If fewer than 16-bytes
@@ -71,6 +83,45 @@ func (c *BatchContext) Read(r io.Reader) error {
 	return readUint64(r, &c.BlockNumber, 5)
 }
 
+// BatchType represents the type of batch being
+// submitted. When the first context in the batch
+// has a timestamp of 0, the blocknumber is interpreted
+// as an enum that represets the type
+type BatchType int8
+
+// Implements the Stringer interface for BatchType
+func (b BatchType) String() string {
+	switch b {
+	case BatchTypeLegacy:
+		return "LEGACY"
+	case BatchTypeZlib:
+		return "ZLIB"
+	default:
+		return ""
+	}
+}
+
+// BatchTypeFromString returns the BatchType
+// enum based on a human readable string
+func BatchTypeFromString(s string) BatchType {
+	switch s {
+	case "zlib", "ZLIB":
+		return BatchTypeZlib
+	case "legacy", "LEGACY":
+		return BatchTypeLegacy
+	default:
+		return BatchTypeLegacy
+	}
+}
+
+const (
+	// BatchTypeLegacy represets the legacy batch type
+	BatchTypeLegacy BatchType = -1
+	// BatchTypeZlib represents a batch type where the
+	// transaction data is compressed using zlib
+	BatchTypeZlib BatchType = 0
+)
+
 // AppendSequencerBatchParams holds the raw data required to submit a batch of
 // L2 txs to L1 CTC contract. Rather than encoding the objects using the
 // standard ABI encoding, a custom encoding is and provided in the call data to
@@ -95,6 +146,9 @@ type AppendSequencerBatchParams struct {
 	// Txs contains all sequencer txs that will be recorded in the L1 CTC
 	// contract.
 	Txs []*CachedTx
+
+	// The type of the batch
+	Type BatchType
 }
 
 // Write encodes the AppendSequencerBatchParams using the following format:
@@ -105,20 +159,73 @@ type AppendSequencerBatchParams struct {
 //  - [num txs ommitted]
 //    - tx_len:                       3 bytes
 //    - tx_bytes:                     tx_len bytes
+//
+// Typed batches include a dummy context as the first context
+// where the timestamp is 0. The blocknumber is interpreted
+// as an enum that defines the type. It is impossible to have
+// a timestamp of 0 in practice, so this safely can indicate
+// that the batch is typed.
+// Type 0 batches have a dummy context where the blocknumber is
+// set to 0. The transaction data is compressed with zlib before
+// submitting the transaction to the chain. The fields should_start_at_element,
+// total_elements_to_append, num_contexts and the contexts themselves
+// are not altered.
+//
+// Note that writing to a bytes.Buffer cannot
+// error, so errors are ignored here
 func (p *AppendSequencerBatchParams) Write(w *bytes.Buffer) error {
-	writeUint64(w, p.ShouldStartAtElement, 5)
-	writeUint64(w, p.TotalElementsToAppend, 3)
+	_ = writeUint64(w, p.ShouldStartAtElement, 5)
+	_ = writeUint64(w, p.TotalElementsToAppend, 3)
+
+	// There must be contexts if there are transactions
+	if len(p.Contexts) == 0 && len(p.Txs) != 0 {
+		return ErrMalformedBatch
+	}
+
+	// There must be transactions if there are contexts
+	if len(p.Txs) == 0 && len(p.Contexts) != 0 {
+		return ErrMalformedBatch
+	}
+
+	// copy the contexts as to not malleate the struct
+	// when it is a typed batch
+	contexts := make([]BatchContext, 0, len(p.Contexts)+1)
+	if p.Type == BatchTypeZlib {
+		// All zero values for the single batch context
+		// is desired here as blocknumber 0 means it is a zlib batch
+		contexts = append(contexts, BatchContext{})
+	}
+	contexts = append(contexts, p.Contexts...)
 
 	// Write number of contexts followed by each fixed-size BatchContext.
-	writeUint64(w, uint64(len(p.Contexts)), 3)
-	for _, context := range p.Contexts {
+	_ = writeUint64(w, uint64(len(contexts)), 3)
+	for _, context := range contexts {
 		context.Write(w)
 	}
 
-	// Write each length-prefixed tx.
-	for _, tx := range p.Txs {
-		writeUint64(w, uint64(tx.Size()), TxLenSize)
-		_, _ = w.Write(tx.RawTx()) // can't fail for bytes.Buffer
+	switch p.Type {
+	case BatchTypeLegacy:
+		// Write each length-prefixed tx.
+		for _, tx := range p.Txs {
+			_ = writeUint64(w, uint64(tx.Size()), TxLenSize)
+			_, _ = w.Write(tx.RawTx()) // can't fail for bytes.Buffer
+		}
+	case BatchTypeZlib:
+		zw := zlib.NewWriter(w)
+		for _, tx := range p.Txs {
+			if err := writeUint64(zw, uint64(tx.Size()), TxLenSize); err != nil {
+				return err
+			}
+			if _, err := zw.Write(tx.RawTx()); err != nil {
+				return err
+			}
+		}
+		if err := zw.Close(); err != nil {
+			return err
+		}
+
+	default:
+		return fmt.Errorf("Unknown batch type: %s", p.Type)
 	}
 
 	return nil
@@ -159,6 +266,8 @@ func (p *AppendSequencerBatchParams) Read(r io.Reader) error {
 		return err
 	}
 
+	// Ensure that contexts is never nil
+	p.Contexts = make([]BatchContext, 0)
 	for i := uint64(0); i < numContexts; i++ {
 		var batchContext BatchContext
 		if err := batchContext.Read(r); err != nil {
@@ -168,14 +277,44 @@ func (p *AppendSequencerBatchParams) Read(r io.Reader) error {
 		p.Contexts = append(p.Contexts, batchContext)
 	}
 
+	// Assume that it is a legacy batch at first
+	p.Type = BatchTypeLegacy
+
+	// Handle backwards compatible batch types
+	if len(p.Contexts) > 0 && p.Contexts[0].Timestamp == 0 {
+		switch p.Contexts[0].BlockNumber {
+		case 0:
+			// zlib compressed transaction data
+			p.Type = BatchTypeZlib
+			// remove the first dummy context
+			p.Contexts = p.Contexts[1:]
+			numContexts--
+
+			zr, err := zlib.NewReader(r)
+			if err != nil {
+				return err
+			}
+			defer zr.Close()
+
+			r = bufio.NewReader(zr)
+		}
+	}
+
 	// Deserialize any transactions. Since the number of txs is ommitted
 	// from the encoding, loop until the stream is consumed.
 	for {
 		var txLen uint64
 		err := readUint64(r, &txLen, TxLenSize)
 		// Getting an EOF when reading the txLen expected for a cleanly
-		// encoded object. Silece the error and return success.
+		// encoded object. Silence the error and return success if
+		// the batch is well formed.
 		if err == io.EOF {
+			if len(p.Contexts) == 0 && len(p.Txs) != 0 {
+				return ErrMalformedBatch
+			}
+			if len(p.Txs) == 0 && len(p.Contexts) != 0 {
+				return ErrMalformedBatch
+			}
 			return nil
 		} else if err != nil {
 			return err
@@ -188,10 +327,11 @@ func (p *AppendSequencerBatchParams) Read(r io.Reader) error {
 
 		p.Txs = append(p.Txs, NewCachedTx(tx))
 	}
+
 }
 
 // writeUint64 writes a the bottom `n` bytes of `val` to `w`.
-func writeUint64(w *bytes.Buffer, val uint64, n uint) {
+func writeUint64(w io.Writer, val uint64, n uint) error {
 	if n < 1 || n > 8 {
 		panic(fmt.Sprintf("invalid number of bytes %d must be 1-8", n))
 	}
@@ -204,7 +344,8 @@ func writeUint64(w *bytes.Buffer, val uint64, n uint) {
 
 	var buf [8]byte
 	byteOrder.PutUint64(buf[:], val)
-	_, _ = w.Write(buf[8-n:]) // can't fail for bytes.Buffer
+	_, err := w.Write(buf[8-n:])
+	return err
 }
 
 // readUint64 reads `n` bytes from `r` and returns them in the lower `n` bytes

go/batch-submitter/drivers/sequencer/encoding_test.go

@@ -65,167 +65,21 @@ type AppendSequencerBatchParamsTest struct {
 	TotalElementsToAppend uint64                   `json:"total_elements_to_append"`
 	Contexts              []sequencer.BatchContext `json:"contexts"`
 	Txs                   []string                 `json:"txs"`
+	Error                 bool                     `json:"error"`
 }
 
-var appendSequencerBatchParamTests = AppendSequencerBatchParamsTestCases{
-	Tests: []AppendSequencerBatchParamsTest{
-		{
-			Name: "empty batch",
-			HexEncoding: "0000000000000000" +
-				"000000",
-			ShouldStartAtElement:  0,
-			TotalElementsToAppend: 0,
-			Contexts:              nil,
-			Txs:                   nil,
-		},
-		{
-			Name: "single tx",
-			HexEncoding: "0000000001000001" +
-				"000000" +
-				"00000ac9808080808080808080",
-			ShouldStartAtElement:  1,
-			TotalElementsToAppend: 1,
-			Contexts:              nil,
-			Txs: []string{
-				"c9808080808080808080",
-			},
-		},
-		{
-			Name: "multiple txs",
-			HexEncoding: "0000000001000004" +
-				"000000" +
-				"00000ac9808080808080808080" +
-				"00000ac9808080808080808080" +
-				"00000ac9808080808080808080" +
-				"00000ac9808080808080808080",
-			ShouldStartAtElement:  1,
-			TotalElementsToAppend: 4,
-			Contexts:              nil,
-			Txs: []string{
-				"c9808080808080808080",
-				"c9808080808080808080",
-				"c9808080808080808080",
-				"c9808080808080808080",
-			},
-		},
-		{
-			Name: "single context",
-			HexEncoding: "0000000001000000" +
-				"000001" +
-				"000102030405060708090a0b0c0d0e0f",
-			ShouldStartAtElement:  1,
-			TotalElementsToAppend: 0,
-			Contexts: []sequencer.BatchContext{
-				{
-					NumSequencedTxs:       0x000102,
-					NumSubsequentQueueTxs: 0x030405,
-					Timestamp:             0x060708090a,
-					BlockNumber:           0x0b0c0d0e0f,
-				},
-			},
-			Txs: nil,
-		},
-		{
-			Name: "multiple contexts",
-			HexEncoding: "0000000001000000" +
-				"000004" +
-				"000102030405060708090a0b0c0d0e0f" +
-				"000102030405060708090a0b0c0d0e0f" +
-				"000102030405060708090a0b0c0d0e0f" +
-				"000102030405060708090a0b0c0d0e0f",
-			ShouldStartAtElement:  1,
-			TotalElementsToAppend: 0,
-			Contexts: []sequencer.BatchContext{
-				{
-					NumSequencedTxs:       0x000102,
-					NumSubsequentQueueTxs: 0x030405,
-					Timestamp:             0x060708090a,
-					BlockNumber:           0x0b0c0d0e0f,
-				},
-				{
-					NumSequencedTxs:       0x000102,
-					NumSubsequentQueueTxs: 0x030405,
-					Timestamp:             0x060708090a,
-					BlockNumber:           0x0b0c0d0e0f,
-				},
-				{
-					NumSequencedTxs:       0x000102,
-					NumSubsequentQueueTxs: 0x030405,
-					Timestamp:             0x060708090a,
-					BlockNumber:           0x0b0c0d0e0f,
-				},
-				{
-					NumSequencedTxs:       0x000102,
-					NumSubsequentQueueTxs: 0x030405,
-					Timestamp:             0x060708090a,
-					BlockNumber:           0x0b0c0d0e0f,
-				},
-			},
-			Txs: nil,
-		},
-		{
-			Name: "complex",
-			HexEncoding: "0102030405060708" +
-				"000004" +
-				"000102030405060708090a0b0c0d0e0f" +
-				"000102030405060708090a0b0c0d0e0f" +
-				"000102030405060708090a0b0c0d0e0f" +
-				"000102030405060708090a0b0c0d0e0f" +
-				"00000ac9808080808080808080" +
-				"00000ac9808080808080808080" +
-				"00000ac9808080808080808080" +
-				"00000ac9808080808080808080",
-			ShouldStartAtElement:  0x0102030405,
-			TotalElementsToAppend: 0x060708,
-			Contexts: []sequencer.BatchContext{
-				{
-					NumSequencedTxs:       0x000102,
-					NumSubsequentQueueTxs: 0x030405,
-					Timestamp:             0x060708090a,
-					BlockNumber:           0x0b0c0d0e0f,
-				},
-				{
-					NumSequencedTxs:       0x000102,
-					NumSubsequentQueueTxs: 0x030405,
-					Timestamp:             0x060708090a,
-					BlockNumber:           0x0b0c0d0e0f,
-				},
-				{
-					NumSequencedTxs:       0x000102,
-					NumSubsequentQueueTxs: 0x030405,
-					Timestamp:             0x060708090a,
-					BlockNumber:           0x0b0c0d0e0f,
-				},
-				{
-					NumSequencedTxs:       0x000102,
-					NumSubsequentQueueTxs: 0x030405,
-					Timestamp:             0x060708090a,
-					BlockNumber:           0x0b0c0d0e0f,
-				},
-			},
-			Txs: []string{
-				"c9808080808080808080",
-				"c9808080808080808080",
-				"c9808080808080808080",
-				"c9808080808080808080",
-			},
-		},
-	},
-}
-
-// TestAppendSequencerBatchParamsEncodeDecodeMatchesJSON ensures that the
-// in-memory test vectors for valid encode/decode stay in sync with the JSON
-// version.
-func TestAppendSequencerBatchParamsEncodeDecodeMatchesJSON(t *testing.T) {
-	t.Parallel()
-
-	jsonBytes, err := json.MarshalIndent(appendSequencerBatchParamTests, "", "\t")
-	require.Nil(t, err)
+var appendSequencerBatchParamTests = AppendSequencerBatchParamsTestCases{}
 
+func init() {
 	data, err := os.ReadFile("./testdata/valid_append_sequencer_batch_params.json")
-	require.Nil(t, err)
+	if err != nil {
+		panic(err)
+	}
 
-	require.Equal(t, jsonBytes, data)
+	err = json.Unmarshal(data, &appendSequencerBatchParamTests)
+	if err != nil {
+		panic(err)
+	}
 }
 
 // TestAppendSequencerBatchParamsEncodeDecode asserts the proper encoding and
@@ -265,6 +119,7 @@ func testAppendSequencerBatchParamsEncodeDecode(
 		TotalElementsToAppend: test.TotalElementsToAppend,
 		Contexts:              test.Contexts,
 		Txs:                   nil,
+		Type:                  sequencer.BatchTypeLegacy,
 	}
 
 	// Decode the batch from the test string.
@@ -273,7 +128,12 @@ func testAppendSequencerBatchParamsEncodeDecode(
 
 	var params sequencer.AppendSequencerBatchParams
 	err = params.Read(bytes.NewReader(rawBytes))
-	require.Nil(t, err)
+	if test.Error {
+		require.ErrorIs(t, err, sequencer.ErrMalformedBatch)
+	} else {
+		require.Nil(t, err)
+	}
+	require.Equal(t, params.Type, sequencer.BatchTypeLegacy)
 
 	// Assert that the decoded params match the expected params. The
 	// transactions are compared serparetly (via hash), since the internal
@@ -290,8 +150,34 @@ func testAppendSequencerBatchParamsEncodeDecode(
 	// Finally, encode the decoded object and assert it matches the original
 	// hex string.
 	paramsBytes, err := params.Serialize()
+
+	// Return early when testing error cases, no need to reserialize again
+	if test.Error {
+		require.ErrorIs(t, err, sequencer.ErrMalformedBatch)
+		return
+	}
+
 	require.Nil(t, err)
 	require.Equal(t, test.HexEncoding, hex.EncodeToString(paramsBytes))
+
+	// Serialize the batches in compressed form
+	params.Type = sequencer.BatchTypeZlib
+	compressedParamsBytes, err := params.Serialize()
+	require.Nil(t, err)
+
+	// Deserialize the compressed batch
+	var paramsCompressed sequencer.AppendSequencerBatchParams
+	err = paramsCompressed.Read(bytes.NewReader(compressedParamsBytes))
+	require.Nil(t, err)
+	require.Equal(t, paramsCompressed.Type, sequencer.BatchTypeZlib)
+
+	expParams.Type = sequencer.BatchTypeZlib
+	decompressedTxs := paramsCompressed.Txs
+	paramsCompressed.Txs = nil
+
+	require.Equal(t, expParams, paramsCompressed)
+	compareTxs(t, expTxs, decompressedTxs)
+	paramsCompressed.Txs = decompressedTxs
 }
 
 // compareTxs compares a list of two transactions, testing each pair by tx hash.

go/batch-submitter/flags/flags.go

@@ -194,6 +194,12 @@ var (
 			"mnemonic. The mnemonic flag must also be set.",
 		EnvVar: prefixEnvVar("PROPOSER_HD_PATH"),
 	}
+	SequencerBatchType = cli.StringFlag{
+		Name:   "sequencer-batch-type",
+		Usage:  "The type of sequencer batch to be submitted. Valid arguments are legacy or zlib.",
+		Value:  "legacy",
+		EnvVar: prefixEnvVar("SEQUENCER_BATCH_TYPE"),
+	}
 	MetricsServerEnableFlag = cli.BoolFlag{
 		Name:   "metrics-server-enable",
 		Usage:  "Whether or not to run the embedded metrics server",
@@ -245,6 +251,7 @@ var optionalFlags = []cli.Flag{
 	SentryDsnFlag,
 	SentryTraceRateFlag,
 	BlockOffsetFlag,
+	SequencerBatchType,
 	SequencerPrivateKeyFlag,
 	ProposerPrivateKeyFlag,
 	MnemonicFlag,