Merge pull request #1802 from cfromknecht/bss-v0

feat: add v0 go batch submitter and gh actions integration

.github/workflows/integration.yml

@@ -18,6 +18,9 @@ jobs:
         image: registry:2
         ports:
           - 5000:5000
+    strategy:
+      matrix:
+        batch-submitter: [ts-batch-submitter, go-batch-submitter]
     env:
         DOCKER_BUILDKIT: 1
         COMPOSE_DOCKER_CLI_BUILD: 1
@@ -41,7 +44,7 @@ jobs:
         working-directory: ./ops
         run: |
           ./scripts/stats.sh &
-          docker-compose up -d
+          docker-compose -f docker-compose.yml -f docker-compose.${{ matrix.batch-submitter }}.yml up -d
 
       - name: Wait for the Sequencer node
         working-directory: ./ops

go/batch-submitter/batch_submitter.go

@@ -4,11 +4,16 @@ import (
 	"context"
 	"crypto/ecdsa"
 	"fmt"
+	"math/big"
 	"net/http"
 	"os"
 	"strconv"
 	"time"
 
+	"github.com/ethereum-optimism/optimism/go/batch-submitter/drivers/proposer"
+	"github.com/ethereum-optimism/optimism/go/batch-submitter/drivers/sequencer"
+	"github.com/ethereum-optimism/optimism/go/batch-submitter/txmgr"
+	l2ethclient "github.com/ethereum-optimism/optimism/l2geth/ethclient"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethclient"
@@ -41,12 +46,25 @@ func Main(gitVersion string) func(ctx *cli.Context) error {
 			defer sentry.Flush(2 * time.Second)
 		}
 
-		_, err = NewBatchSubmitter(cfg, gitVersion)
+		log.Info("Initializing batch submitter")
+
+		batchSubmitter, err := NewBatchSubmitter(cfg, gitVersion)
 		if err != nil {
 			log.Error("Unable to create batch submitter", "error", err)
 			return err
 		}
 
+		log.Info("Starting batch submitter")
+
+		if err := batchSubmitter.Start(); err != nil {
+			return err
+		}
+		defer batchSubmitter.Stop()
+
+		log.Info("Batch submitter started")
+
+		<-(chan struct{})(nil)
+
 		return nil
 	}
 }
@@ -57,11 +75,14 @@ type BatchSubmitter struct {
 	ctx              context.Context
 	cfg              Config
 	l1Client         *ethclient.Client
-	l2Client         *ethclient.Client
+	l2Client         *l2ethclient.Client
 	sequencerPrivKey *ecdsa.PrivateKey
 	proposerPrivKey  *ecdsa.PrivateKey
 	ctcAddress       common.Address
 	sccAddress       common.Address
+
+	batchTxService    *Service
+	batchStateService *Service
 }
 
 // NewBatchSubmitter initializes the BatchSubmitter, gathering any resources
@@ -118,14 +139,14 @@ func NewBatchSubmitter(cfg Config, gitVersion string) (*BatchSubmitter, error) {
 		return nil, err
 	}
 
-	// Connect to L1 and L2 providers. Perform these lastsince they are the
+	// Connect to L1 and L2 providers. Perform these last since they are the
 	// most expensive.
-	l1Client, err := dialEthClientWithTimeout(ctx, cfg.L1EthRpc)
+	l1Client, err := dialL1EthClientWithTimeout(ctx, cfg.L1EthRpc)
 	if err != nil {
 		return nil, err
 	}
 
-	l2Client, err := dialEthClientWithTimeout(ctx, cfg.L2EthRpc)
+	l2Client, err := dialL2EthClientWithTimeout(ctx, cfg.L2EthRpc)
 	if err != nil {
 		return nil, err
 	}
@@ -134,6 +155,70 @@ func NewBatchSubmitter(cfg Config, gitVersion string) (*BatchSubmitter, error) {
 		go runMetricsServer(cfg.MetricsHostname, cfg.MetricsPort)
 	}
 
+	chainID, err := l1Client.ChainID(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	txManagerConfig := txmgr.Config{
+		MinGasPrice:          gasPriceFromGwei(1),
+		MaxGasPrice:          gasPriceFromGwei(cfg.MaxGasPriceInGwei),
+		GasRetryIncrement:    gasPriceFromGwei(cfg.GasRetryIncrement),
+		ResubmissionTimeout:  cfg.ResubmissionTimeout,
+		ReceiptQueryInterval: time.Second,
+	}
+
+	var batchTxService *Service
+	if cfg.RunTxBatchSubmitter {
+		batchTxDriver, err := sequencer.NewDriver(sequencer.Config{
+			Name:        "SEQUENCER",
+			L1Client:    l1Client,
+			L2Client:    l2Client,
+			BlockOffset: cfg.BlockOffset,
+			MaxTxSize:   cfg.MaxL1TxSize,
+			CTCAddr:     ctcAddress,
+			ChainID:     chainID,
+			PrivKey:     sequencerPrivKey,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		batchTxService = NewService(ServiceConfig{
+			Context:         ctx,
+			Driver:          batchTxDriver,
+			PollInterval:    cfg.PollInterval,
+			L1Client:        l1Client,
+			TxManagerConfig: txManagerConfig,
+		})
+	}
+
+	var batchStateService *Service
+	if cfg.RunStateBatchSubmitter {
+		batchStateDriver, err := proposer.NewDriver(proposer.Config{
+			Name:        "PROPOSER",
+			L1Client:    l1Client,
+			L2Client:    l2Client,
+			BlockOffset: cfg.BlockOffset,
+			MaxTxSize:   cfg.MaxL1TxSize,
+			SCCAddr:     sccAddress,
+			CTCAddr:     ctcAddress,
+			ChainID:     chainID,
+			PrivKey:     proposerPrivKey,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		batchStateService = NewService(ServiceConfig{
+			Context:         ctx,
+			Driver:          batchStateDriver,
+			PollInterval:    cfg.PollInterval,
+			L1Client:        l1Client,
+			TxManagerConfig: txManagerConfig,
+		})
+	}
+
 	return &BatchSubmitter{
 		ctx:               ctx,
 		cfg:               cfg,
@@ -143,9 +228,34 @@ func NewBatchSubmitter(cfg Config, gitVersion string) (*BatchSubmitter, error) {
 		proposerPrivKey:   proposerPrivKey,
 		ctcAddress:        ctcAddress,
 		sccAddress:        sccAddress,
+		batchTxService:    batchTxService,
+		batchStateService: batchStateService,
 	}, nil
 }
 
+func (b *BatchSubmitter) Start() error {
+	if b.cfg.RunTxBatchSubmitter {
+		if err := b.batchTxService.Start(); err != nil {
+			return err
+		}
+	}
+	if b.cfg.RunStateBatchSubmitter {
+		if err := b.batchStateService.Start(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (b *BatchSubmitter) Stop() {
+	if b.cfg.RunTxBatchSubmitter {
+		_ = b.batchTxService.Stop()
+	}
+	if b.cfg.RunStateBatchSubmitter {
+		_ = b.batchStateService.Stop()
+	}
+}
+
 // parseWalletPrivKeyAndContractAddr returns the wallet private key to use for
 // sending transactions as well as the contract address to send to for a
 // particular sub-service.
@@ -191,10 +301,10 @@ func runMetricsServer(hostname string, port uint64) {
 	_ = http.ListenAndServe(metricsAddr, nil)
 }
 
-// dialEthClientWithTimeout attempts to dial the L1 or L2 provider using the
+// dialL1EthClientWithTimeout attempts to dial the L1 provider using the
 // provided URL. If the dial doesn't complete within defaultDialTimeout seconds,
 // this method will return an error.
-func dialEthClientWithTimeout(ctx context.Context, url string) (
+func dialL1EthClientWithTimeout(ctx context.Context, url string) (
 	*ethclient.Client, error) {
 
 	ctxt, cancel := context.WithTimeout(ctx, defaultDialTimeout)
@@ -203,6 +313,18 @@ func dialEthClientWithTimeout(ctx context.Context, url string) (
 	return ethclient.DialContext(ctxt, url)
 }
 
+// dialL2EthClientWithTimeout attempts to dial the L2 provider using the
+// provided URL. If the dial doesn't complete within defaultDialTimeout seconds,
+// this method will return an error.
+func dialL2EthClientWithTimeout(ctx context.Context, url string) (
+	*l2ethclient.Client, error) {
+
+	ctxt, cancel := context.WithTimeout(ctx, defaultDialTimeout)
+	defer cancel()
+
+	return l2ethclient.DialContext(ctxt, url)
+}
+
 // traceRateToFloat64 converts a time.Duration into a valid float64 for the
 // Sentry client. The client only accepts values between 0.0 and 1.0, so this
 // method clamps anything greater than 1 second to 1.0.
@@ -213,3 +335,7 @@ func traceRateToFloat64(rate time.Duration) float64 {
 	}
 	return rate64
 }
+
+func gasPriceFromGwei(gasPriceInGwei uint64) *big.Int {
+	return new(big.Int).SetUint64(gasPriceInGwei * 1e9)
+}

go/batch-submitter/config.go

@@ -223,6 +223,10 @@ func NewConfig(ctx *cli.Context) (Config, error) {
 // ensure that it is well-formed.
 func ValidateConfig(cfg *Config) error {
 	// Sanity check log level.
+	if cfg.LogLevel == "" {
+		cfg.LogLevel = "debug"
+	}
+
 	_, err := log.LvlFromString(cfg.LogLevel)
 	if err != nil {
 		return err

go/batch-submitter/drivers/proposer/driver.go

+package proposer
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum-optimism/optimism/go/batch-submitter/bindings/ctc"
+	"github.com/ethereum-optimism/optimism/go/batch-submitter/bindings/scc"
+	l2types "github.com/ethereum-optimism/optimism/l2geth/core/types"
+	l2ethclient "github.com/ethereum-optimism/optimism/l2geth/ethclient"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethclient"
+)
+
+var bigOne = new(big.Int).SetUint64(1) //nolint:unused
+
+type Config struct {
+	Name        string
+	L1Client    *ethclient.Client
+	L2Client    *l2ethclient.Client
+	BlockOffset uint64
+	MaxTxSize   uint64
+	SCCAddr     common.Address
+	CTCAddr     common.Address
+	ChainID     *big.Int
+	PrivKey     *ecdsa.PrivateKey
+}
+
+type Driver struct {
+	cfg         Config
+	sccContract *scc.StateCommitmentChain
+	ctcContract *ctc.CanonicalTransactionChain
+	walletAddr  common.Address
+}
+
+func NewDriver(cfg Config) (*Driver, error) {
+	sccContract, err := scc.NewStateCommitmentChain(
+		cfg.SCCAddr, cfg.L1Client,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	ctcContract, err := ctc.NewCanonicalTransactionChain(
+		cfg.CTCAddr, cfg.L1Client,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	walletAddr := crypto.PubkeyToAddress(cfg.PrivKey.PublicKey)
+
+	return &Driver{
+		cfg:         cfg,
+		sccContract: sccContract,
+		ctcContract: ctcContract,
+		walletAddr:  walletAddr,
+	}, nil
+}
+
+// Name is an identifier used to prefix logs for a particular service.
+func (d *Driver) Name() string {
+	return d.cfg.Name
+}
+
+// WalletAddr is the wallet address used to pay for batch transaction fees.
+func (d *Driver) WalletAddr() common.Address {
+	return d.walletAddr
+}
+
+// GetBatchBlockRange returns the start and end L2 block heights that need to be
+// processed. Note that the end value is *exclusive*, therefore if the returned
+// values are identical nothing needs to be processed.
+func (d *Driver) GetBatchBlockRange(
+	ctx context.Context) (*big.Int, *big.Int, error) {
+
+	blockOffset := new(big.Int).SetUint64(d.cfg.BlockOffset)
+	maxBatchSize := new(big.Int).SetUint64(1)
+
+	start, err := d.sccContract.GetTotalElements(&bind.CallOpts{
+		Pending: false,
+		Context: ctx,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	start.Add(start, blockOffset)
+
+	totalElements, err := d.ctcContract.GetTotalElements(&bind.CallOpts{
+		Pending: false,
+		Context: ctx,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	totalElements.Add(totalElements, blockOffset)
+
+	// Take min(start + blockOffset + maxBatchSize, totalElements).
+	end := new(big.Int).Add(start, maxBatchSize)
+	if totalElements.Cmp(end) < 0 {
+		end.Set(totalElements)
+	}
+
+	if start.Cmp(end) > 0 {
+		return nil, nil, fmt.Errorf("invalid range, "+
+			"end(%v) < start(%v)", end, start)
+	}
+
+	return start, end, nil
+}
+
+// SubmitBatchTx transforms the L2 blocks between start and end into a batch
+// transaction using the given nonce and gasPrice. The final transaction is
+// published and returned to the call.
+func (d *Driver) SubmitBatchTx(
+	ctx context.Context,
+	start, end, nonce, gasPrice *big.Int) (*types.Transaction, error) {
+
+	var blocks []*l2types.Block
+	for i := new(big.Int).Set(start); i.Cmp(end) < 0; i.Add(i, bigOne) {
+		block, err := d.cfg.L2Client.BlockByNumber(ctx, i)
+		if err != nil {
+			return nil, err
+		}
+
+		blocks = append(blocks, block)
+
+		// TODO(conner): remove when moving to multiple blocks
+		break //nolint
+	}
+
+	var stateRoots = make([][32]byte, 0, len(blocks))
+	for _, block := range blocks {
+		stateRoots = append(stateRoots, block.Root())
+	}
+
+	opts, err := bind.NewKeyedTransactorWithChainID(
+		d.cfg.PrivKey, d.cfg.ChainID,
+	)
+	if err != nil {
+		return nil, err
+	}
+	opts.Nonce = nonce
+	opts.Context = ctx
+	opts.GasPrice = gasPrice
+
+	blockOffset := new(big.Int).SetUint64(d.cfg.BlockOffset)
+	offsetStartsAtIndex := new(big.Int).Sub(start, blockOffset)
+
+	return d.sccContract.AppendStateBatch(opts, stateRoots, offsetStartsAtIndex)
+}

go/batch-submitter/drivers/sequencer/driver.go

+package sequencer
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"encoding/hex"
+	"fmt"
+	"math/big"
+	"strings"
+
+	"github.com/ethereum-optimism/optimism/go/batch-submitter/bindings/ctc"
+	l2types "github.com/ethereum-optimism/optimism/l2geth/core/types"
+	l2ethclient "github.com/ethereum-optimism/optimism/l2geth/ethclient"
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+const (
+	appendSequencerBatchMethodName = "appendSequencerBatch"
+)
+
+var bigOne = new(big.Int).SetUint64(1)
+
+type Config struct {
+	Name        string
+	L1Client    *ethclient.Client
+	L2Client    *l2ethclient.Client
+	BlockOffset uint64
+	MaxTxSize   uint64
+	CTCAddr     common.Address
+	ChainID     *big.Int
+	PrivKey     *ecdsa.PrivateKey
+}
+
+type Driver struct {
+	cfg            Config
+	ctcContract    *ctc.CanonicalTransactionChain
+	rawCtcContract *bind.BoundContract
+	walletAddr     common.Address
+	ctcABI         *abi.ABI
+}
+
+func NewDriver(cfg Config) (*Driver, error) {
+	ctcContract, err := ctc.NewCanonicalTransactionChain(
+		cfg.CTCAddr, cfg.L1Client,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	parsed, err := abi.JSON(strings.NewReader(
+		ctc.CanonicalTransactionChainABI,
+	))
+	if err != nil {
+		return nil, err
+	}
+
+	ctcABI, err := ctc.CanonicalTransactionChainMetaData.GetAbi()
+	if err != nil {
+		return nil, err
+	}
+
+	rawCtcContract := bind.NewBoundContract(
+		cfg.CTCAddr, parsed, cfg.L1Client, cfg.L1Client,
+		cfg.L1Client,
+	)
+
+	walletAddr := crypto.PubkeyToAddress(cfg.PrivKey.PublicKey)
+
+	return &Driver{
+		cfg:            cfg,
+		ctcContract:    ctcContract,
+		rawCtcContract: rawCtcContract,
+		walletAddr:     walletAddr,
+		ctcABI:         ctcABI,
+	}, nil
+}
+
+// Name is an identifier used to prefix logs for a particular service.
+func (d *Driver) Name() string {
+	return d.cfg.Name
+}
+
+// WalletAddr is the wallet address used to pay for batch transaction fees.
+func (d *Driver) WalletAddr() common.Address {
+	return d.walletAddr
+}
+
+// GetBatchBlockRange returns the start and end L2 block heights that need to be
+// processed. Note that the end value is *exclusive*, therefore if the returned
+// values are identical nothing needs to be processed.
+func (d *Driver) GetBatchBlockRange(
+	ctx context.Context) (*big.Int, *big.Int, error) {
+
+	blockOffset := new(big.Int).SetUint64(d.cfg.BlockOffset)
+
+	start, err := d.ctcContract.GetTotalElements(&bind.CallOpts{
+		Pending: false,
+		Context: ctx,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+	start.Add(start, blockOffset)
+
+	latestHeader, err := d.cfg.L2Client.HeaderByNumber(ctx, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Add one because end is *exclusive*.
+	end := new(big.Int).Add(latestHeader.Number, bigOne)
+
+	if start.Cmp(end) > 0 {
+		return nil, nil, fmt.Errorf("invalid range, "+
+			"end(%v) < start(%v)", end, start)
+	}
+
+	return start, end, nil
+}
+
+// SubmitBatchTx transforms the L2 blocks between start and end into a batch
+// transaction using the given nonce and gasPrice. The final transaction is
+// published and returned to the call.
+func (d *Driver) SubmitBatchTx(
+	ctx context.Context,
+	start, end, nonce, gasPrice *big.Int) (*types.Transaction, error) {
+
+	name := d.cfg.Name
+
+	log.Info(name+" submitting batch tx", "start", start, "end", end,
+		"gasPrice", gasPrice)
+
+	var blocks []*l2types.Block
+	for i := new(big.Int).Set(start); i.Cmp(end) < 0; i.Add(i, bigOne) {
+		block, err := d.cfg.L2Client.BlockByNumber(ctx, i)
+		if err != nil {
+			return nil, err
+		}
+
+		blocks = append(blocks, block)
+
+		// TODO(conner): remove when moving to multiple blocks
+		break //nolint
+	}
+
+	var batchElements = make([]BatchElement, 0, len(blocks))
+	for _, block := range blocks {
+		batchElements = append(batchElements, BatchElementFromBlock(block))
+	}
+
+	shouldStartAt := start.Uint64()
+	batchParams, err := GenSequencerBatchParams(
+		shouldStartAt, d.cfg.BlockOffset, batchElements,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Info(name+" batch params", "params", fmt.Sprintf("%#v", batchParams))
+
+	batchArguments, err := batchParams.Serialize()
+	if err != nil {
+		return nil, err
+	}
+
+	appendSequencerBatchID := d.ctcABI.Methods[appendSequencerBatchMethodName].ID
+	batchCallData := append(appendSequencerBatchID, batchArguments...)
+
+	if uint64(len(batchCallData)) > d.cfg.MaxTxSize {
+		panic("call data too large")
+	}
+
+	log.Info(name+" batch call data", "data", hex.EncodeToString(batchCallData))
+
+	opts, err := bind.NewKeyedTransactorWithChainID(
+		d.cfg.PrivKey, d.cfg.ChainID,
+	)
+	if err != nil {
+		return nil, err
+	}
+	opts.Nonce = nonce
+	opts.Context = ctx
+	opts.GasPrice = gasPrice
+
+	return d.rawCtcContract.RawTransact(opts, batchCallData)
+}

go/batch-submitter/go.mod

@@ -3,10 +3,9 @@ module github.com/ethereum-optimism/optimism/go/batch-submitter
 go 1.16
 
 require (
-	github.com/btcsuite/btcd v0.22.0-beta // indirect
 	github.com/decred/dcrd/hdkeychain/v3 v3.0.0
 	github.com/ethereum-optimism/optimism/l2geth v1.0.0
-	github.com/ethereum/go-ethereum v1.10.11
+	github.com/ethereum/go-ethereum v1.10.12
 	github.com/getsentry/sentry-go v0.11.0
 	github.com/prometheus/client_golang v1.11.0
 	github.com/stretchr/testify v1.7.0

go/batch-submitter/service.go

+package batchsubmitter
+
+import (
+	"context"
+	"math/big"
+	"sync"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/go/batch-submitter/txmgr"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// Driver is an interface for creating and submitting batch transactions for a
+// specific contract.
+type Driver interface {
+	// Name is an identifier used to prefix logs for a particular service.
+	Name() string
+
+	// WalletAddr is the wallet address used to pay for batch transaction
+	// fees.
+	WalletAddr() common.Address
+
+	// GetBatchBlockRange returns the start and end L2 block heights that
+	// need to be processed. Note that the end value is *exclusive*,
+	// therefore if the returned values are identical nothing needs to be
+	// processed.
+	GetBatchBlockRange(ctx context.Context) (*big.Int, *big.Int, error)
+
+	// SubmitBatchTx transforms the L2 blocks between start and end into a
+	// batch transaction using the given nonce and gasPrice. The final
+	// transaction is published and returned to the call.
+	SubmitBatchTx(
+		ctx context.Context,
+		start, end, nonce, gasPrice *big.Int,
+	) (*types.Transaction, error)
+}
+
+type ServiceConfig struct {
+	Context         context.Context
+	Driver          Driver
+	PollInterval    time.Duration
+	L1Client        *ethclient.Client
+	TxManagerConfig txmgr.Config
+}
+
+type Service struct {
+	cfg    ServiceConfig
+	ctx    context.Context
+	cancel func()
+
+	txMgr txmgr.TxManager
+
+	wg sync.WaitGroup
+}
+
+func NewService(cfg ServiceConfig) *Service {
+	ctx, cancel := context.WithCancel(cfg.Context)
+
+	txMgr := txmgr.NewSimpleTxManager(
+		cfg.Driver.Name(), cfg.TxManagerConfig, cfg.L1Client,
+	)
+
+	return &Service{
+		cfg:    cfg,
+		ctx:    ctx,
+		cancel: cancel,
+		txMgr:  txMgr,
+	}
+}
+
+func (s *Service) Start() error {
+	s.wg.Add(1)
+	go s.eventLoop()
+	return nil
+}
+
+func (s *Service) Stop() error {
+	s.cancel()
+	s.wg.Wait()
+	return nil
+}
+
+func (s *Service) eventLoop() {
+	defer s.wg.Done()
+
+	name := s.cfg.Driver.Name()
+
+	for {
+		select {
+		case <-time.After(s.cfg.PollInterval):
+			log.Info(name + " fetching current block range")
+
+			start, end, err := s.cfg.Driver.GetBatchBlockRange(s.ctx)
+			if err != nil {
+				log.Error(name+" unable to get block range", "err", err)
+				continue
+			}
+
+			log.Info(name+" block range", "start", start, "end", end)
+
+			// No new updates.
+			if start.Cmp(end) == 0 {
+				continue
+			}
+
+			nonce64, err := s.cfg.L1Client.NonceAt(
+				s.ctx, s.cfg.Driver.WalletAddr(), nil,
+			)
+			if err != nil {
+				log.Error(name+" unable to get current nonce",
+					"err", err)
+				continue
+			}
+			nonce := new(big.Int).SetUint64(nonce64)
+
+			sendTx := func(
+				ctx context.Context,
+				gasPrice *big.Int,
+			) (*types.Transaction, error) {
+				log.Info(name+" attempting batch tx", "start", start,
+					"end", end, "nonce", nonce,
+					"gasPrice", gasPrice)
+				return s.cfg.Driver.SubmitBatchTx(
+					ctx, start, end, nonce, gasPrice,
+				)
+			}
+
+			receipt, err := s.txMgr.Send(s.ctx, sendTx)
+			if err != nil {
+				log.Error(name+" unable to publish batch tx",
+					"err", err)
+				continue
+			}
+
+			log.Info(name+" batch tx successfully published",
+				"tx_hash", receipt.TxHash)
+
+		case err := <-s.ctx.Done():
+			log.Error(name+" service shutting down", "err", err)
+			return
+		}
+	}
+}

go/batch-submitter/txmgr/txmgr.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"math/big"
+	"strings"
 	"sync"
 	"time"
 
@@ -25,6 +26,8 @@ type SendTxFunc = func(
 
 // Config houses parameters for altering the behavior of a SimpleTxManager.
 type Config struct {
+	Name string
+
 	// MinGasPrice is the minimum gas price (in gwei). This is used as the
 	// initial publication attempt.
 	MinGasPrice *big.Int
@@ -78,13 +81,17 @@ type ReceiptSource interface {
 // SimpleTxManager is a implementation of TxManager that performs linear fee
 // bumping of a tx until it confirms.
 type SimpleTxManager struct {
+	name    string
 	cfg     Config
 	backend ReceiptSource
 }
 
 // NewSimpleTxManager initializes a new SimpleTxManager with the passed Config.
-func NewSimpleTxManager(cfg Config, backend ReceiptSource) *SimpleTxManager {
+func NewSimpleTxManager(
+	name string, cfg Config, backend ReceiptSource) *SimpleTxManager {
+
 	return &SimpleTxManager{
+		name:    name,
 		cfg:     cfg,
 		backend: backend,
 	}
@@ -99,6 +106,8 @@ func NewSimpleTxManager(cfg Config, backend ReceiptSource) *SimpleTxManager {
 func (m *SimpleTxManager) Send(
 	ctx context.Context, sendTx SendTxFunc) (*types.Receipt, error) {
 
+	name := m.name
+
 	// Initialize a wait group to track any spawned goroutines, and ensure
 	// we properly clean up any dangling resources this method generates.
 	// We assert that this is the case thoroughly in our unit tests.
@@ -121,14 +130,18 @@ func (m *SimpleTxManager) Send(
 		// Sign and publish transaction with current gas price.
 		tx, err := sendTx(ctxc, gasPrice)
 		if err != nil {
-			log.Error("Unable to publish transaction",
+			if err == context.Canceled ||
+				strings.Contains(err.Error(), "context canceled") {
+				return
+			}
+			log.Error(name+" unable to publish transaction",
 				"gas_price", gasPrice, "err", err)
 			// TODO(conner): add retry?
 			return
 		}
 
 		txHash := tx.Hash()
-		log.Info("Transaction published successfully", "hash", txHash,
+		log.Info(name+" transaction published successfully", "hash", txHash,
 			"gas_price", gasPrice)
 
 		// Wait for the transaction to be mined, reporting the receipt
@@ -137,7 +150,7 @@ func (m *SimpleTxManager) Send(
 			ctxc, m.backend, tx, m.cfg.ReceiptQueryInterval,
 		)
 		if err != nil {
-			log.Trace("Send tx failed", "hash", txHash,
+			log.Debug(name+" send tx failed", "hash", txHash,
 				"gas_price", gasPrice, "err", err)
 		}
 		if receipt != nil {
@@ -145,7 +158,7 @@ func (m *SimpleTxManager) Send(
 			// if more than one receipt is discovered.
 			select {
 			case receiptChan <- receipt:
-				log.Trace("Send tx succeeded", "hash", txHash,
+				log.Trace(name+" send tx succeeded", "hash", txHash,
 					"gas_price", gasPrice)
 			default:
 			}

go/batch-submitter/txmgr/txmgr_test.go

@@ -83,7 +83,7 @@ type testHarness struct {
 // configuration.
 func newTestHarnessWithConfig(cfg txmgr.Config) *testHarness {
 	backend := newMockBackend()
-	mgr := txmgr.NewSimpleTxManager(cfg, backend)
+	mgr := txmgr.NewSimpleTxManager("TEST", cfg, backend)
 
 	return &testHarness{
 		cfg:     cfg,

l2geth/cmd/devp2p/dns_cloudflare.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
@@ -79,7 +80,7 @@ func (c *cloudflareClient) checkZone(name string) error {
 		c.zoneID = id
 	}
 	log.Info(fmt.Sprintf("Checking Permissions on zone %s", c.zoneID))
-	zone, err := c.ZoneDetails(c.zoneID)
+	zone, err := c.ZoneDetails(context.Background(), c.zoneID)
 	if err != nil {
 		return err
 	}
@@ -112,7 +113,7 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
 	records = lrecords
 
 	log.Info(fmt.Sprintf("Retrieving existing TXT records on %s", name))
-	entries, err := c.DNSRecords(c.zoneID, cloudflare.DNSRecord{Type: "TXT"})
+	entries, err := c.DNSRecords(context.Background(), c.zoneID, cloudflare.DNSRecord{Type: "TXT"})
 	if err != nil {
 		return err
 	}
@@ -134,12 +135,12 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
 			if path != name {
 				ttl = treeNodeTTL // Max TTL permitted by Cloudflare
 			}
-			_, err = c.CreateDNSRecord(c.zoneID, cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl})
+			_, err = c.CreateDNSRecord(context.Background(), c.zoneID, cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl})
 		} else if old.Content != val {
 			// Entry already exists, only change its content.
 			log.Info(fmt.Sprintf("Updating %s from %q to %q", path, old.Content, val))
 			old.Content = val
-			err = c.UpdateDNSRecord(c.zoneID, old.ID, old)
+			err = c.UpdateDNSRecord(context.Background(), c.zoneID, old.ID, old)
 		} else {
 			log.Info(fmt.Sprintf("Skipping %s = %q", path, val))
 		}
@@ -155,7 +156,7 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
 		}
 		// Stale entry, nuke it.
 		log.Info(fmt.Sprintf("Deleting %s = %q", path, entry.Content))
-		if err := c.DeleteDNSRecord(c.zoneID, entry.ID); err != nil {
+		if err := c.DeleteDNSRecord(context.Background(), c.zoneID, entry.ID); err != nil {
 			return fmt.Errorf("failed to delete %s: %v", path, err)
 		}
 	}

l2geth/core/types/transaction_meta.go

@@ -7,6 +7,7 @@ package types
 import (
 	"bytes"
 	"encoding/binary"
+	"fmt"
 	"math/big"
 
 	"github.com/ethereum-optimism/optimism/l2geth/common"
@@ -31,6 +32,19 @@ func (q QueueOrigin) String() string {
 	}
 }
 
+func (q *QueueOrigin) UnmarshalJSON(b []byte) error {
+	switch string(b) {
+	case "\"sequencer\"":
+		*q = QueueOriginSequencer
+		return nil
+	case "\"l1\"":
+		*q = QueueOriginL1ToL2
+		return nil
+	default:
+		return fmt.Errorf("Unknown QueueOrigin: %q", b)
+	}
+}
+
 //go:generate gencodec -type TransactionMeta -out gen_tx_meta_json.go
 
 type TransactionMeta struct {

l2geth/crypto/secp256k1/panic_cb.go

-// Copyright 2015 Jeffrey Wilcke, Felix Lange, Gustav Simonsson. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-package secp256k1
-
-import "C"
-import "unsafe"
-
-// Callbacks for converting libsecp256k1 internal faults into
-// recoverable Go panics.
-
-//export secp256k1GoPanicIllegal
-func secp256k1GoPanicIllegal(msg *C.char, data unsafe.Pointer) {
-	panic("illegal argument: " + C.GoString(msg))
-}
-
-//export secp256k1GoPanicError
-func secp256k1GoPanicError(msg *C.char, data unsafe.Pointer) {
-	panic("internal error: " + C.GoString(msg))
-}

l2geth/crypto/secp256k1/secp256.go

-// Copyright 2015 Jeffrey Wilcke, Felix Lange, Gustav Simonsson. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be found in
-// the LICENSE file.
-
-// Package secp256k1 wraps the bitcoin secp256k1 C library.
 package secp256k1
 
-/*
-#cgo CFLAGS: -I./libsecp256k1
-#cgo CFLAGS: -I./libsecp256k1/src/
-#define USE_NUM_NONE
-#define USE_FIELD_10X26
-#define USE_FIELD_INV_BUILTIN
-#define USE_SCALAR_8X32
-#define USE_SCALAR_INV_BUILTIN
-#define NDEBUG
-#include "./libsecp256k1/src/secp256k1.c"
-#include "./libsecp256k1/src/modules/recovery/main_impl.h"
-#include "ext.h"
-
-typedef void (*callbackFunc) (const char* msg, void* data);
-extern void secp256k1GoPanicIllegal(const char* msg, void* data);
-extern void secp256k1GoPanicError(const char* msg, void* data);
-*/
-import "C"
-
 import (
-	"errors"
 	"math/big"
-	"unsafe"
-)
-
-var context *C.secp256k1_context
 
-func init() {
-	// around 20 ms on a modern CPU.
-	context = C.secp256k1_context_create_sign_verify()
-	C.secp256k1_context_set_illegal_callback(context, C.callbackFunc(C.secp256k1GoPanicIllegal), nil)
-	C.secp256k1_context_set_error_callback(context, C.callbackFunc(C.secp256k1GoPanicError), nil)
-}
+	"github.com/ethereum/go-ethereum/crypto/secp256k1"
+)
 
 var (
-	ErrInvalidMsgLen       = errors.New("invalid message length, need 32 bytes")
-	ErrInvalidSignatureLen = errors.New("invalid signature length")
-	ErrInvalidRecoveryID   = errors.New("invalid signature recovery id")
-	ErrInvalidKey          = errors.New("invalid private key")
-	ErrInvalidPubkey       = errors.New("invalid public key")
-	ErrSignFailed          = errors.New("signing failed")
-	ErrRecoverFailed       = errors.New("recovery failed")
+	ErrInvalidMsgLen       = secp256k1.ErrInvalidMsgLen
+	ErrInvalidSignatureLen = secp256k1.ErrInvalidSignatureLen
+	ErrInvalidRecoveryID   = secp256k1.ErrInvalidRecoveryID
+	ErrInvalidKey          = secp256k1.ErrInvalidKey
+	ErrInvalidPubkey       = secp256k1.ErrInvalidPubkey
+	ErrSignFailed          = secp256k1.ErrSignFailed
+	ErrRecoverFailed       = secp256k1.ErrRecoverFailed
 )
 
 // Sign creates a recoverable ECDSA signature.
@@ -56,34 +23,7 @@ var (
 // directly by an attacker. It is usually preferable to use a cryptographic
 // hash function on any input before handing it to this function.
 func Sign(msg []byte, seckey []byte) ([]byte, error) {
-	if len(msg) != 32 {
-		return nil, ErrInvalidMsgLen
-	}
-	if len(seckey) != 32 {
-		return nil, ErrInvalidKey
-	}
-	seckeydata := (*C.uchar)(unsafe.Pointer(&seckey[0]))
-	if C.secp256k1_ec_seckey_verify(context, seckeydata) != 1 {
-		return nil, ErrInvalidKey
-	}
-
-	var (
-		msgdata   = (*C.uchar)(unsafe.Pointer(&msg[0]))
-		noncefunc = C.secp256k1_nonce_function_rfc6979
-		sigstruct C.secp256k1_ecdsa_recoverable_signature
-	)
-	if C.secp256k1_ecdsa_sign_recoverable(context, &sigstruct, msgdata, seckeydata, noncefunc, nil) == 0 {
-		return nil, ErrSignFailed
-	}
-
-	var (
-		sig     = make([]byte, 65)
-		sigdata = (*C.uchar)(unsafe.Pointer(&sig[0]))
-		recid   C.int
-	)
-	C.secp256k1_ecdsa_recoverable_signature_serialize_compact(context, sigdata, &recid, &sigstruct)
-	sig[64] = byte(recid) // add back recid to get 65 bytes sig
-	return sig, nil
+	return secp256k1.Sign(msg, seckey)
 }
 
 // RecoverPubkey returns the public key of the signer.
@@ -91,77 +31,22 @@ func Sign(msg []byte, seckey []byte) ([]byte, error) {
 // sig must be a 65-byte compact ECDSA signature containing the
 // recovery id as the last element.
 func RecoverPubkey(msg []byte, sig []byte) ([]byte, error) {
-	if len(msg) != 32 {
-		return nil, ErrInvalidMsgLen
-	}
-	if err := checkSignature(sig); err != nil {
-		return nil, err
-	}
-
-	var (
-		pubkey  = make([]byte, 65)
-		sigdata = (*C.uchar)(unsafe.Pointer(&sig[0]))
-		msgdata = (*C.uchar)(unsafe.Pointer(&msg[0]))
-	)
-	if C.secp256k1_ext_ecdsa_recover(context, (*C.uchar)(unsafe.Pointer(&pubkey[0])), sigdata, msgdata) == 0 {
-		return nil, ErrRecoverFailed
-	}
-	return pubkey, nil
+	return secp256k1.RecoverPubkey(msg, sig)
 }
 
 // VerifySignature checks that the given pubkey created signature over message.
 // The signature should be in [R || S] format.
 func VerifySignature(pubkey, msg, signature []byte) bool {
-	if len(msg) != 32 || len(signature) != 64 || len(pubkey) == 0 {
-		return false
-	}
-	sigdata := (*C.uchar)(unsafe.Pointer(&signature[0]))
-	msgdata := (*C.uchar)(unsafe.Pointer(&msg[0]))
-	keydata := (*C.uchar)(unsafe.Pointer(&pubkey[0]))
-	return C.secp256k1_ext_ecdsa_verify(context, sigdata, msgdata, keydata, C.size_t(len(pubkey))) != 0
+	return secp256k1.VerifySignature(pubkey, msg, signature)
 }
 
 // DecompressPubkey parses a public key in the 33-byte compressed format.
 // It returns non-nil coordinates if the public key is valid.
 func DecompressPubkey(pubkey []byte) (x, y *big.Int) {
-	if len(pubkey) != 33 {
-		return nil, nil
-	}
-	var (
-		pubkeydata = (*C.uchar)(unsafe.Pointer(&pubkey[0]))
-		pubkeylen  = C.size_t(len(pubkey))
-		out        = make([]byte, 65)
-		outdata    = (*C.uchar)(unsafe.Pointer(&out[0]))
-		outlen     = C.size_t(len(out))
-	)
-	if C.secp256k1_ext_reencode_pubkey(context, outdata, outlen, pubkeydata, pubkeylen) == 0 {
-		return nil, nil
-	}
-	return new(big.Int).SetBytes(out[1:33]), new(big.Int).SetBytes(out[33:])
+	return secp256k1.DecompressPubkey(pubkey)
 }
 
 // CompressPubkey encodes a public key to 33-byte compressed format.
 func CompressPubkey(x, y *big.Int) []byte {
-	var (
-		pubkey     = S256().Marshal(x, y)
-		pubkeydata = (*C.uchar)(unsafe.Pointer(&pubkey[0]))
-		pubkeylen  = C.size_t(len(pubkey))
-		out        = make([]byte, 33)
-		outdata    = (*C.uchar)(unsafe.Pointer(&out[0]))
-		outlen     = C.size_t(len(out))
-	)
-	if C.secp256k1_ext_reencode_pubkey(context, outdata, outlen, pubkeydata, pubkeylen) == 0 {
-		panic("libsecp256k1 error")
-	}
-	return out
-}
-
-func checkSignature(sig []byte) error {
-	if len(sig) != 65 {
-		return ErrInvalidSignatureLen
-	}
-	if sig[64] >= 4 {
-		return ErrInvalidRecoveryID
-	}
-	return nil
+	return secp256k1.CompressPubkey(x, y)
 }

l2geth/ethclient/ethclient.go

@@ -154,6 +154,13 @@ func (ec *Client) getBlock(ctx context.Context, method string, args ...interface
 		if tx.From != nil {
 			setSenderFromServer(tx.tx, *tx.From, body.Hash)
 		}
+		meta := types.NewTransactionMeta(
+			tx.meta.L1BlockNumber, tx.meta.L1Timestamp,
+			tx.meta.L1MessageSender, tx.meta.QueueOrigin,
+			tx.meta.Index, tx.meta.QueueIndex,
+			tx.meta.RawTransaction,
+		)
+		tx.tx.SetTransactionMeta(meta)
 		txs[i] = tx.tx
 	}
 	return types.NewBlockWithHeader(head).WithBody(txs, uncles), nil
@@ -182,9 +189,30 @@ func (ec *Client) HeaderByNumber(ctx context.Context, number *big.Int) (*types.H
 
 type rpcTransaction struct {
 	tx   *types.Transaction
+	meta *rpcTransactionMeta
 	txExtraInfo
 }
 
+//go:generate gencodec -type rpcTransactionMeta -field-override rpcTransactionMetaMarshaling -out gen_rpc_tx_meta_json.go
+
+type rpcTransactionMeta struct {
+	L1BlockNumber   *big.Int          `json:"l1BlockNumber"`
+	L1Timestamp     uint64            `json:"l1Timestamp"`
+	L1MessageSender *common.Address   `json:"l1MessageSender"`
+	QueueOrigin     types.QueueOrigin `json:"queueOrigin"`
+	Index           *uint64           `json:"index"`
+	QueueIndex      *uint64           `json:"queueIndex"`
+	RawTransaction  []byte            `json:"rawTransaction"`
+}
+
+type rpcTransactionMetaMarshaling struct {
+	L1BlockNumber  *hexutil.Big
+	L1Timestamp    hexutil.Uint64
+	Index          *hexutil.Uint64
+	QueueIndex     *hexutil.Uint64
+	RawTransaction hexutil.Bytes
+}
+
 type txExtraInfo struct {
 	BlockNumber *string         `json:"blockNumber,omitempty"`
 	BlockHash   *common.Hash    `json:"blockHash,omitempty"`
@@ -195,6 +223,9 @@ func (tx *rpcTransaction) UnmarshalJSON(msg []byte) error {
 	if err := json.Unmarshal(msg, &tx.tx); err != nil {
 		return err
 	}
+	if err := json.Unmarshal(msg, &tx.meta); err != nil {
+		return err
+	}
 	return json.Unmarshal(msg, &tx.txExtraInfo)
 }
 

l2geth/ethclient/gen_rpc_tx_meta_json.go

+// Code generated by github.com/fjl/gencodec. DO NOT EDIT.
+
+package ethclient
+
+import (
+	"encoding/json"
+	"math/big"
+
+	"github.com/ethereum-optimism/optimism/l2geth/common"
+	"github.com/ethereum-optimism/optimism/l2geth/common/hexutil"
+	"github.com/ethereum-optimism/optimism/l2geth/core/types"
+)
+
+var _ = (*rpcTransactionMetaMarshaling)(nil)
+
+// MarshalJSON marshals as JSON.
+func (r rpcTransactionMeta) MarshalJSON() ([]byte, error) {
+	type rpcTransactionMeta struct {
+		L1BlockNumber   *hexutil.Big      `json:"l1BlockNumber"`
+		L1Timestamp     hexutil.Uint64    `json:"l1Timestamp"`
+		L1MessageSender *common.Address   `json:"l1MessageSender"`
+		QueueOrigin     types.QueueOrigin `json:"queueOrigin"`
+		Index           *hexutil.Uint64   `json:"index"`
+		QueueIndex      *hexutil.Uint64   `json:"queueIndex"`
+		RawTransaction  hexutil.Bytes     `json:"rawTransaction"`
+	}
+	var enc rpcTransactionMeta
+	enc.L1BlockNumber = (*hexutil.Big)(r.L1BlockNumber)
+	enc.L1Timestamp = hexutil.Uint64(r.L1Timestamp)
+	enc.L1MessageSender = r.L1MessageSender
+	enc.QueueOrigin = r.QueueOrigin
+	enc.Index = (*hexutil.Uint64)(r.Index)
+	enc.QueueIndex = (*hexutil.Uint64)(r.QueueIndex)
+	enc.RawTransaction = r.RawTransaction
+	return json.Marshal(&enc)
+}
+
+// UnmarshalJSON unmarshals from JSON.
+func (r *rpcTransactionMeta) UnmarshalJSON(input []byte) error {
+	type rpcTransactionMeta struct {
+		L1BlockNumber   *hexutil.Big       `json:"l1BlockNumber"`
+		L1Timestamp     *hexutil.Uint64    `json:"l1Timestamp"`
+		L1MessageSender *common.Address    `json:"l1MessageSender"`
+		QueueOrigin     *types.QueueOrigin `json:"queueOrigin"`
+		Index           *hexutil.Uint64    `json:"index"`
+		QueueIndex      *hexutil.Uint64    `json:"queueIndex"`
+		RawTransaction  *hexutil.Bytes     `json:"rawTransaction"`
+	}
+	var dec rpcTransactionMeta
+	if err := json.Unmarshal(input, &dec); err != nil {
+		return err
+	}
+	if dec.L1BlockNumber != nil {
+		r.L1BlockNumber = (*big.Int)(dec.L1BlockNumber)
+	}
+	if dec.L1Timestamp != nil {
+		r.L1Timestamp = uint64(*dec.L1Timestamp)
+	}
+	if dec.L1MessageSender != nil {
+		r.L1MessageSender = dec.L1MessageSender
+	}
+	if dec.QueueOrigin != nil {
+		r.QueueOrigin = *dec.QueueOrigin
+	}
+	if dec.Index != nil {
+		r.Index = (*uint64)(dec.Index)
+	}
+	if dec.QueueIndex != nil {
+		r.QueueIndex = (*uint64)(dec.QueueIndex)
+	}
+	if dec.RawTransaction != nil {
+		r.RawTransaction = *dec.RawTransaction
+	}
+	return nil
+}

l2geth/go.mod

@@ -3,69 +3,59 @@ module github.com/ethereum-optimism/optimism/l2geth
 go 1.15
 
 require (
-	github.com/Azure/azure-pipeline-go v0.2.2 // indirect
 	github.com/Azure/azure-storage-blob-go v0.7.0
-	github.com/Azure/go-autorest/autorest/adal v0.8.0 // indirect
-	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
-	github.com/VictoriaMetrics/fastcache v1.5.7
+	github.com/VictoriaMetrics/fastcache v1.6.0
 	github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847
 	github.com/aws/aws-sdk-go v1.42.6
-	github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6
+	github.com/btcsuite/btcd v0.22.0-beta
 	github.com/cespare/cp v0.1.0
-	github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9
+	github.com/cloudflare/cloudflare-go v0.14.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea
 	github.com/docker/docker v20.10.10+incompatible
-	github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c
+	github.com/edsrzf/mmap-go v1.0.0
 	github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa
-	github.com/fatih/color v1.3.0
-	github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc
+	github.com/ethereum/go-ethereum v1.10.12
+	github.com/fatih/color v1.7.0
+	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff
-	github.com/go-ole/go-ole v1.2.1 // indirect
 	github.com/go-resty/resty/v2 v2.4.0
 	github.com/go-stack/stack v1.8.0
-	github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c
-	github.com/golang/snappy v0.0.1
-	github.com/google/go-cmp v0.5.1 // indirect
-	github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989
-	github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277
-	github.com/hashicorp/golang-lru v0.5.4
-	github.com/huin/goupnp v1.0.0
-	github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883
+	github.com/golang/protobuf v1.4.3
+	github.com/golang/snappy v0.0.4
+	github.com/gorilla/websocket v1.4.2
+	github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29
+	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
+	github.com/huin/goupnp v1.0.2
+	github.com/influxdata/influxdb v1.8.3
 	github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458
 	github.com/jarcoal/httpmock v1.0.8
-	github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21
-	github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356
-	github.com/kr/pretty v0.1.0 // indirect
-	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/mattn/go-colorable v0.1.0
-	github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035
-	github.com/naoina/go-stringutil v0.1.0 // indirect
+	github.com/julienschmidt/httprouter v1.2.0
+	github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559
+	github.com/mattn/go-colorable v0.1.8
+	github.com/mattn/go-isatty v0.0.12
 	github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416
-	github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c
+	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222
 	github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7
-	github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150
+	github.com/prometheus/tsdb v0.7.1
 	github.com/rjeczalik/notify v0.9.1
 	github.com/robertkrimen/otto v0.0.0-20191219234010-c382bd3c16ff
-	github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00
-	github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 // indirect
+	github.com/rs/cors v1.7.0
 	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4
 	github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570
 	github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect
-	github.com/stretchr/testify v1.4.0
-	github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d
+	github.com/stretchr/testify v1.7.0
+	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef
 	github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
-	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
-	golang.org/x/sys v0.0.0-20210423082822-04245dca01da
+	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912
 	golang.org/x/text v0.3.6
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce
-	gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9
+	gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gopkg.in/urfave/cli.v1 v1.20.0
 	gotest.tools/v3 v3.0.3 // indirect

l2geth/rpc/constants_unix.go

@@ -21,7 +21,7 @@ package rpc
 /*
 #include <sys/un.h>
 
-int max_socket_path_size() {
+int max_socket_path_size2() {
 struct sockaddr_un s;
 return sizeof(s.sun_path);
 }
@@ -29,5 +29,5 @@ return sizeof(s.sun_path);
 import "C"
 
 var (
-	max_path_size = C.max_socket_path_size()
+	max_path_size = C.max_socket_path_size2()
 )

ops/Makefile

+BATCH_SUBMITTER ?= docker-compose.ts-batch-submitter.yml
+DOCKER_COMPOSE_CMD := docker-compose \
+	-f docker-compose.yml \
+	-f $(BATCH_SUBMITTER)
+
 build:
 	DOCKER_BUILDKIT=1 \
-	docker-compose \
-		-f docker-compose.yml build
+	$(DOCKER_COMPOSE_CMD) \
+		build
 .PHONY: build
 
 up: down
 	DOCKER_BUILDKIT=1 \
-	docker-compose \
-		-f docker-compose.yml \
+	$(DOCKER_COMPOSE_CMD) \
 		up --build --detach
 .PHONY: up
 
 down:
-	docker-compose \
-		-f docker-compose.yml \
+	$(DOCKER_COMPOSE_CMD) \
 		down
 .PHONY: down
 
 ps:
-	docker-compose \
-		-f docker-compose.yml \
+	$(DOCKER_COMPOSE_CMD) \
 		ps
 .PHONY: ps
 
 up-metrics: down-metrics
 	DOCKER_BUILDKIT=1 \
-	docker-compose \
-		-f docker-compose.yml \
+	$(DOCKER_COMPOSE_CMD) \
 		-f docker-compose-metrics.yml \
 		up --build --detach
 .PHONY: up-metrics
 
 down-metrics:
-	docker-compose \
-		-f docker-compose.yml \
+	$(DOCKER_COMPOSE_CMD) \
 		-f docker-compose-metrics.yml \
 		down
 .PHONY: down-metrics
 
 ps-metrics:
-	docker-compose \
-		-f docker-compose.yml \
+	$(DOCKER_COMPOSE_CMD) \
 		-f docker-compose-metrics.yml \
 		ps
 .PHONY: ps

ops/README.md

@@ -22,9 +22,11 @@ Supplementing the base configuration is an additional metric enabling file, `doc
 
 Also available for testing is the `rpc-proxy` service in the `docker-compose-rpc-proxy.yml` file. It can be used to restrict what RPC methods are allowed to the Sequencer.
 
-The base stack can be started and stopped with a command like this (there is no need to specify the default docker-compose.yml)
+The base stack can be started and stopped with a command like this:
 ```
 docker-compose \
+    -f docker-compose.yml \
+    -f docker-compose.ts-batch-submitter.yml \
     up --build --detach
 ```
 
@@ -34,13 +36,17 @@ To start the stack with monitoring enabled, just add the metric composition file
 ```
 docker-compose \
     -f docker-compose.yml \
+    -f docker-compose.ts-batch-submitter.yml \
     -f docker-compose-metrics.yml \
     up --build --detach
 ```
 
 Optionally, run a verifier along the rest of the stack. Run a replica with the same command by switching the service name!
 ```
-docker-compose up --scale \
+docker-compose 
+    -f docker-compose.yml \
+    -f docker-compose.ts-batch-submitter.yml \
+    up --scale \
     verifier=1 \
     --build --detach
 ```
@@ -52,6 +58,24 @@ A Makefile has been provided for convience. The following targets are available.
 - make up-metrics
 - make down-metrics
 
+## Using the Go Batch Submitter
+
+The existing Typescript batch submitter is in the process of being reimplemented
+in Go. During this transition, the user is required to specify which batch
+submitter to use with docker-compose.
+
+The commands above all use the Typescript batch submitter, by specifying
+`-f docker-compose.ts-batch-submitter.yml`. This can be swapped out for the go
+batch submitter by supplying `-f docker-compose.go-batch-submitter.yml` instead.
+
+Additionally, the `make` targets assume the use of the Typescript batch
+submitter. This can be overridden by setting the `BATCH_SUBMITTER` environment
+variable, e.g. `BATCH_SUBMITTER=docker-compose.go-batch-submitter.yml make up`.
+
+Once the transition is complete, specifying the desired batch submitter will be
+obsolete, and the Go batch submitter will be selected by default from the
+`docker-compose.yml` file and `Makefile`.
+
 ## Cross domain communication
 
 By default, the `message-relayer` service is turned off. This means that

ops/docker-compose.go-batch-submitter.yml

+services:
+  batch_submitter:
+    depends_on:
+      - l1_chain
+      - deployer
+      - l2geth
+    build:
+      context: ..
+      dockerfile: ./ops/docker/Dockerfile.batch-submitter-service
+    entrypoint: ./batch-submitter.sh
+    env_file:
+      - ./envs/batch-submitter.env
+    environment:
+        L1_ETH_RPC: http://l1_chain:8545
+        L2_ETH_RPC: http://l2geth:8545
+        URL: http://deployer:8081/addresses.json
+        BATCH_SUBMITTER_SEQUENCER_PRIVATE_KEY: "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d"
+        BATCH_SUBMITTER_PROPOSER_PRIVATE_KEY: "0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a"

ops/docker-compose.ts-batch-submitter.yml

+services:
+  batch_submitter:
+    depends_on:
+      - l1_chain
+      - deployer
+      - l2geth
+    build:
+      context: ..
+      dockerfile: ./ops/docker/Dockerfile.packages
+      target: batch-submitter
+    entrypoint: ./batches.sh
+    env_file:
+      - ./envs/batches.env
+    environment:
+        L1_NODE_WEB3_URL: http://l1_chain:8545
+        L2_NODE_WEB3_URL: http://l2geth:8545
+        URL: http://deployer:8081/addresses.json
+        SEQUENCER_PRIVATE_KEY: "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d"
+        PROPOSER_PRIVATE_KEY: "0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a"

ops/docker-compose.yml

@@ -122,25 +122,6 @@ services:
         POLLING_INTERVAL: 500
         GET_LOGS_INTERVAL: 500
 
-  batch_submitter:
-    depends_on:
-      - l1_chain
-      - deployer
-      - l2geth
-    build:
-      context: ..
-      dockerfile: ./ops/docker/Dockerfile.packages
-      target: batch-submitter
-    entrypoint: ./batches.sh
-    env_file:
-      - ./envs/batches.env
-    environment:
-        L1_NODE_WEB3_URL: http://l1_chain:8545
-        L2_NODE_WEB3_URL: http://l2geth:8545
-        URL: http://deployer:8081/addresses.json
-        SEQUENCER_PRIVATE_KEY: "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d"
-        PROPOSER_PRIVATE_KEY: "0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a"
-
   verifier:
     depends_on:
       - l1_chain

ops/docker/Dockerfile.batch-submitter-service

+FROM golang:1.15-alpine3.13 as builder
+
+RUN apk add --no-cache make gcc musl-dev linux-headers git jq bash
+
+COPY ./l2geth /l2geth
+COPY ./go/batch-submitter/go.mod ./go/batch-submitter/go.sum /go/batch-submitter/
+WORKDIR /go/batch-submitter
+RUN go mod graph | grep -v l2geth | awk '{if ($1 !~ "@") print $2}' | xargs -n 1 go get
+COPY ./go/batch-submitter/ ./
+RUN make
+
+FROM alpine:3.13
+
+RUN apk add --no-cache ca-certificates jq curl
+COPY --from=builder /go/batch-submitter/batch-submitter /usr/local/bin/
+
+COPY ./ops/scripts/batch-submitter.sh .
+ENTRYPOINT ["batch-submitter"]

ops/docker/Dockerfile.geth

@@ -3,8 +3,11 @@ FROM golang:1.15-alpine3.13 as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers git
 
-ADD ./l2geth /go-ethereum
-RUN cd /go-ethereum && make geth
+COPY ./l2geth/go.mod ./l2geth/go.sum /go-ethereum/
+WORKDIR /go-ethereum
+RUN go mod download
+COPY ./l2geth ./
+RUN make geth
 
 # Pull Geth into a second stage deploy alpine container
 FROM alpine:3.13

ops/envs/batch-submitter.env

+BUILD_ENV=development
+ETH_NETWORK_NAME=clique
+
+LOG_LEVEL=debug
+BATCH_SUBMITTER_LOG_LEVEL=debug
+BATCH_SUBMITTER_MAX_L1_TX_SIZE=90000
+BATCH_SUBMITTER_MAX_BATCH_SUBMISSION_TIME=0
+BATCH_SUBMITTER_POLL_INTERVAL=500ms
+BATCH_SUBMITTER_NUM_CONFIRMATIONS=1
+BATCH_SUBMITTER_RESUBMISSION_TIMEOUT=1s
+BATCH_SUBMITTER_FINALITY_CONFIRMATIONS=0
+BATCH_SUBMITTER_RUN_TX_BATCH_SUBMITTER=true
+BATCH_SUBMITTER_RUN_STATE_BATCH_SUBMITTER=true
+BATCH_SUBMITTER_SAFE_MINIMUM_ETHER_BALANCE=0
+BATCH_SUBMITTER_CLEAR_PENDING_TXS=false

ops/scripts/batch-submitter.sh

+#!/bin/sh
+
+set -e
+
+RETRIES=${RETRIES:-40}
+
+if [[ ! -z "$URL" ]]; then
+    # get the addrs from the URL provided
+    ADDRESSES=$(curl --fail --show-error --silent --retry-connrefused --retry $RETRIES --retry-delay 5 $URL)
+    # set the env
+    export CTC_ADDRESS=$(echo $ADDRESSES | jq -r '.CanonicalTransactionChain')
+    export SCC_ADDRESS=$(echo $ADDRESSES | jq -r '.StateCommitmentChain')
+fi
+
+
+# waits for l2geth to be up
+curl --fail \
+    --show-error \
+    --silent \
+    --retry-connrefused \
+    --retry $RETRIES \
+    --retry-delay 1 \
+    --output /dev/null \
+    $L2_ETH_RPC
+
+# go
+exec batch-submitter "$@"