Merge branch 'develop' into aj/disable-hive

.circleci/config.yml

@@ -880,7 +880,7 @@ jobs:
           patterns: indexer
       - run:
           name: Lint
-          command: golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint --timeout 2m -e "errors.As" -e "errors.Is" ./...
+          command: golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint --timeout 4m -e "errors.As" -e "errors.Is" ./...
           working_directory: indexer
       - run:
           name: install geth

indexer/database/blocks.go

@@ -18,8 +18,8 @@ import (
  */
 
 type BlockHeader struct {
-	Hash       common.Hash `gorm:"primaryKey;serializer:json"`
-	ParentHash common.Hash `gorm:"serializer:json"`
+	Hash       common.Hash `gorm:"primaryKey;serializer:bytes"`
+	ParentHash common.Hash `gorm:"serializer:bytes"`
 	Number     U256
 	Timestamp  uint64
 
@@ -50,14 +50,14 @@ type LegacyStateBatch struct {
 	// violating the primary key constraint.
 	Index uint64 `gorm:"primaryKey;default:0"`
 
-	Root                common.Hash `gorm:"serializer:json"`
+	Root                common.Hash `gorm:"serializer:bytes"`
 	Size                uint64
 	PrevTotal           uint64
 	L1ContractEventGUID uuid.UUID
 }
 
 type OutputProposal struct {
-	OutputRoot    common.Hash `gorm:"primaryKey;serializer:json"`
+	OutputRoot    common.Hash `gorm:"primaryKey;serializer:bytes"`
 	L2OutputIndex U256
 	L2BlockNumber U256
 

indexer/database/bridge_messages.go

@@ -16,7 +16,7 @@ import (
  */
 
 type BridgeMessage struct {
-	MessageHash common.Hash `gorm:"primaryKey;serializer:json"`
+	MessageHash common.Hash `gorm:"primaryKey;serializer:bytes"`
 	Nonce       U256
 
 	SentMessageEventGUID    uuid.UUID
@@ -28,12 +28,12 @@ type BridgeMessage struct {
 
 type L1BridgeMessage struct {
 	BridgeMessage         `gorm:"embedded"`
-	TransactionSourceHash common.Hash `gorm:"serializer:json"`
+	TransactionSourceHash common.Hash `gorm:"serializer:bytes"`
 }
 
 type L2BridgeMessage struct {
 	BridgeMessage             `gorm:"embedded"`
-	TransactionWithdrawalHash common.Hash `gorm:"serializer:json"`
+	TransactionWithdrawalHash common.Hash `gorm:"serializer:bytes"`
 }
 
 type BridgeMessagesView interface {

indexer/database/bridge_transactions.go

@@ -8,7 +8,6 @@ import (
 	"gorm.io/gorm"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
 )
 
 /**
@@ -16,16 +15,16 @@ import (
  */
 
 type Transaction struct {
-	FromAddress common.Address `gorm:"serializer:json"`
-	ToAddress   common.Address `gorm:"serializer:json"`
+	FromAddress common.Address `gorm:"serializer:bytes"`
+	ToAddress   common.Address `gorm:"serializer:bytes"`
 	Amount      U256
-	Data        hexutil.Bytes `gorm:"serializer:json"`
+	Data        Bytes `gorm:"serializer:bytes"`
 	Timestamp   uint64
 }
 
 type L1TransactionDeposit struct {
-	SourceHash           common.Hash `gorm:"serializer:json;primaryKey"`
-	L2TransactionHash    common.Hash `gorm:"serializer:json"`
+	SourceHash           common.Hash `gorm:"serializer:bytes;primaryKey"`
+	L2TransactionHash    common.Hash `gorm:"serializer:bytes"`
 	InitiatedL1EventGUID uuid.UUID
 
 	Tx       Transaction `gorm:"embedded"`
@@ -33,7 +32,7 @@ type L1TransactionDeposit struct {
 }
 
 type L2TransactionWithdrawal struct {
-	WithdrawalHash       common.Hash `gorm:"serializer:json;primaryKey"`
+	WithdrawalHash       common.Hash `gorm:"serializer:bytes;primaryKey"`
 	Nonce                U256
 	InitiatedL2EventGUID uuid.UUID
 

indexer/database/bridge_transfers.go

@@ -19,12 +19,12 @@ var (
  */
 
 type TokenPair struct {
-	LocalTokenAddress  common.Address `gorm:"serializer:json"`
-	RemoteTokenAddress common.Address `gorm:"serializer:json"`
+	LocalTokenAddress  common.Address `gorm:"serializer:bytes"`
+	RemoteTokenAddress common.Address `gorm:"serializer:bytes"`
 }
 
 type BridgeTransfer struct {
-	CrossDomainMessageHash *common.Hash `gorm:"serializer:json"`
+	CrossDomainMessageHash *common.Hash `gorm:"serializer:bytes"`
 
 	Tx        Transaction `gorm:"embedded"`
 	TokenPair TokenPair   `gorm:"embedded"`
@@ -32,27 +32,27 @@ type BridgeTransfer struct {
 
 type L1BridgeDeposit struct {
 	BridgeTransfer        `gorm:"embedded"`
-	TransactionSourceHash common.Hash `gorm:"primaryKey;serializer:json"`
+	TransactionSourceHash common.Hash `gorm:"primaryKey;serializer:bytes"`
 }
 
 type L1BridgeDepositWithTransactionHashes struct {
 	L1BridgeDeposit L1BridgeDeposit `gorm:"embedded"`
 
-	L1TransactionHash common.Hash `gorm:"serializer:json"`
-	L2TransactionHash common.Hash `gorm:"serializer:json"`
+	L1TransactionHash common.Hash `gorm:"serializer:bytes"`
+	L2TransactionHash common.Hash `gorm:"serializer:bytes"`
 }
 
 type L2BridgeWithdrawal struct {
 	BridgeTransfer            `gorm:"embedded"`
-	TransactionWithdrawalHash common.Hash `gorm:"primaryKey;serializer:json"`
+	TransactionWithdrawalHash common.Hash `gorm:"primaryKey;serializer:bytes"`
 }
 
 type L2BridgeWithdrawalWithTransactionHashes struct {
 	L2BridgeWithdrawal L2BridgeWithdrawal `gorm:"embedded"`
-	L2TransactionHash  common.Hash        `gorm:"serializer:json"`
+	L2TransactionHash  common.Hash        `gorm:"serializer:bytes"`
 
-	ProvenL1TransactionHash    common.Hash `gorm:"serializer:json"`
-	FinalizedL1TransactionHash common.Hash `gorm:"serializer:json"`
+	ProvenL1TransactionHash    common.Hash `gorm:"serializer:bytes"`
+	FinalizedL1TransactionHash common.Hash `gorm:"serializer:bytes"`
 }
 
 type BridgeTransfersView interface {

indexer/database/contract_events.go

@@ -21,12 +21,12 @@ type ContractEvent struct {
 	GUID uuid.UUID `gorm:"primaryKey"`
 
 	// Some useful derived fields
-	BlockHash       common.Hash    `gorm:"serializer:json"`
-	ContractAddress common.Address `gorm:"serializer:json"`
-	TransactionHash common.Hash    `gorm:"serializer:json"`
+	BlockHash       common.Hash    `gorm:"serializer:bytes"`
+	ContractAddress common.Address `gorm:"serializer:bytes"`
+	TransactionHash common.Hash    `gorm:"serializer:bytes"`
 	LogIndex        uint64
 
-	EventSignature common.Hash `gorm:"serializer:json"`
+	EventSignature common.Hash `gorm:"serializer:bytes"`
 	Timestamp      uint64
 
 	// NOTE: NOT ALL THE DERIVED FIELDS ON `types.Log` ARE

indexer/database/db.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 
 	"github.com/ethereum-optimism/optimism/indexer/config"
+	_ "github.com/ethereum-optimism/optimism/indexer/database/serializers"
+
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"

indexer/database/serializers/bytes.go

+package serializers
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"gorm.io/gorm/schema"
+)
+
+type BytesSerializer struct{}
+type BytesInterface interface{ Bytes() []byte }
+type SetBytesInterface interface{ SetBytes([]byte) }
+
+func init() {
+	schema.RegisterSerializer("bytes", BytesSerializer{})
+}
+
+func (BytesSerializer) Scan(ctx context.Context, field *schema.Field, dst reflect.Value, dbValue interface{}) error {
+	if dbValue == nil {
+		return nil
+	}
+
+	hexStr, ok := dbValue.(string)
+	if !ok {
+		return fmt.Errorf("expected hex string as the database value: %T", dbValue)
+	}
+
+	b, err := hexutil.Decode(hexStr)
+	if err != nil {
+		return fmt.Errorf("failed to decode database value: %w", err)
+	}
+
+	fieldValue := reflect.New(field.FieldType)
+	fieldInterface := fieldValue.Interface()
+
+	// Detect if we're deserializing into a pointer. If so, we'll need to
+	// also allocate memory to where the allocated pointer should point to
+	if field.FieldType.Kind() == reflect.Pointer {
+		nestedField := fieldValue.Elem()
+		if nestedField.Elem().Kind() == reflect.Pointer {
+			return fmt.Errorf("double pointers are the max depth supported: %T", fieldValue)
+		}
+
+		// We'll want to call `SetBytes` on the pointer to
+		// the allocated memory and not the double pointer
+		nestedField.Set(reflect.New(field.FieldType.Elem()))
+		fieldInterface = nestedField.Interface()
+	}
+
+	fieldSetBytes, ok := fieldInterface.(SetBytesInterface)
+	if !ok {
+		return fmt.Errorf("field does not satisfy the `SetBytes([]byte)` interface: %T", fieldInterface)
+	}
+
+	fieldSetBytes.SetBytes(b)
+	field.ReflectValueOf(ctx, dst).Set(fieldValue.Elem())
+	return nil
+}
+
+func (BytesSerializer) Value(ctx context.Context, field *schema.Field, dst reflect.Value, fieldValue interface{}) (interface{}, error) {
+	if fieldValue == nil || (field.FieldType.Kind() == reflect.Pointer && reflect.ValueOf(fieldValue).IsNil()) {
+		return nil, nil
+	}
+
+	fieldBytes, ok := fieldValue.(BytesInterface)
+	if !ok {
+		return nil, fmt.Errorf("field does not satisfy the `Bytes() []byte` interface")
+	}
+
+	hexStr := hexutil.Encode(fieldBytes.Bytes())
+	return hexStr, nil
+}

indexer/database/rlp_serializer.go → indexer/database/serializers/rlp.go

-package database
+package serializers
 
 import (
 	"context"
@@ -13,38 +13,28 @@ import (
 
 type RLPSerializer struct{}
 
-type RLPInterface interface {
-	rlp.Encoder
-	rlp.Decoder
-}
-
 func init() {
 	schema.RegisterSerializer("rlp", RLPSerializer{})
 }
 
 func (RLPSerializer) Scan(ctx context.Context, field *schema.Field, dst reflect.Value, dbValue interface{}) error {
-	fieldValue := reflect.New(field.FieldType)
-	if dbValue != nil {
-		var bytes []byte
-		switch v := dbValue.(type) {
-		case []byte:
-			bytes = v
-		case string:
-			b, err := hexutil.Decode(v)
-			if err != nil {
-				return err
-			}
-			bytes = b
-		default:
-			return fmt.Errorf("unrecognized RLP bytes: %#v", dbValue)
-		}
+	if dbValue == nil {
+		return nil
+	}
+
+	hexStr, ok := dbValue.(string)
+	if !ok {
+		return fmt.Errorf("expected hex string as the database value: %T", dbValue)
+	}
 
-		if len(bytes) > 0 {
-			err := rlp.DecodeBytes(bytes, fieldValue.Interface())
-			if err != nil {
-				return err
-			}
-		}
+	b, err := hexutil.Decode(hexStr)
+	if err != nil {
+		return fmt.Errorf("failed to decode database value: %w", err)
+	}
+
+	fieldValue := reflect.New(field.FieldType)
+	if err := rlp.DecodeBytes(b, fieldValue.Interface()); err != nil {
+		return fmt.Errorf("failed to decode rlp bytes: %w", err)
 	}
 
 	field.ReflectValueOf(ctx, dst).Set(fieldValue.Elem())
@@ -52,18 +42,15 @@ func (RLPSerializer) Scan(ctx context.Context, field *schema.Field, dst reflect.
 }
 
 func (RLPSerializer) Value(ctx context.Context, field *schema.Field, dst reflect.Value, fieldValue interface{}) (interface{}, error) {
-	// Even though rlp.Encode takes an interface and will error out if the passed interface does not
-	// satisfy the interface, we check here since we also want to make sure this type satisfies the
-	// rlp.Decoder interface as well
-	i := reflect.TypeOf(new(RLPInterface)).Elem()
-	if !reflect.TypeOf(fieldValue).Implements(i) {
-		return nil, fmt.Errorf("%T does not satisfy RLP Encoder & Decoder interface", fieldValue)
+	if fieldValue == nil || (field.FieldType.Kind() == reflect.Pointer && reflect.ValueOf(fieldValue).IsNil()) {
+		return nil, nil
 	}
 
 	rlpBytes, err := rlp.EncodeToBytes(fieldValue)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to encode rlp bytes: %w", err)
 	}
 
-	return hexutil.Bytes(rlpBytes).MarshalText()
+	hexStr := hexutil.Encode(rlpBytes)
+	return hexStr, nil
 }

indexer/database/types.go

@@ -93,3 +93,12 @@ func (h *RLPHeader) Header() *types.Header {
 func (h *RLPHeader) Hash() common.Hash {
 	return h.Header().Hash()
 }
+
+type Bytes []byte
+
+func (b Bytes) Bytes() []byte {
+	return b[:]
+}
+func (b *Bytes) SetBytes(bytes []byte) {
+	*b = bytes
+}

op-chain-ops/cmd/registry-data/main.go

+package main
+
+import (
+	"bytes"
+	"compress/flate"
+	"compress/gzip"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	opservice "github.com/ethereum-optimism/optimism/op-service"
+	"github.com/ethereum-optimism/optimism/op-service/jsonutil"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/mattn/go-isatty"
+	"github.com/urfave/cli/v2"
+)
+
+const EnvPrefix = "OP_CHAIN_OPS_REGISTRY_DATA"
+
+var (
+	L2GenesisFlag = &cli.PathFlag{
+		Name:    "l2-genesis",
+		Value:   "genesis.json",
+		Usage:   "Path to genesis json (go-ethereum format)",
+		EnvVars: opservice.PrefixEnvVar(EnvPrefix, "L2_GENESIS"),
+	}
+	L2GenesisHeaderFlag = &cli.PathFlag{
+		Name:    "l2-genesis-header",
+		Value:   "genesis-header.json",
+		Usage:   "Alternative to l2-genesis flag, if genesis-state is omitted. Path to block header at genesis",
+		EnvVars: opservice.PrefixEnvVar(EnvPrefix, "L2_GENESIS_HEADER"),
+	}
+	BytecodesDirFlag = &cli.PathFlag{
+		Name:    "bytecodes-dir",
+		Value:   "superchain-registry/superchain/bytecodes",
+		Usage:   "Collection of gzipped L2 bytecodes",
+		EnvVars: opservice.PrefixEnvVar(EnvPrefix, "BYTECODES_DIR"),
+	}
+	OutputFlag = &cli.PathFlag{
+		Name:    "output",
+		Value:   "out.json.gz",
+		Usage:   "output gzipped JSON file path to write to",
+		EnvVars: opservice.PrefixEnvVar(EnvPrefix, "OUTPUT"),
+	}
+)
+
+func main() {
+	log.Root().SetHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd()))))
+
+	app := &cli.App{
+		Name:  "registry-genesis",
+		Usage: "Prepare superchain-registry genesis data files based on full genesis dump",
+		Flags: []cli.Flag{
+			L2GenesisFlag,
+			L2GenesisHeaderFlag,
+			BytecodesDirFlag,
+			OutputFlag,
+		},
+		Action: entrypoint,
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Crit("error while generating registry data", "err", err)
+	}
+}
+
+type GenesisAccount struct {
+	CodeHash common.Hash                                          `json:"codeHash,omitempty"`
+	Storage  jsonutil.LazySortedJsonMap[common.Hash, common.Hash] `json:"storage,omitempty"`
+	Balance  *hexutil.Big                                         `json:"balance,omitempty"`
+	Nonce    uint64                                               `json:"nonce,omitempty"`
+}
+
+type Genesis struct {
+	Nonce      uint64         `json:"nonce"`
+	Timestamp  uint64         `json:"timestamp"`
+	ExtraData  []byte         `json:"extraData"`
+	GasLimit   uint64         `json:"gasLimit"`
+	Difficulty *hexutil.Big   `json:"difficulty"`
+	Mixhash    common.Hash    `json:"mixHash"`
+	Coinbase   common.Address `json:"coinbase"`
+	Number     uint64         `json:"number"`
+	GasUsed    uint64         `json:"gasUsed"`
+	ParentHash common.Hash    `json:"parentHash"`
+	BaseFee    *hexutil.Big   `json:"baseFeePerGas"`
+
+	Alloc jsonutil.LazySortedJsonMap[common.Address, GenesisAccount] `json:"alloc"`
+	// For genesis definitions without full state (OP-Mainnet, OP-Goerli)
+	StateHash *common.Hash `json:"stateHash,omitempty"`
+}
+
+func loadJSON[X any](inputPath string) (*X, error) {
+	if inputPath == "" {
+		return nil, errors.New("no path specified")
+	}
+	f, err := os.OpenFile(inputPath, os.O_RDONLY, 0)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file %q: %w", inputPath, err)
+	}
+	defer f.Close()
+	var obj X
+	if err := json.NewDecoder(f).Decode(&obj); err != nil {
+		return nil, fmt.Errorf("failed to decode file %q: %w", inputPath, err)
+	}
+	return &obj, nil
+}
+
+func writeGzipJSON(outputPath string, value any) error {
+	f, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
+	if err != nil {
+		return fmt.Errorf("failed to open output file: %w", err)
+	}
+	defer f.Close()
+	w, err := gzip.NewWriterLevel(f, flate.BestCompression)
+	if err != nil {
+		return fmt.Errorf("failed to create gzip writer: %w", err)
+	}
+	defer w.Close()
+	enc := json.NewEncoder(w)
+	if err := enc.Encode(value); err != nil {
+		return fmt.Errorf("failed to encode to JSON: %w", err)
+	}
+	return nil
+}
+
+func entrypoint(ctx *cli.Context) error {
+	genesisPath := ctx.Path(L2GenesisFlag.Name)
+	if genesisPath == "" {
+		// When the genesis-state is too large, or not meant to be available, only the header data is made available.
+		// This allows the user to verify the header-chain starting from genesis, and state-sync the latest state,
+		// skipping the historical state.
+		// Archive nodes that depend on this historical state should instantiate the chain from a full genesis dump
+		// with allocation data, or datadir.
+		genesisHeaderPath := ctx.Path(L2GenesisHeaderFlag.Name)
+		genesisHeader, err := loadJSON[types.Header](genesisHeaderPath)
+		if err != nil {
+			return fmt.Errorf("genesis-header %q failed to load: %w", genesisHeaderPath, err)
+		}
+		if genesisHeader.TxHash != types.EmptyTxsHash {
+			return errors.New("genesis-header based genesis must have no transactions")
+		}
+		if genesisHeader.ReceiptHash != types.EmptyReceiptsHash {
+			return errors.New("genesis-header based genesis must have no receipts")
+		}
+		if genesisHeader.UncleHash != types.EmptyUncleHash {
+			return errors.New("genesis-header based genesis must have no uncle hashes")
+		}
+		if genesisHeader.WithdrawalsHash != nil && *genesisHeader.WithdrawalsHash != types.EmptyWithdrawalsHash {
+			return errors.New("genesis-header based genesis must have no withdrawals")
+		}
+		out := Genesis{
+			Nonce:      genesisHeader.Nonce.Uint64(),
+			Timestamp:  genesisHeader.Time,
+			ExtraData:  genesisHeader.Extra,
+			GasLimit:   genesisHeader.GasLimit,
+			Difficulty: (*hexutil.Big)(genesisHeader.Difficulty),
+			Mixhash:    genesisHeader.MixDigest,
+			Coinbase:   genesisHeader.Coinbase,
+			Number:     genesisHeader.Number.Uint64(),
+			GasUsed:    genesisHeader.GasUsed,
+			ParentHash: genesisHeader.ParentHash,
+			BaseFee:    (*hexutil.Big)(genesisHeader.BaseFee),
+			Alloc:      make(jsonutil.LazySortedJsonMap[common.Address, GenesisAccount]),
+			StateHash:  &genesisHeader.Root,
+		}
+		if err := writeGzipJSON(ctx.Path(OutputFlag.Name), out); err != nil {
+			return fmt.Errorf("failed to write output: %w", err)
+		}
+		return nil
+	}
+
+	genesis, err := loadJSON[core.Genesis](genesisPath)
+	if err != nil {
+		return fmt.Errorf("failed to load L2 genesis: %w", err)
+	}
+
+	// export all contract bytecodes, write them to bytecodes collection
+	bytecodesDir := ctx.Path(BytecodesDirFlag.Name)
+	if err := os.MkdirAll(bytecodesDir, 0755); err != nil {
+		return fmt.Errorf("failed to make bytecodes dir: %w", err)
+	}
+	for addr, account := range genesis.Alloc {
+		if len(account.Code) > 0 {
+			codeHash := crypto.Keccak256Hash(account.Code)
+			name := filepath.Join(bytecodesDir, fmt.Sprintf("%s.bin.gz", codeHash))
+			_, err := os.Stat(name)
+			if err != nil {
+				if os.IsNotExist(err) {
+					var buf bytes.Buffer
+					w, err := gzip.NewWriterLevel(&buf, 9)
+					if err != nil {
+						return fmt.Errorf("failed to construct gzip writer for bytecode %s: %w", codeHash, err)
+					}
+					if _, err := w.Write(account.Code); err != nil {
+						return fmt.Errorf("failed to write bytecode %s to gzip writer: %w", codeHash, err)
+					}
+					if err := w.Close(); err != nil {
+						return fmt.Errorf("failed to close gzip writer: %w", err)
+					}
+					// new bytecode
+					if err := os.WriteFile(name, buf.Bytes(), 0755); err != nil {
+						return fmt.Errorf("failed to write bytecode %s of account %s: %w", codeHash, addr, err)
+					}
+				} else {
+					return fmt.Errorf("failed to check for pre-existing bytecode %s for address %s: %w", codeHash, addr, err)
+				}
+			} // else: already exists
+		}
+	}
+
+	// convert into allocation data
+	out := Genesis{
+		Nonce:      genesis.Nonce,
+		Timestamp:  genesis.Timestamp,
+		ExtraData:  genesis.ExtraData,
+		GasLimit:   genesis.GasLimit,
+		Difficulty: (*hexutil.Big)(genesis.Difficulty),
+		Mixhash:    genesis.Mixhash,
+		Coinbase:   genesis.Coinbase,
+		Number:     genesis.Number,
+		GasUsed:    genesis.GasUsed,
+		ParentHash: genesis.ParentHash,
+		BaseFee:    (*hexutil.Big)(genesis.BaseFee),
+		Alloc:      make(jsonutil.LazySortedJsonMap[common.Address, GenesisAccount]),
+	}
+
+	// write genesis, but only reference code by code-hash, and don't encode the L2 predeploys to save space.
+	for addr, account := range genesis.Alloc {
+		var codeHash common.Hash
+		if len(account.Code) > 0 {
+			codeHash = crypto.Keccak256Hash(account.Code)
+		}
+		outAcc := GenesisAccount{
+			CodeHash: codeHash,
+			Nonce:    account.Nonce,
+		}
+		if account.Balance != nil && account.Balance.Cmp(common.Big0) != 0 {
+			outAcc.Balance = (*hexutil.Big)(account.Balance)
+		}
+		if len(account.Storage) > 0 {
+			outAcc.Storage = make(jsonutil.LazySortedJsonMap[common.Hash, common.Hash])
+			for k, v := range account.Storage {
+				outAcc.Storage[k] = v
+			}
+		}
+		out.Alloc[addr] = outAcc
+	}
+
+	// write genesis alloc
+	if err := writeGzipJSON(ctx.Path(OutputFlag.Name), out); err != nil {
+		return fmt.Errorf("failed to write output: %w", err)
+	}
+	return nil
+}

op-e2e/e2eutils/disputegame/helper.go

@@ -15,12 +15,14 @@ import (
 	"github.com/ethereum-optimism/optimism/op-challenger/fault/alphabet"
 	"github.com/ethereum-optimism/optimism/op-challenger/fault/cannon"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/challenger"
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/transactions"
 	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait"
 	"github.com/ethereum-optimism/optimism/op-node/rollup"
 	"github.com/ethereum-optimism/optimism/op-node/testlog"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/stretchr/testify/require"
@@ -106,14 +108,9 @@ func (h *FactoryHelper) StartAlphabetGame(ctx context.Context, claimedAlphabet s
 	extraData := make([]byte, 64)
 	binary.BigEndian.PutUint64(extraData[24:], l2BlockNumber)
 	binary.BigEndian.PutUint64(extraData[56:], l1Head.Uint64())
-	h.opts.NoSend = true
-	tx, err := h.factory.Create(h.opts, alphabetGameType, rootClaim, extraData)
-	h.require.NoError(err, "create fault dispute game (estimate gas)")
-
-	// Now send with increased gas. This provides a buffer if the output oracle search is now more expensive
-	h.opts.NoSend = false
-	h.opts.GasLimit = tx.Gas() * 2
-	tx, err = h.factory.Create(h.opts, alphabetGameType, rootClaim, extraData)
+	tx, err := transactions.PadGasEstimate(h.opts, 2, func(opts *bind.TransactOpts) (*types.Transaction, error) {
+		return h.factory.Create(opts, alphabetGameType, rootClaim, extraData)
+	})
 	h.require.NoError(err, "create fault dispute game")
 	h.opts.GasLimit = 0
 	rcpt, err := wait.ForReceiptOK(ctx, h.client, tx.Hash())
@@ -199,16 +196,10 @@ func (h *FactoryHelper) createCannonGame(ctx context.Context, l2BlockNumber uint
 	extraData := make([]byte, 64)
 	binary.BigEndian.PutUint64(extraData[24:], l2BlockNumber)
 	binary.BigEndian.PutUint64(extraData[56:], l1Head.Uint64())
-	h.opts.NoSend = true
-	tx, err := h.factory.Create(h.opts, cannonGameType, rootClaim, extraData)
-	h.require.NoError(err, "create fault dispute game (estimate gas)")
-
-	// Now send with increased gas. This provides a buffer if the output oracle search is now more expensive
-	h.opts.NoSend = false
-	h.opts.GasLimit = tx.Gas() * 2
-	tx, err = h.factory.Create(h.opts, cannonGameType, rootClaim, extraData)
+	tx, err := transactions.PadGasEstimate(h.opts, 2, func(opts *bind.TransactOpts) (*types.Transaction, error) {
+		return h.factory.Create(opts, cannonGameType, rootClaim, extraData)
+	})
 	h.require.NoError(err, "create fault dispute game")
-	h.opts.GasLimit = 0
 	rcpt, err := wait.ForReceiptOK(ctx, h.client, tx.Hash())
 	h.require.NoError(err, "wait for create fault dispute game receipt to be OK")
 	h.require.Len(rcpt.Logs, 1, "should have emitted a single DisputeGameCreated event")

op-e2e/e2eutils/transactions/gas.go

+package transactions
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+// TxBuilder creates and sends a transaction using the supplied bind.TransactOpts.
+// Returns the created transaction and any error reported.
+type TxBuilder func(opts *bind.TransactOpts) (*types.Transaction, error)
+
+// PadGasEstimate multiplies the gas estimate for a transaction by the specified paddingFactor before sending the
+// actual transaction. Useful for cases where the gas required is variable.
+// The builder will be invoked twice, first with NoSend=true to estimate the gas and the second time with
+// NoSend=false and GasLimit including the requested padding.
+func PadGasEstimate(opts *bind.TransactOpts, paddingFactor float64, builder TxBuilder) (*types.Transaction, error) {
+	// Take a copy of the opts to avoid mutating the original
+	o := *opts
+	o.NoSend = true
+	tx, err := builder(&o)
+	if err != nil {
+		return nil, fmt.Errorf("failed to estimate gas: %w", err)
+	}
+	gas := float64(tx.Gas()) * paddingFactor
+	o.GasLimit = uint64(gas)
+	o.NoSend = false
+	return builder(&o)
+}

op-e2e/tx_helper.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/ethereum-optimism/optimism/op-bindings/bindings"
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/transactions"
 	"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
@@ -29,19 +30,12 @@ func SendDepositTx(t *testing.T, cfg SystemConfig, l1Client *ethclient.Client, l
 	require.Nil(t, err)
 
 	// Finally send TX
-	l1Opts.NoSend = true
-	tx, err := depositContract.DepositTransaction(l1Opts, l2Opts.ToAddr, l2Opts.Value, l2Opts.GasLimit, l2Opts.IsCreation, l2Opts.Data)
-	require.Nil(t, err, "with deposit tx")
-
-	l1Opts.NoSend = false
 	// Add 10% padding for the L1 gas limit because the estimation process can be affected by the 1559 style cost scale
 	// for buying L2 gas in the portal contracts.
-	l1Opts.GasLimit = tx.Gas() + (tx.Gas() / 10)
-
-	// Now resend with gas specified
-	tx, err = depositContract.DepositTransaction(l1Opts, l2Opts.ToAddr, l2Opts.Value, l2Opts.GasLimit, l2Opts.IsCreation, l2Opts.Data)
+	tx, err := transactions.PadGasEstimate(l1Opts, 1.1, func(opts *bind.TransactOpts) (*types.Transaction, error) {
+		return depositContract.DepositTransaction(opts, l2Opts.ToAddr, l2Opts.Value, l2Opts.GasLimit, l2Opts.IsCreation, l2Opts.Data)
+	})
 	require.Nil(t, err, "with deposit tx")
-	l1Opts.GasLimit = 0
 
 	// Wait for transaction on L1
 	receipt, err := waitForTransaction(tx.Hash(), l1Client, 10*time.Duration(cfg.DeployConfig.L1BlockTime)*time.Second)

op-node/flags/p2p_flags.go

@@ -50,7 +50,8 @@ var (
 	// Banning Flag - whether or not we want to act on the scoring
 	Banning = &cli.BoolFlag{
 		Name:     "p2p.ban.peers",
-		Usage:    "Enables peer banning. This should ONLY be enabled once certain peer scoring is working correctly.",
+		Usage:    "Enables peer banning.",
+		Value:    true,
 		Required: false,
 		EnvVars:  p2pEnv("PEER_BANNING"),
 	}

op-service/jsonutil/sortedmap.go

+package jsonutil
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+)
+
+// LazySortedJsonMap provides sorted encoding order for JSON maps.
+// The sorting is lazy: in-memory it's just a map, until it sorts just-in-time when the map is encoded to JSON.
+// Warning: the just-in-time sorting requires a full allocation of the map structure and keys slice during encoding.
+// Sorting order is not enforced when decoding from JSON.
+type LazySortedJsonMap[K comparable, V any] map[K]V
+
+func (m LazySortedJsonMap[K, V]) MarshalJSON() ([]byte, error) {
+	keys := make([]string, 0, len(m))
+	values := make(map[string]V)
+	for k, v := range m {
+		s := fmt.Sprintf("%q", any(k)) // format as quoted string
+		keys = append(keys, s)
+		values[s] = v
+	}
+	sort.Strings(keys)
+	var out []byte
+	out = append(out, '{')
+	for i, k := range keys {
+		out = append(out, k...) // quotes are already included
+		out = append(out, ':')
+		v, err := json.Marshal(values[k])
+		if err != nil {
+			return nil, fmt.Errorf("failed to encode value of %s: %w", k, err)
+		}
+		out = append(out, v...)
+		if i != len(keys)-1 {
+			out = append(out, ',')
+		}
+	}
+	out = append(out, '}')
+	return out, nil
+}
+
+func (m *LazySortedJsonMap[K, V]) UnmarshalJSON(data []byte) error {
+	return json.Unmarshal(data, (*map[K]V)(m))
+}

op-service/jsonutil/sortedmap_test.go

+package jsonutil
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/stretchr/testify/require"
+)
+
+type LazySortedJsonMapTestCase[K comparable, V any] struct {
+	Object LazySortedJsonMap[K, V]
+	Json   string
+}
+
+func (tc *LazySortedJsonMapTestCase[K, V]) Run(t *testing.T) {
+	t.Run("Marshal", func(t *testing.T) {
+		got, err := json.Marshal(tc.Object)
+		require.NoError(t, err)
+		require.Equal(t, tc.Json, string(got), "json output must match")
+	})
+	t.Run("Unmarshal", func(t *testing.T) {
+		var dest LazySortedJsonMap[K, V]
+		err := json.Unmarshal([]byte(tc.Json), &dest)
+		require.NoError(t, err)
+		require.Equal(t, len(tc.Object), len(dest), "lengths match")
+		for k, v := range tc.Object {
+			require.Equal(t, v, dest[k], "values of %q match", k)
+		}
+	})
+}
+
+func TestLazySortedJsonMap(t *testing.T) {
+	testCases := []interface{ Run(t *testing.T) }{
+		&LazySortedJsonMapTestCase[string, int]{Object: LazySortedJsonMap[string, int]{}, Json: `{}`},
+		&LazySortedJsonMapTestCase[string, int]{Object: LazySortedJsonMap[string, int]{"a": 1, "c": 2, "b": 3}, Json: `{"a":1,"b":3,"c":2}`},
+		&LazySortedJsonMapTestCase[common.Address, int]{Object: LazySortedJsonMap[common.Address, int]{
+			common.HexToAddress("0x4100000000000000000000000000000000000000"): 123,
+			common.HexToAddress("0x4200000000000000000000000000000000000000"): 100,
+			common.HexToAddress("0x4200000000000000000000000000000000000001"): 100,
+		},
+			Json: `{"0x4100000000000000000000000000000000000000":123,` +
+				`"0x4200000000000000000000000000000000000000":100,` +
+				`"0x4200000000000000000000000000000000000001":100}`},
+	}
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("case %d", i), tc.Run)
+	}
+}

package.json

@@ -68,7 +68,7 @@
     "markdownlint-cli2": "0.4.0",
     "mkdirp": "^1.0.4",
     "mocha": "^10.2.0",
-    "nx": "16.7.1",
+    "nx": "16.7.2",
     "nyc": "^15.1.0",
     "patch-package": "^8.0.0",
     "prettier": "^2.8.0",

packages/contracts-ts/package.json

@@ -62,7 +62,7 @@
     "tsup": "^7.1.0",
     "typescript": "^5.1.6",
     "vite": "^4.4.6",
-    "vitest": "^0.33.0"
+    "vitest": "^0.34.2"
   },
   "peerDependencies": {
     "@wagmi/core": ">1.0.0",

packages/fee-estimation/package.json

@@ -46,7 +46,7 @@
     "typescript": "^5.1.6",
     "viem": "^1.3.1",
     "vite": "^4.4.6",
-    "vitest": "^0.33.0"
+    "vitest": "^0.34.2"
   },
   "peerDependencies": {
     "viem": "^0.3.30"