Merge pull request #6939 from ethereum-optimism/indexer.bytes.serialization

fix(indexer): replace json gorm serialization

indexer/database/blocks.go

@@ -18,8 +18,8 @@ import (
  */
 
 type BlockHeader struct {
-	Hash       common.Hash `gorm:"primaryKey;serializer:json"`
-	ParentHash common.Hash `gorm:"serializer:json"`
+	Hash       common.Hash `gorm:"primaryKey;serializer:bytes"`
+	ParentHash common.Hash `gorm:"serializer:bytes"`
 	Number     U256
 	Timestamp  uint64
 
@@ -50,14 +50,14 @@ type LegacyStateBatch struct {
 	// violating the primary key constraint.
 	Index uint64 `gorm:"primaryKey;default:0"`
 
-	Root                common.Hash `gorm:"serializer:json"`
+	Root                common.Hash `gorm:"serializer:bytes"`
 	Size                uint64
 	PrevTotal           uint64
 	L1ContractEventGUID uuid.UUID
 }
 
 type OutputProposal struct {
-	OutputRoot    common.Hash `gorm:"primaryKey;serializer:json"`
+	OutputRoot    common.Hash `gorm:"primaryKey;serializer:bytes"`
 	L2OutputIndex U256
 	L2BlockNumber U256
 

indexer/database/bridge_messages.go

@@ -16,7 +16,7 @@ import (
  */
 
 type BridgeMessage struct {
-	MessageHash common.Hash `gorm:"primaryKey;serializer:json"`
+	MessageHash common.Hash `gorm:"primaryKey;serializer:bytes"`
 	Nonce       U256
 
 	SentMessageEventGUID    uuid.UUID
@@ -28,12 +28,12 @@ type BridgeMessage struct {
 
 type L1BridgeMessage struct {
 	BridgeMessage         `gorm:"embedded"`
-	TransactionSourceHash common.Hash `gorm:"serializer:json"`
+	TransactionSourceHash common.Hash `gorm:"serializer:bytes"`
 }
 
 type L2BridgeMessage struct {
 	BridgeMessage             `gorm:"embedded"`
-	TransactionWithdrawalHash common.Hash `gorm:"serializer:json"`
+	TransactionWithdrawalHash common.Hash `gorm:"serializer:bytes"`
 }
 
 type BridgeMessagesView interface {

indexer/database/bridge_transactions.go

@@ -8,7 +8,6 @@ import (
 	"gorm.io/gorm"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
 )
 
 /**
@@ -16,16 +15,16 @@ import (
  */
 
 type Transaction struct {
-	FromAddress common.Address `gorm:"serializer:json"`
-	ToAddress   common.Address `gorm:"serializer:json"`
+	FromAddress common.Address `gorm:"serializer:bytes"`
+	ToAddress   common.Address `gorm:"serializer:bytes"`
 	Amount      U256
-	Data        hexutil.Bytes `gorm:"serializer:json"`
+	Data        Bytes `gorm:"serializer:bytes"`
 	Timestamp   uint64
 }
 
 type L1TransactionDeposit struct {
-	SourceHash           common.Hash `gorm:"serializer:json;primaryKey"`
-	L2TransactionHash    common.Hash `gorm:"serializer:json"`
+	SourceHash           common.Hash `gorm:"serializer:bytes;primaryKey"`
+	L2TransactionHash    common.Hash `gorm:"serializer:bytes"`
 	InitiatedL1EventGUID uuid.UUID
 
 	Tx       Transaction `gorm:"embedded"`
@@ -33,7 +32,7 @@ type L1TransactionDeposit struct {
 }
 
 type L2TransactionWithdrawal struct {
-	WithdrawalHash       common.Hash `gorm:"serializer:json;primaryKey"`
+	WithdrawalHash       common.Hash `gorm:"serializer:bytes;primaryKey"`
 	Nonce                U256
 	InitiatedL2EventGUID uuid.UUID
 

indexer/database/bridge_transfers.go

@@ -19,12 +19,12 @@ var (
  */
 
 type TokenPair struct {
-	LocalTokenAddress  common.Address `gorm:"serializer:json"`
-	RemoteTokenAddress common.Address `gorm:"serializer:json"`
+	LocalTokenAddress  common.Address `gorm:"serializer:bytes"`
+	RemoteTokenAddress common.Address `gorm:"serializer:bytes"`
 }
 
 type BridgeTransfer struct {
-	CrossDomainMessageHash *common.Hash `gorm:"serializer:json"`
+	CrossDomainMessageHash *common.Hash `gorm:"serializer:bytes"`
 
 	Tx        Transaction `gorm:"embedded"`
 	TokenPair TokenPair   `gorm:"embedded"`
@@ -32,27 +32,27 @@ type BridgeTransfer struct {
 
 type L1BridgeDeposit struct {
 	BridgeTransfer        `gorm:"embedded"`
-	TransactionSourceHash common.Hash `gorm:"primaryKey;serializer:json"`
+	TransactionSourceHash common.Hash `gorm:"primaryKey;serializer:bytes"`
 }
 
 type L1BridgeDepositWithTransactionHashes struct {
 	L1BridgeDeposit L1BridgeDeposit `gorm:"embedded"`
 
-	L1TransactionHash common.Hash `gorm:"serializer:json"`
-	L2TransactionHash common.Hash `gorm:"serializer:json"`
+	L1TransactionHash common.Hash `gorm:"serializer:bytes"`
+	L2TransactionHash common.Hash `gorm:"serializer:bytes"`
 }
 
 type L2BridgeWithdrawal struct {
 	BridgeTransfer            `gorm:"embedded"`
-	TransactionWithdrawalHash common.Hash `gorm:"primaryKey;serializer:json"`
+	TransactionWithdrawalHash common.Hash `gorm:"primaryKey;serializer:bytes"`
 }
 
 type L2BridgeWithdrawalWithTransactionHashes struct {
 	L2BridgeWithdrawal L2BridgeWithdrawal `gorm:"embedded"`
-	L2TransactionHash  common.Hash        `gorm:"serializer:json"`
+	L2TransactionHash  common.Hash        `gorm:"serializer:bytes"`
 
-	ProvenL1TransactionHash    common.Hash `gorm:"serializer:json"`
-	FinalizedL1TransactionHash common.Hash `gorm:"serializer:json"`
+	ProvenL1TransactionHash    common.Hash `gorm:"serializer:bytes"`
+	FinalizedL1TransactionHash common.Hash `gorm:"serializer:bytes"`
 }
 
 type BridgeTransfersView interface {

indexer/database/contract_events.go

@@ -21,12 +21,12 @@ type ContractEvent struct {
 	GUID uuid.UUID `gorm:"primaryKey"`
 
 	// Some useful derived fields
-	BlockHash       common.Hash    `gorm:"serializer:json"`
-	ContractAddress common.Address `gorm:"serializer:json"`
-	TransactionHash common.Hash    `gorm:"serializer:json"`
+	BlockHash       common.Hash    `gorm:"serializer:bytes"`
+	ContractAddress common.Address `gorm:"serializer:bytes"`
+	TransactionHash common.Hash    `gorm:"serializer:bytes"`
 	LogIndex        uint64
 
-	EventSignature common.Hash `gorm:"serializer:json"`
+	EventSignature common.Hash `gorm:"serializer:bytes"`
 	Timestamp      uint64
 
 	// NOTE: NOT ALL THE DERIVED FIELDS ON `types.Log` ARE

indexer/database/db.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 
 	"github.com/ethereum-optimism/optimism/indexer/config"
+	_ "github.com/ethereum-optimism/optimism/indexer/database/serializers"
+
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"

indexer/database/serializers/bytes.go

+package serializers
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"gorm.io/gorm/schema"
+)
+
+type BytesSerializer struct{}
+type BytesInterface interface{ Bytes() []byte }
+type SetBytesInterface interface{ SetBytes([]byte) }
+
+func init() {
+	schema.RegisterSerializer("bytes", BytesSerializer{})
+}
+
+func (BytesSerializer) Scan(ctx context.Context, field *schema.Field, dst reflect.Value, dbValue interface{}) error {
+	if dbValue == nil {
+		return nil
+	}
+
+	hexStr, ok := dbValue.(string)
+	if !ok {
+		return fmt.Errorf("expected hex string as the database value: %T", dbValue)
+	}
+
+	b, err := hexutil.Decode(hexStr)
+	if err != nil {
+		return fmt.Errorf("failed to decode database value: %w", err)
+	}
+
+	fieldValue := reflect.New(field.FieldType)
+	fieldInterface := fieldValue.Interface()
+
+	// Detect if we're deserializing into a pointer. If so, we'll need to
+	// also allocate memory to where the allocated pointer should point to
+	if field.FieldType.Kind() == reflect.Pointer {
+		nestedField := fieldValue.Elem()
+		if nestedField.Elem().Kind() == reflect.Pointer {
+			return fmt.Errorf("double pointers are the max depth supported: %T", fieldValue)
+		}
+
+		// We'll want to call `SetBytes` on the pointer to
+		// the allocated memory and not the double pointer
+		nestedField.Set(reflect.New(field.FieldType.Elem()))
+		fieldInterface = nestedField.Interface()
+	}
+
+	fieldSetBytes, ok := fieldInterface.(SetBytesInterface)
+	if !ok {
+		return fmt.Errorf("field does not satisfy the `SetBytes([]byte)` interface: %T", fieldInterface)
+	}
+
+	fieldSetBytes.SetBytes(b)
+	field.ReflectValueOf(ctx, dst).Set(fieldValue.Elem())
+	return nil
+}
+
+func (BytesSerializer) Value(ctx context.Context, field *schema.Field, dst reflect.Value, fieldValue interface{}) (interface{}, error) {
+	if fieldValue == nil || (field.FieldType.Kind() == reflect.Pointer && reflect.ValueOf(fieldValue).IsNil()) {
+		return nil, nil
+	}
+
+	fieldBytes, ok := fieldValue.(BytesInterface)
+	if !ok {
+		return nil, fmt.Errorf("field does not satisfy the `Bytes() []byte` interface")
+	}
+
+	hexStr := hexutil.Encode(fieldBytes.Bytes())
+	return hexStr, nil
+}

indexer/database/rlp_serializer.go → indexer/database/serializers/rlp.go

-package database
+package serializers
 
 import (
 	"context"
@@ -13,38 +13,28 @@ import (
 
 type RLPSerializer struct{}
 
-type RLPInterface interface {
-	rlp.Encoder
-	rlp.Decoder
-}
-
 func init() {
 	schema.RegisterSerializer("rlp", RLPSerializer{})
 }
 
 func (RLPSerializer) Scan(ctx context.Context, field *schema.Field, dst reflect.Value, dbValue interface{}) error {
-	fieldValue := reflect.New(field.FieldType)
-	if dbValue != nil {
-		var bytes []byte
-		switch v := dbValue.(type) {
-		case []byte:
-			bytes = v
-		case string:
-			b, err := hexutil.Decode(v)
-			if err != nil {
-				return err
+	if dbValue == nil {
+		return nil
 	}
-			bytes = b
-		default:
-			return fmt.Errorf("unrecognized RLP bytes: %#v", dbValue)
+
+	hexStr, ok := dbValue.(string)
+	if !ok {
+		return fmt.Errorf("expected hex string as the database value: %T", dbValue)
 	}
 
-		if len(bytes) > 0 {
-			err := rlp.DecodeBytes(bytes, fieldValue.Interface())
+	b, err := hexutil.Decode(hexStr)
 	if err != nil {
-				return err
-			}
+		return fmt.Errorf("failed to decode database value: %w", err)
 	}
+
+	fieldValue := reflect.New(field.FieldType)
+	if err := rlp.DecodeBytes(b, fieldValue.Interface()); err != nil {
+		return fmt.Errorf("failed to decode rlp bytes: %w", err)
 	}
 
 	field.ReflectValueOf(ctx, dst).Set(fieldValue.Elem())
@@ -52,18 +42,15 @@ func (RLPSerializer) Scan(ctx context.Context, field *schema.Field, dst reflect.
 }
 
 func (RLPSerializer) Value(ctx context.Context, field *schema.Field, dst reflect.Value, fieldValue interface{}) (interface{}, error) {
-	// Even though rlp.Encode takes an interface and will error out if the passed interface does not
-	// satisfy the interface, we check here since we also want to make sure this type satisfies the
-	// rlp.Decoder interface as well
-	i := reflect.TypeOf(new(RLPInterface)).Elem()
-	if !reflect.TypeOf(fieldValue).Implements(i) {
-		return nil, fmt.Errorf("%T does not satisfy RLP Encoder & Decoder interface", fieldValue)
+	if fieldValue == nil || (field.FieldType.Kind() == reflect.Pointer && reflect.ValueOf(fieldValue).IsNil()) {
+		return nil, nil
 	}
 
 	rlpBytes, err := rlp.EncodeToBytes(fieldValue)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to encode rlp bytes: %w", err)
 	}
 
-	return hexutil.Bytes(rlpBytes).MarshalText()
+	hexStr := hexutil.Encode(rlpBytes)
+	return hexStr, nil
 }

indexer/database/types.go

@@ -93,3 +93,12 @@ func (h *RLPHeader) Header() *types.Header {
 func (h *RLPHeader) Hash() common.Hash {
 	return h.Header().Hash()
 }
+
+type Bytes []byte
+
+func (b Bytes) Bytes() []byte {
+	return b[:]
+}
+func (b *Bytes) SetBytes(bytes []byte) {
+	*b = bytes
+}