index proven withdrawals. Complete bi-directional bridge indexing. Fix bug...

index proven withdrawals. Complete bi-directional bridge indexing. Fix bug with log index overlapping per block

indexer/processor/contract_events.go

@@ -13,10 +13,15 @@ import (
 	"github.com/google/uuid"
 )
 
+type ProcessedContractEventLogIndexKey struct {
+	header common.Hash
+	index  uint
+}
+
 type ProcessedContractEvents struct {
 	events            []*database.ContractEvent
 	eventsBySignature map[common.Hash][]*database.ContractEvent
-	eventByLogIndex   map[uint]*database.ContractEvent
+	eventByLogIndex   map[ProcessedContractEventLogIndexKey]*database.ContractEvent
 	eventLog          map[uuid.UUID]*types.Log
 }
 
@@ -24,7 +29,7 @@ func NewProcessedContractEvents() *ProcessedContractEvents {
 	return &ProcessedContractEvents{
 		events:            []*database.ContractEvent{},
 		eventsBySignature: make(map[common.Hash][]*database.ContractEvent),
-		eventByLogIndex:   make(map[uint]*database.ContractEvent),
+		eventByLogIndex:   make(map[ProcessedContractEventLogIndexKey]*database.ContractEvent),
 		eventLog:          make(map[uuid.UUID]*types.Log),
 	}
 }
@@ -34,13 +39,13 @@ func (p *ProcessedContractEvents) AddLog(log *types.Log, time uint64) *database.
 
 	p.events = append(p.events, &contractEvent)
 	p.eventsBySignature[contractEvent.EventSignature] = append(p.eventsBySignature[contractEvent.EventSignature], &contractEvent)
-	p.eventByLogIndex[log.Index] = &contractEvent
+	p.eventByLogIndex[ProcessedContractEventLogIndexKey{log.BlockHash, log.Index}] = &contractEvent
 	p.eventLog[contractEvent.GUID] = log
 
 	return &contractEvent
 }
 
-func DecodeFromProcessedEvents[ABI any](p *ProcessedContractEvents, name string, contractAbi *abi.ABI) ([]*ABI, error) {
+func DecodeFromProcessedContractEvents[ABI any](p *ProcessedContractEvents, name string, contractAbi *abi.ABI) ([]*ABI, error) {
 	eventAbi, ok := contractAbi.Events[name]
 	if !ok {
 		return nil, errors.New(fmt.Sprintf("event %s not present in supplied ABI", name))
@@ -85,7 +90,7 @@ func UnpackLog(out interface{}, log *types.Log, name string, contractAbi *abi.AB
 	} else if len(log.Topics) == 0 {
 		return errors.New("anonymous events are not supported")
 	} else if log.Topics[0] != eventAbi.ID {
-		return errors.New("event signature mismatch not present in supplied ABI")
+		return errors.New("event signature mismatch")
 	}
 
 	err := contractAbi.UnpackIntoInterface(out, name, log.Data)

indexer/processor/l1_processor.go

@@ -231,7 +231,7 @@ func l1ProcessFn(processLog log.Logger, ethClient node.EthClient, l1Contracts L1
 			}
 
 			// forward along contract events to the bridge processor
-			err = l1BridgeProcessContractEvents(processLog, db, processedContractEvents)
+			err = l1BridgeProcessContractEvents(processLog, db, ethClient, processedContractEvents)
 			if err != nil {
 				return err
 			}
@@ -244,15 +244,18 @@ func l1ProcessFn(processLog log.Logger, ethClient node.EthClient, l1Contracts L1
 	}
 }
 
-func l1BridgeProcessContractEvents(processLog log.Logger, db *database.DB, events *ProcessedContractEvents) error {
-	initiatedBridgeEvents, err := StandardBridgeInitiatedEvents(events)
+func l1BridgeProcessContractEvents(processLog log.Logger, db *database.DB, ethClient node.EthClient, events *ProcessedContractEvents) error {
+	rawEthClient := ethclient.NewClient(ethClient.RawRpcClient())
+
+	// Process New Deposits
+	initiatedDepositEvents, err := StandardBridgeInitiatedEvents(events)
 	if err != nil {
 		return err
 	}
 
-	l1StandardBridgeDeposits := make([]*database.Deposit, len(initiatedBridgeEvents))
-	for i, initiatedBridgeEvent := range initiatedBridgeEvents {
-		l1StandardBridgeDeposits[i] = &database.Deposit{
+	deposits := make([]*database.Deposit, len(initiatedDepositEvents))
+	for i, initiatedBridgeEvent := range initiatedDepositEvents {
+		deposits[i] = &database.Deposit{
 			GUID:                 uuid.New(),
 			InitiatedL1EventGUID: initiatedBridgeEvent.RawEvent.GUID,
 			SentMessageNonce:     database.U256{Int: initiatedBridgeEvent.CrossDomainMessengerNonce},
@@ -267,14 +270,103 @@ func l1BridgeProcessContractEvents(processLog log.Logger, db *database.DB, event
 		}
 	}
 
-	if len(l1StandardBridgeDeposits) > 0 {
-		processLog.Info("detected L1StandardBridge deposits", "num", len(l1StandardBridgeDeposits))
-		err := db.Bridge.StoreDeposits(l1StandardBridgeDeposits)
+	if len(deposits) > 0 {
+		processLog.Info("detected L1StandardBridge deposits", "num", len(deposits))
+		err := db.Bridge.StoreDeposits(deposits)
 		if err != nil {
 			return err
 		}
 	}
 
-	// no-op
+	// Prove L2 Withdrawals
+	provenWithdrawalEvents, err := OptimismPortalWithdrawalProvenEvents(events)
+	if err != nil {
+		return err
+	}
+
+	// we manually keep track since not every proven withdrawal is a standard bridge withdrawal
+	numProvenWithdrawals := 0
+	for _, provenWithdrawalEvent := range provenWithdrawalEvents {
+		withdrawalHash := provenWithdrawalEvent.WithdrawalHash
+		withdrawal, err := db.Bridge.WithdrawalByHash(withdrawalHash)
+		if err != nil {
+			return err
+		}
+
+		// Check if the L2Processor is behind or really has missed an event. We can compare against the
+		// OptimismPortal#ProvenWithdrawal on-chain mapping relative to the latest indexed L2 height
+		if withdrawal == nil {
+			var bridgeAddress common.Address
+			var portalAddress common.Address
+			if provenWithdrawalEvent.From != bridgeAddress || provenWithdrawalEvent.To != bridgeAddress {
+				// non-bridge withdrawal
+				continue
+			}
+
+			// Query for the the proven withdrawal on-chain
+			provenWithdrawal, err := OptimismPortalQueryProvenWithdrawal(rawEthClient, portalAddress, withdrawalHash)
+			if err != nil {
+				return err
+			}
+
+			latestL2Header, err := db.Blocks.LatestL2BlockHeader()
+			if err != nil {
+				return err
+			}
+
+			if latestL2Header == nil || provenWithdrawal.L2OutputIndex.Cmp(latestL2Header.Number.Int) > 0 {
+				processLog.Warn("behind on indexed L2 withdrawals")
+				return errors.New("waiting for L2Processor to catch up")
+			} else {
+				processLog.Crit("missing indexed withdrawal for this proven event")
+				return errors.New("missing withdrawal message")
+			}
+		}
+
+		err = db.Bridge.MarkProvenWithdrawalEvent(withdrawal.GUID, provenWithdrawalEvent.RawEvent.GUID)
+		if err != nil {
+			return err
+		}
+
+		numProvenWithdrawals++
+	}
+
+	if numProvenWithdrawals > 0 {
+		processLog.Info("proven L2StandardBridge withdrawals", "size", numProvenWithdrawals)
+	}
+
+	// Finalize Pending Withdrawals
+	finalizedWithdrawalEvents, err := StandardBridgeFinalizedEvents(rawEthClient, events)
+	if err != nil {
+		return err
+	}
+
+	for _, finalizedBridgeEvent := range finalizedWithdrawalEvents {
+		nonce := finalizedBridgeEvent.CrossDomainMessengerNonce
+		withdrawal, err := db.Bridge.WithdrawalByMessageNonce(nonce)
+		if err != nil {
+			processLog.Error("error querying associated withdrawal messsage using nonce", "cross_domain_messenger_nonce", nonce)
+			return err
+		}
+
+		// Since we have to prove the event on-chain first, we don't need to check if the processor is
+		// behind. we're definitely in an error state if we cannot find the withdrawal when parsing this even
+		if withdrawal == nil {
+			processLog.Crit("missing indexed withdrawal for this finalization event")
+			return errors.New("missing withdrawal message")
+		}
+
+		err = db.Bridge.MarkFinalizedWithdrawalEvent(withdrawal.GUID, finalizedBridgeEvent.RawEvent.GUID)
+		if err != nil {
+			processLog.Error("error finalizing withdrawal", "err", err)
+			return err
+		}
+	}
+
+	if len(finalizedWithdrawalEvents) > 0 {
+		processLog.Info("finalized L2StandardBridge withdrawals", "num", len(finalizedWithdrawalEvents))
+	}
+
+	// a-ok!
 	return nil
 }

indexer/processor/l2_processor.go

@@ -7,10 +7,11 @@ import (
 
 	"github.com/ethereum-optimism/optimism/indexer/database"
 	"github.com/ethereum-optimism/optimism/indexer/node"
+	"github.com/ethereum-optimism/optimism/op-bindings/bindings"
+	"github.com/google/uuid"
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/log"
@@ -168,6 +169,56 @@ func l2ProcessFn(processLog log.Logger, ethClient node.EthClient, l2Contracts L2
 func l2BridgeProcessContractEvents(processLog log.Logger, db *database.DB, ethClient node.EthClient, events *ProcessedContractEvents) error {
 	rawEthClient := ethclient.NewClient(ethClient.RawRpcClient())
 
+	l2ToL1MessagePasserABI, err := bindings.L2ToL1MessagePasserMetaData.GetAbi()
+	if err != nil {
+		return err
+	}
+
+	messagePassedEventAbi := l2ToL1MessagePasserABI.Events["MessagePassed"]
+
+	// Process New Withdrawals
+	initiatedWithdrawalEvents, err := StandardBridgeInitiatedEvents(events)
+	if err != nil {
+		return err
+	}
+
+	withdrawals := make([]*database.Withdrawal, len(initiatedWithdrawalEvents))
+	for i, initiatedBridgeEvent := range initiatedWithdrawalEvents {
+		log := events.eventLog[initiatedBridgeEvent.RawEvent.GUID]
+
+		// extract the withdrawal hash from the MessagePassed event
+		var msgPassedData bindings.L2ToL1MessagePasserMessagePassed
+		msgPassedLog := events.eventLog[events.eventByLogIndex[ProcessedContractEventLogIndexKey{log.BlockHash, log.Index + 1}].GUID]
+		err := UnpackLog(&msgPassedData, msgPassedLog, messagePassedEventAbi.Name, l2ToL1MessagePasserABI)
+		if err != nil {
+			return err
+		}
+
+		withdrawals[i] = &database.Withdrawal{
+			GUID:                 uuid.New(),
+			InitiatedL2EventGUID: initiatedBridgeEvent.RawEvent.GUID,
+			SentMessageNonce:     database.U256{Int: initiatedBridgeEvent.CrossDomainMessengerNonce},
+			WithdrawalHash:       msgPassedData.WithdrawalHash,
+			TokenPair:            database.TokenPair{L1TokenAddress: initiatedBridgeEvent.LocalToken, L2TokenAddress: initiatedBridgeEvent.RemoteToken},
+			Tx: database.Transaction{
+				FromAddress: initiatedBridgeEvent.From,
+				ToAddress:   initiatedBridgeEvent.To,
+				Amount:      database.U256{Int: initiatedBridgeEvent.Amount},
+				Data:        initiatedBridgeEvent.ExtraData,
+				Timestamp:   initiatedBridgeEvent.RawEvent.Timestamp,
+			},
+		}
+	}
+
+	if len(withdrawals) > 0 {
+		processLog.Info("detected L2StandardBridge withdrawals", "num", len(withdrawals))
+		err := db.Bridge.StoreWithdrawals(withdrawals)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Finalize Deposits
 	finalizationBridgeEvents, err := StandardBridgeFinalizedEvents(rawEthClient, events)
 	if err != nil {
 		return err
@@ -186,14 +237,12 @@ func l2BridgeProcessContractEvents(processLog log.Logger, db *database.DB, ethCl
 				return err
 			}
 
-			// Check if the L1Processor is behind or really has missed an event. Since the decimal representation
-			// of the nonce will be large (first two bytes indicate the version), we log the nonce as needed in hex format
+			// Check if the L1Processor is behind or really has missed an event
 			if latestNonce == nil || nonce.Cmp(latestNonce) > 0 {
-				processLog.Warn("behind on indexed L1 deposits", "deposit_message_nonce", hexutil.EncodeBig(nonce), "latest_deposit_message_nonce", hexutil.EncodeBig(latestNonce))
+				processLog.Warn("behind on indexed L1 deposits")
 				return errors.New("waiting for L1Processor to catch up")
 			} else {
-				log := events.eventLog[finalizedBridgeEvent.RawEvent.GUID]
-				processLog.Crit("missing indexed deposit for this finalization event", "deposit_cross_domain_message_nonce", hexutil.EncodeBig(nonce), "tx_hash", log.TxHash)
+				processLog.Crit("missing indexed deposit for this finalization event")
 				return errors.New("missing deposit message")
 			}
 		}
@@ -205,9 +254,8 @@ func l2BridgeProcessContractEvents(processLog log.Logger, db *database.DB, ethCl
 		}
 	}
 
-	numFinalizedDeposits := len(finalizationBridgeEvents)
-	if numFinalizedDeposits > 0 {
-		processLog.Info("finalized L1StandardBridge deposits", "size", numFinalizedDeposits)
+	if len(finalizationBridgeEvents) > 0 {
+		processLog.Info("finalized L1StandardBridge deposits", "size", len(finalizationBridgeEvents))
 	}
 
 	// a-ok

indexer/processor/standard_bridge.go

@@ -67,7 +67,10 @@ func StandardBridgeFinalizedEvents(rawEthClient *ethclient.Client, events *Proce
 	return append(ethBridgeFinalizedEvents, erc20BridgeFinalizedEvents...), nil
 }
 
-func _standardBridgeInitiatedEvents[BridgeEvent bindings.L1StandardBridgeETHBridgeInitiated | bindings.L1StandardBridgeERC20BridgeInitiated](events *ProcessedContractEvents) ([]StandardBridgeInitiatedEvent, error) {
+// parse out eth or erc20 bridge initiated events
+func _standardBridgeInitiatedEvents[BridgeEvent bindings.L1StandardBridgeETHBridgeInitiated | bindings.L1StandardBridgeERC20BridgeInitiated](
+	events *ProcessedContractEvents,
+) ([]StandardBridgeInitiatedEvent, error) {
 	l1StandardBridgeABI, err := bindings.L1StandardBridgeMetaData.GetAbi()
 	if err != nil {
 		return nil, err
@@ -80,10 +83,10 @@ func _standardBridgeInitiatedEvents[BridgeEvent bindings.L1StandardBridgeETHBrid
 
 	sentMessageEventAbi := l1CrossDomainMessengerABI.Events["SentMessage"]
 
-	var bridgeData BridgeEvent
+	var tmp BridgeEvent
 	var eventName string
 	var finalizeMethodName string
-	switch any(bridgeData).(type) {
+	switch any(tmp).(type) {
 	case bindings.L1StandardBridgeETHBridgeInitiated:
 		eventName = "ETHBridgeInitiated"
 		finalizeMethodName = "finalizeBridgeETH"
@@ -98,15 +101,18 @@ func _standardBridgeInitiatedEvents[BridgeEvent bindings.L1StandardBridgeETHBrid
 	initiatedBridgeEvents := make([]StandardBridgeInitiatedEvent, len(processedInitiatedBridgeEvents))
 	for i, bridgeInitiatedEvent := range processedInitiatedBridgeEvents {
 		log := events.eventLog[bridgeInitiatedEvent.GUID]
-		err := UnpackLog(&bridgeData, log, eventName, l1StandardBridgeABI)
+
+		bridgeData := new(BridgeEvent)
+		err := UnpackLog(bridgeData, log, eventName, l1StandardBridgeABI)
 		if err != nil {
 			return nil, err
 		}
 
 		// Look for the sent message event to extract the associated messager nonce
-		//       - The `SentMessage` event is the second after the bridge initiated event. BridgeInitiated -> Portal#DepositTransaction -> SentMesage ...
+		//   - L1: BridgeInitiated -> Portal#DepositTransaction -> SentMessage ...
+		//   - L1: BridgeInitiated -> L2ToL1MessagePasser#MessagePassed -> SentMessage ...
 		var sentMsgData bindings.L1CrossDomainMessengerSentMessage
-		sentMsgLog := events.eventLog[events.eventByLogIndex[log.Index+2].GUID]
+		sentMsgLog := events.eventLog[events.eventByLogIndex[ProcessedContractEventLogIndexKey{log.BlockHash, log.Index + 2}].GUID]
 		err = UnpackLog(&sentMsgData, sentMsgLog, sentMessageEventAbi.Name, l1CrossDomainMessengerABI)
 		if err != nil {
 			return nil, err
@@ -115,12 +121,14 @@ func _standardBridgeInitiatedEvents[BridgeEvent bindings.L1StandardBridgeETHBrid
 		var erc20BridgeData *bindings.L1StandardBridgeERC20BridgeInitiated
 		var expectedCrossDomainMessage []byte
 		switch any(bridgeData).(type) {
-		case bindings.L1StandardBridgeETHBridgeInitiated:
-			ethBridgeData := any(bridgeData).(bindings.L1StandardBridgeETHBridgeInitiated)
+		case *bindings.L1StandardBridgeETHBridgeInitiated:
+			ethBridgeData := any(bridgeData).(*bindings.L1StandardBridgeETHBridgeInitiated)
 			expectedCrossDomainMessage, err = l1StandardBridgeABI.Pack(finalizeMethodName, ethBridgeData.From, ethBridgeData.To, ethBridgeData.Amount, ethBridgeData.ExtraData)
 			if err != nil {
 				return nil, err
 			}
+
+			// represent eth bridge as an erc20
 			erc20BridgeData = &bindings.L1StandardBridgeERC20BridgeInitiated{
 				// Represent ETH using the hardcoded address
 				LocalToken: ethAddress, RemoteToken: ethAddress,
@@ -128,9 +136,8 @@ func _standardBridgeInitiatedEvents[BridgeEvent bindings.L1StandardBridgeETHBrid
 				From: ethBridgeData.From, To: ethBridgeData.To, Amount: ethBridgeData.Amount, ExtraData: ethBridgeData.ExtraData,
 			}
 
-		case bindings.L1StandardBridgeERC20BridgeInitiated:
+		case *bindings.L1StandardBridgeERC20BridgeInitiated:
 			_temp := any(bridgeData).(bindings.L1StandardBridgeERC20BridgeInitiated)
-			// TODO: fix this as bridgeData is the same pointer
 			erc20BridgeData = &_temp
 			expectedCrossDomainMessage, err = l1StandardBridgeABI.Pack(finalizeMethodName, erc20BridgeData.RemoteToken, erc20BridgeData.LocalToken, erc20BridgeData.From, erc20BridgeData.To, erc20BridgeData.Amount, erc20BridgeData.ExtraData)
 			if err != nil {
@@ -148,7 +155,11 @@ func _standardBridgeInitiatedEvents[BridgeEvent bindings.L1StandardBridgeETHBrid
 	return initiatedBridgeEvents, nil
 }
 
-func _standardBridgeFinalizedEvents[BridgeEvent bindings.L1StandardBridgeETHBridgeFinalized | bindings.L1StandardBridgeERC20BridgeFinalized](rawEthClient *ethclient.Client, events *ProcessedContractEvents) ([]StandardBridgeFinalizedEvent, error) {
+// parse out eth or erc20 bridge finalization events
+func _standardBridgeFinalizedEvents[BridgeEvent bindings.L1StandardBridgeETHBridgeFinalized | bindings.L1StandardBridgeERC20BridgeFinalized](
+	rawEthClient *ethclient.Client,
+	events *ProcessedContractEvents,
+) ([]StandardBridgeFinalizedEvent, error) {
 	l1StandardBridgeABI, err := bindings.L1StandardBridgeMetaData.GetAbi()
 	if err != nil {
 		return nil, err
@@ -177,13 +188,15 @@ func _standardBridgeFinalizedEvents[BridgeEvent bindings.L1StandardBridgeETHBrid
 	finalizedBridgeEvents := make([]StandardBridgeFinalizedEvent, len(processedFinalizedBridgeEvents))
 	for i, bridgeFinalizedEvent := range processedFinalizedBridgeEvents {
 		log := events.eventLog[bridgeFinalizedEvent.GUID]
+
+		var bridgeData BridgeEvent
 		err := UnpackLog(&bridgeData, log, eventName, l1StandardBridgeABI)
 		if err != nil {
 			return nil, err
 		}
 
 		// Look for the RelayedMessage event that follows right after the BridgeFinalized Event
-		relayedMsgLog := events.eventLog[events.eventByLogIndex[log.Index+1].GUID]
+		relayedMsgLog := events.eventLog[events.eventByLogIndex[ProcessedContractEventLogIndexKey{log.BlockHash, log.Index + 1}].GUID]
 		if relayedMsgLog.Topics[0] != relayedMessageEventAbi.ID {
 			return nil, errors.New("unexpected bridge event ordering")
 		}
@@ -224,7 +237,6 @@ func _standardBridgeFinalizedEvents[BridgeEvent bindings.L1StandardBridgeETHBrid
 
 		case bindings.L1StandardBridgeERC20BridgeInitiated:
 			_temp := any(bridgeData).(bindings.L1StandardBridgeERC20BridgeFinalized)
-			// TODO: fix this as bridgeData is the same pointer
 			erc20BridgeData = &_temp
 		}