Merge pull request #6768 from ethereum-optimism/08-12-feat_indexer_Implement_pagination

feat(indexer): Implement cursor based pagination

indexer/api/api_test.go

@@ -46,15 +46,6 @@ func (mbv *MockBridgeTransfersView) L1BridgeDepositWithFilter(filter database.Br
 	return &deposit, nil
 }
 
-func (mbv *MockBridgeTransfersView) L1BridgeDepositsByAddress(address common.Address) ([]*database.L1BridgeDepositWithTransactionHashes, error) {
-	return []*database.L1BridgeDepositWithTransactionHashes{
-		{
-			L1BridgeDeposit:   deposit,
-			L1TransactionHash: common.HexToHash("0x123"),
-		},
-	}, nil
-}
-
 func (mbv *MockBridgeTransfersView) L2BridgeWithdrawal(address common.Hash) (*database.L2BridgeWithdrawal, error) {
 	return &withdrawal, nil
 }
@@ -63,15 +54,27 @@ func (mbv *MockBridgeTransfersView) L2BridgeWithdrawalWithFilter(filter database
 	return &withdrawal, nil
 }
 
-func (mbv *MockBridgeTransfersView) L2BridgeWithdrawalsByAddress(address common.Address) ([]*database.L2BridgeWithdrawalWithTransactionHashes, error) {
-	return []*database.L2BridgeWithdrawalWithTransactionHashes{
-		{
-			L2BridgeWithdrawal: withdrawal,
-			L2TransactionHash:  common.HexToHash("0x789"),
+func (mbv *MockBridgeTransfersView) L1BridgeDepositsByAddress(address common.Address, cursor string, limit int) (*database.L1BridgeDepositsResponse, error) {
+	return &database.L1BridgeDepositsResponse{
+		Deposits: []*database.L1BridgeDepositWithTransactionHashes{
+			{
+				L1BridgeDeposit:   deposit,
+				L1TransactionHash: common.HexToHash("0x123"),
+			},
 		},
 	}, nil
 }
 
+func (mbv *MockBridgeTransfersView) L2BridgeWithdrawalsByAddress(address common.Address, cursor string, limit int) (*database.L2BridgeWithdrawalsResponse, error) {
+	return &database.L2BridgeWithdrawalsResponse{
+		Withdrawals: []*database.L2BridgeWithdrawalWithTransactionHashes{
+			{
+				L2BridgeWithdrawal: withdrawal,
+				L2TransactionHash:  common.HexToHash("0x789"),
+			},
+		},
+	}, nil
+}
 func TestHealthz(t *testing.T) {
 	logger := testlog.Logger(t, log.LvlInfo)
 	api := NewApi(&MockBridgeTransfersView{}, logger)

indexer/api/routes/deposits.go

@@ -2,6 +2,7 @@ package routes
 
 import (
 	"net/http"
+	"strconv"
 
 	"github.com/ethereum-optimism/optimism/indexer/database"
 	"github.com/ethereum/go-ethereum/common"
@@ -29,9 +30,9 @@ type DepositResponse struct {
 
 // TODO this is original spec but maybe include the l2 block info too for the relayed tx
 // FIXME make a pure function that returns a struct instead of newWithdrawalResponse
-func newDepositResponse(deposits []*database.L1BridgeDepositWithTransactionHashes) DepositResponse {
-	items := make([]DepositItem, len(deposits))
-	for _, deposit := range deposits {
+func newDepositResponse(deposits *database.L1BridgeDepositsResponse) DepositResponse {
+	items := make([]DepositItem, len(deposits.Deposits))
+	for _, deposit := range deposits.Deposits {
 		item := DepositItem{
 			Guid: deposit.L1BridgeDeposit.TransactionSourceHash.String(),
 			Block: Block{
@@ -71,16 +72,30 @@ func newDepositResponse(deposits []*database.L1BridgeDepositWithTransactionHashe
 	}
 
 	return DepositResponse{
-		Cursor:      "42042042-4204-4204-4204-420420420420", // TODO
-		HasNextPage: false,                                  // TODO
+		Cursor:      deposits.Cursor,
+		HasNextPage: deposits.HasNextPage,
 		Items:       items,
 	}
 }
 
 func (h Routes) L1DepositsHandler(w http.ResponseWriter, r *http.Request) {
 	address := common.HexToAddress(chi.URLParam(r, "address"))
+	cursor := r.URL.Query().Get("cursor")
+	limitQuery := r.URL.Query().Get("limit")
 
-	deposits, err := h.BridgeTransfersView.L1BridgeDepositsByAddress(address)
+	defaultLimit := 100
+	limit := defaultLimit
+	if limitQuery != "" {
+		parsedLimit, err := strconv.Atoi(limitQuery)
+		if err != nil {
+			http.Error(w, "Limit could not be parsed into a number", http.StatusBadRequest)
+			h.Logger.Error("Invalid limit")
+			h.Logger.Error(err.Error())
+		}
+		limit = parsedLimit
+	}
+
+	deposits, err := h.BridgeTransfersView.L1BridgeDepositsByAddress(address, cursor, limit)
 	if err != nil {
 		http.Error(w, "Internal server error reading deposits", http.StatusInternalServerError)
 		h.Logger.Error("Unable to read deposits from DB")

indexer/api/routes/withdrawals.go

@@ -2,6 +2,7 @@ package routes
 
 import (
 	"net/http"
+	"strconv"
 
 	"github.com/ethereum-optimism/optimism/indexer/database"
 	"github.com/ethereum/go-ethereum/common"
@@ -42,9 +43,9 @@ type WithdrawalResponse struct {
 }
 
 // FIXME make a pure function that returns a struct instead of newWithdrawalResponse
-func newWithdrawalResponse(withdrawals []*database.L2BridgeWithdrawalWithTransactionHashes) WithdrawalResponse {
-	items := make([]WithdrawalItem, len(withdrawals))
-	for _, withdrawal := range withdrawals {
+func newWithdrawalResponse(withdrawals *database.L2BridgeWithdrawalsResponse) WithdrawalResponse {
+	items := make([]WithdrawalItem, len(withdrawals.Withdrawals))
+	for _, withdrawal := range withdrawals.Withdrawals {
 		item := WithdrawalItem{
 			Guid: withdrawal.L2BridgeWithdrawal.TransactionWithdrawalHash.String(),
 			Block: Block{
@@ -96,23 +97,34 @@ func newWithdrawalResponse(withdrawals []*database.L2BridgeWithdrawalWithTransac
 	}
 
 	return WithdrawalResponse{
-		Cursor:      "42042042-0420-4204-2042-420420420420", // TODO
-		HasNextPage: true,                                   // TODO
+		Cursor:      withdrawals.Cursor,
+		HasNextPage: withdrawals.HasNextPage,
 		Items:       items,
 	}
 }
 
 func (h Routes) L2WithdrawalsHandler(w http.ResponseWriter, r *http.Request) {
 	address := common.HexToAddress(chi.URLParam(r, "address"))
+	cursor := r.URL.Query().Get("cursor")
+	limitQuery := r.URL.Query().Get("limit")
 
-	withdrawals, err := h.BridgeTransfersView.L2BridgeWithdrawalsByAddress(address)
+	defaultLimit := 100
+	limit := defaultLimit
+	if limitQuery != "" {
+		parsedLimit, err := strconv.Atoi(limitQuery)
+		if err != nil {
+			http.Error(w, "Limit could not be parsed into a number", http.StatusBadRequest)
+			h.Logger.Error("Invalid limit")
+			h.Logger.Error(err.Error())
+		}
+		limit = parsedLimit
+	}
+	withdrawals, err := h.BridgeTransfersView.L2BridgeWithdrawalsByAddress(address, cursor, limit)
 	if err != nil {
-		http.Error(w, "Internal server error fetching withdrawals", http.StatusInternalServerError)
-		h.Logger.Error("Unable to read deposits from DB")
+		http.Error(w, "Internal server error reading withdrawals", http.StatusInternalServerError)
+		h.Logger.Error("Unable to read withdrawals from DB")
 		h.Logger.Error(err.Error())
-		return
 	}
-
 	response := newWithdrawalResponse(withdrawals)
 
 	jsonResponse(w, h.Logger, response, http.StatusOK)

indexer/database/bridge_transfers.go

@@ -2,6 +2,7 @@ package database
 
 import (
 	"errors"
+	"fmt"
 
 	"gorm.io/gorm"
 
@@ -59,11 +60,11 @@ type L2BridgeWithdrawalWithTransactionHashes struct {
 type BridgeTransfersView interface {
 	L1BridgeDeposit(common.Hash) (*L1BridgeDeposit, error)
 	L1BridgeDepositWithFilter(BridgeTransfer) (*L1BridgeDeposit, error)
-	L1BridgeDepositsByAddress(common.Address) ([]*L1BridgeDepositWithTransactionHashes, error)
+	L1BridgeDepositsByAddress(common.Address, string, int) (*L1BridgeDepositsResponse, error)
 
 	L2BridgeWithdrawal(common.Hash) (*L2BridgeWithdrawal, error)
 	L2BridgeWithdrawalWithFilter(BridgeTransfer) (*L2BridgeWithdrawal, error)
-	L2BridgeWithdrawalsByAddress(common.Address) ([]*L2BridgeWithdrawalWithTransactionHashes, error)
+	L2BridgeWithdrawalsByAddress(common.Address, string, int) (*L2BridgeWithdrawalsResponse, error)
 }
 
 type BridgeTransfersDB interface {
@@ -122,9 +123,20 @@ func (db *bridgeTransfersDB) L1BridgeDepositWithFilter(filter BridgeTransfer) (*
 	return &deposit, nil
 }
 
+type L1BridgeDepositsResponse struct {
+	Deposits    []*L1BridgeDepositWithTransactionHashes
+	Cursor      string
+	HasNextPage bool
+}
+
 // L1BridgeDepositsByAddress retrieves a list of deposits intiated by the specified address, coupled with the L1/L2 transaction
 // hashes that complete the bridge transaction.
-func (db *bridgeTransfersDB) L1BridgeDepositsByAddress(address common.Address) ([]*L1BridgeDepositWithTransactionHashes, error) {
+func (db *bridgeTransfersDB) L1BridgeDepositsByAddress(address common.Address, cursor string, limit int) (*L1BridgeDepositsResponse, error) {
+	defaultLimit := 100
+	if limit <= 0 {
+		limit = defaultLimit
+	}
+
 	depositsQuery := db.gorm.Table("l1_bridge_deposits").Select(`
 l1_bridge_deposits.*,
 l1_contract_events.transaction_hash AS l1_transaction_hash,
@@ -134,8 +146,11 @@ l1_transaction_deposits.l2_transaction_hash`)
 	depositsQuery = depositsQuery.Joins("INNER JOIN l1_transaction_deposits ON l1_bridge_deposits.transaction_source_hash = l1_transaction_deposits.source_hash")
 	depositsQuery = depositsQuery.Joins("INNER JOIN l1_contract_events ON l1_transaction_deposits.initiated_l1_event_guid = l1_contract_events.guid")
 
-	// add in cursoring options
-	filteredQuery := depositsQuery.Where(&Transaction{FromAddress: address}).Order("l1_bridge_deposits.timestamp DESC").Limit(100)
+	if cursor != "" {
+		depositsQuery = depositsQuery.Where("l1_bridge_deposits.transaction_source_hash < ?", cursor)
+	}
+
+	filteredQuery := depositsQuery.Where(&Transaction{FromAddress: address}).Order("l1_bridge_deposits.transaction_source_hash DESC").Limit(limit + 1)
 
 	deposits := []*L1BridgeDepositWithTransactionHashes{}
 	result := filteredQuery.Scan(&deposits)
@@ -146,7 +161,24 @@ l1_transaction_deposits.l2_transaction_hash`)
 		return nil, result.Error
 	}
 
-	return deposits, nil
+	hasNextPage := false
+	if len(deposits) > limit {
+		hasNextPage = true
+		deposits = deposits[:limit]
+	}
+
+	nextCursor := ""
+	if hasNextPage {
+		nextCursor = deposits[len(deposits)-1].L1TransactionHash.String()
+	}
+
+	response := &L1BridgeDepositsResponse{
+		Deposits:    deposits,
+		Cursor:      nextCursor,
+		HasNextPage: hasNextPage,
+	}
+
+	return response, nil
 }
 
 /**
@@ -186,9 +218,20 @@ func (db *bridgeTransfersDB) L2BridgeWithdrawalWithFilter(filter BridgeTransfer)
 	return &withdrawal, nil
 }
 
+type L2BridgeWithdrawalsResponse struct {
+	Withdrawals []*L2BridgeWithdrawalWithTransactionHashes
+	Cursor      string
+	HasNextPage bool
+}
+
 // L2BridgeDepositsByAddress retrieves a list of deposits intiated by the specified address, coupled with the L1/L2 transaction hashes
 // that complete the bridge transaction. The hashes that correspond to with the Bedrock multistep withdrawal process are also surfaced
-func (db *bridgeTransfersDB) L2BridgeWithdrawalsByAddress(address common.Address) ([]*L2BridgeWithdrawalWithTransactionHashes, error) {
+func (db *bridgeTransfersDB) L2BridgeWithdrawalsByAddress(address common.Address, cursor string, limit int) (*L2BridgeWithdrawalsResponse, error) {
+	defaultLimit := 100
+	if limit <= 0 {
+		limit = defaultLimit
+	}
+
 	withdrawalsQuery := db.gorm.Table("l2_bridge_withdrawals").Select(`
 l2_bridge_withdrawals.*,
 l2_contract_events.transaction_hash AS l2_transaction_hash,
@@ -200,8 +243,11 @@ finalized_l1_contract_events.transaction_hash AS finalized_l1_transaction_hash`)
 	withdrawalsQuery = withdrawalsQuery.Joins("LEFT JOIN l1_contract_events AS proven_l1_contract_events ON l2_transaction_withdrawals.proven_l1_event_guid = proven_l1_contract_events.guid")
 	withdrawalsQuery = withdrawalsQuery.Joins("LEFT JOIN l1_contract_events AS finalized_l1_contract_events ON l2_transaction_withdrawals.finalized_l1_event_guid = finalized_l1_contract_events.guid")
 
-	// add in cursoring options
-	filteredQuery := withdrawalsQuery.Where(&Transaction{FromAddress: address}).Order("l2_bridge_withdrawals.timestamp DESC").Limit(100)
+	if cursor != "" {
+		withdrawalsQuery = withdrawalsQuery.Where("l2_bridge_withdrawals.id < ?", cursor)
+	}
+
+	filteredQuery := withdrawalsQuery.Where(&Transaction{FromAddress: address}).Order("l2_bridge_withdrawals.timestamp DESC").Limit(limit + 1)
 
 	withdrawals := []*L2BridgeWithdrawalWithTransactionHashes{}
 	result := filteredQuery.Scan(&withdrawals)
@@ -212,5 +258,28 @@ finalized_l1_contract_events.transaction_hash AS finalized_l1_transaction_hash`)
 		return nil, result.Error
 	}
 
-	return withdrawals, nil
+	hasNextPage := false
+	if len(withdrawals) > limit {
+		hasNextPage = true
+		withdrawals = withdrawals[:limit]
+	}
+
+	nextCursor := ""
+	if hasNextPage {
+		nextCursor = fmt.Sprintf("%d", withdrawals[len(withdrawals)-1].L2TransactionHash)
+	}
+
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			return nil, nil
+		}
+		return nil, result.Error
+	}
+	response := &L2BridgeWithdrawalsResponse{
+		Withdrawals: withdrawals,
+		Cursor:      nextCursor,
+		HasNextPage: hasNextPage,
+	}
+
+	return response, nil
 }

indexer/migrations/20230523_create_schema.sql

@@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS l2_block_headers (
     rlp_bytes VARCHAR NOT NULL
 );
 
-/** 
+/**
  * EVENT DATA
  */