op-program: Implement retries in prefetcher block re-exec (#13819)

* op-program: Implement retries in prefetcher block re-exec

* add canon block with error test case

* add comment on error case in rexec

op-program/host/prefetcher/prefetcher_test.go

@@ -624,15 +624,23 @@ func TestFetchL2Output(t *testing.T) {
 func TestFetchL2BlockData(t *testing.T) {
 	chainID := uint64(14)
 
-	testBlockExec := func(t *testing.T, err error) {
+	testBlockExec := func(t *testing.T, clientErrs []error) {
+		require.NotEmpty(t, clientErrs)
 		prefetcher, _, _, l2Clients, _ := createPrefetcher(t)
 		l2Client := l2Clients.sources[defaultChainID]
 		rng := rand.New(rand.NewSource(123))
 		block, _ := testutils.RandomBlock(rng, 10)
 		disputedBlockHash := common.Hash{0xab}
 
-		l2Client.ExpectInfoAndTxsByHash(block.Hash(), eth.BlockToInfo(block), block.Transactions(), nil)
-		l2Client.ExpectInfoAndTxsByHash(disputedBlockHash, eth.BlockToInfo(nil), nil, err)
+		isCanonical := clientErrs[len(clientErrs)-1] == nil
+
+		for _, clientErr := range clientErrs {
+			l2Client.ExpectInfoAndTxsByHash(disputedBlockHash, eth.BlockToInfo(nil), nil, clientErr)
+		}
+		if !isCanonical {
+			l2Client.ExpectInfoAndTxsByHash(block.Hash(), eth.BlockToInfo(block), block.Transactions(), nil)
+		}
+
 		defer l2Client.MockDebugClient.AssertExpectations(t)
 		prefetcher.executor = &mockExecutor{}
 		hint := l2.L2BlockDataHint{
@@ -642,9 +650,13 @@ func TestFetchL2BlockData(t *testing.T) {
 		}.Hint()
 
 		require.NoError(t, prefetcher.Hint(hint))
-		require.True(t, prefetcher.executor.(*mockExecutor).invoked)
-		require.Equal(t, prefetcher.executor.(*mockExecutor).blockNumber, block.NumberU64()+1)
-		require.Equal(t, prefetcher.executor.(*mockExecutor).chainID, chainID)
+		if isCanonical {
+			require.False(t, prefetcher.executor.(*mockExecutor).invoked)
+		} else {
+			require.True(t, prefetcher.executor.(*mockExecutor).invoked)
+			require.Equal(t, prefetcher.executor.(*mockExecutor).blockNumber, block.NumberU64()+1)
+			require.Equal(t, prefetcher.executor.(*mockExecutor).chainID, chainID)
+		}
 
 		data, err := prefetcher.kvStore.Get(BlockDataKey(disputedBlockHash).Key())
 		require.NoError(t, err)
@@ -655,11 +667,17 @@ func TestFetchL2BlockData(t *testing.T) {
 		require.NoError(t, prefetcher.Hint(hint))
 		require.False(t, prefetcher.executor.(*mockExecutor).invoked)
 	}
-	t.Run("exec block not found", func(t *testing.T) {
-		testBlockExec(t, ethereum.NotFound)
+	t.Run("exec block is canonical", func(t *testing.T) {
+		testBlockExec(t, []error{nil})
+	})
+	t.Run("exec block is canonical with errors", func(t *testing.T) {
+		testBlockExec(t, []error{errors.New("fetch error"), nil})
+	})
+	t.Run("exec block is not canonical", func(t *testing.T) {
+		testBlockExec(t, []error{ethereum.NotFound})
 	})
-	t.Run("exec block fetch error", func(t *testing.T) {
-		testBlockExec(t, errors.New("fetch error"))
+	t.Run("exec block is not canonical with fetch error", func(t *testing.T) {
+		testBlockExec(t, []error{errors.New("fetch error"), ethereum.NotFound})
 	})
 
 	t.Run("no exec", func(t *testing.T) {

op-program/host/prefetcher/reexec.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 
 	hostcommon "github.com/ethereum-optimism/optimism/op-program/host/common"
+	"github.com/ethereum-optimism/optimism/op-service/retry"
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 )
@@ -19,20 +20,29 @@ type ProgramExecutor interface {
 // re-derive the block.
 func (p *Prefetcher) nativeReExecuteBlock(
 	ctx context.Context, agreedBlockHash, blockHash common.Hash, chainID uint64) error {
-	// Avoid retries as the block may not be canonical and unavailable
-
+	// Avoid using the retrying source to prevent indefinite retries as the block may not be canonical and unavailable
 	source, err := p.l2Sources.ForChainIDWithoutRetries(chainID)
 	if err != nil {
 		return err
 	}
-	_, _, err = source.InfoAndTxsByHash(ctx, blockHash)
-	if err == nil {
+	notFound, err := retry.Do(ctx, maxAttempts, retry.Exponential(), func() (bool, error) {
+		_, _, err := source.InfoAndTxsByHash(ctx, blockHash)
+		if errors.Is(err, ethereum.NotFound) {
+			return true, nil
+		}
+		if err != nil {
+			p.logger.Warn("Failed to retrieve l2 info and txs", "hash", blockHash, "err", err)
+		}
+		return false, err
+	})
+	if !notFound && err == nil {
 		// we already have the data needed for the program to re-execute
 		return nil
 	}
-	if !errors.Is(err, ethereum.NotFound) {
-		p.logger.Error("Failed to fetch block", "block_hash", blockHash, "err", err)
+	if notFound {
+		p.logger.Error("Requested block is not canonical", "block_hash", blockHash, "err", err)
 	}
+	// Else, i.e. there was an error, then we still want to rebuild the block
 
 	retrying, err := p.l2Sources.ForChainID(chainID)
 	if err != nil {