op-program: Retrying fetching if preimage disappears between fetching and retrying read

op-program/host/prefetcher/prefetcher.go

@@ -58,14 +58,18 @@ func (p *Prefetcher) Hint(hint string) error {
 func (p *Prefetcher) GetPreimage(ctx context.Context, key common.Hash) ([]byte, error) {
 	p.logger.Trace("Pre-image requested", "key", key)
 	pre, err := p.kvStore.Get(key)
-	if errors.Is(err, kvstore.ErrNotFound) && p.lastHint != "" {
+	// Use a loop to keep retrying the prefetch as long as the key is not found
+	// This handles the case where the prefetch downloads a preimage, but it is then deleted unexpectedly
+	// before we get to read it.
+	for errors.Is(err, kvstore.ErrNotFound) && p.lastHint != "" {
 		hint := p.lastHint
-		p.lastHint = ""
 		if err := p.prefetch(ctx, hint); err != nil {
 			return nil, fmt.Errorf("prefetch failed: %w", err)
 		}
-		// Should now be available
-		return p.kvStore.Get(key)
+		pre, err = p.kvStore.Get(key)
+		if err != nil {
+			p.logger.Error("Fetched pre-images for last hint but did not find required key", "hint", hint, "key", key)
+		}
 	}
 	return pre, err
 }

op-program/host/prefetcher/prefetcher_test.go

@@ -306,6 +306,41 @@ func TestBadHints(t *testing.T) {
 	})
 }
 
+func TestRetryWhenNotAvailableAfterPrefetching(t *testing.T) {
+	rng := rand.New(rand.NewSource(123))
+	node := testutils.RandomData(rng, 30)
+	hash := crypto.Keccak256Hash(node)
+
+	_, l1Source, l2Cl, kv := createPrefetcher(t)
+	putsToIgnore := 2
+	kv = &unreliableKvStore{KV: kv, putsToIgnore: putsToIgnore}
+	prefetcher := NewPrefetcher(testlog.Logger(t, log.LvlInfo), l1Source, l2Cl, kv)
+
+	// Expect one call for each ignored put, plus one more request for when the put succeeds
+	for i := 0; i < putsToIgnore+1; i++ {
+		l2Cl.ExpectNodeByHash(hash, node, nil)
+	}
+	defer l2Cl.MockDebugClient.AssertExpectations(t)
+
+	oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+	result := oracle.NodeByHash(hash)
+	require.EqualValues(t, node, result)
+}
+
+type unreliableKvStore struct {
+	kvstore.KV
+	putsToIgnore int
+}
+
+func (s *unreliableKvStore) Put(k common.Hash, v []byte) error {
+	if s.putsToIgnore > 0 {
+		s.putsToIgnore--
+		return nil
+	}
+	println("storing")
+	return s.KV.Put(k, v)
+}
+
 type l2Client struct {
 	*testutils.MockL2Client
 	*testutils.MockDebugClient