feat(op-challenger): resume large preimage uploads (#9135)

rabbit's foot
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

fixes
Co-authored-by: Roberto Bayardo <roberto.bayardo@coinbase.com>

op-challenger/game/fault/contracts/oracle.go

@@ -150,12 +150,15 @@ func (c *PreimageOracleContract) GetActivePreimages(ctx context.Context, blockHa
 		idents = append(idents, c.decodePreimageIdent(result))
 	}
 
-	// Fetch the metadata for each preimage
+	return c.GetProposalMetadata(ctx, block, idents...)
+}
+
+func (c *PreimageOracleContract) GetProposalMetadata(ctx context.Context, block batching.Block, idents ...gameTypes.LargePreimageIdent) ([]gameTypes.LargePreimageMetaData, error) {
 	var calls []*batching.ContractCall
 	for _, ident := range idents {
 		calls = append(calls, c.contract.Call(methodProposalMetadata, ident.Claimant, ident.UUID))
 	}
-	results, err = c.multiCaller.Call(ctx, block, calls...)
+	results, err := c.multiCaller.Call(ctx, block, calls...)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load proposal metadata: %w", err)
 	}
@@ -172,7 +175,6 @@ func (c *PreimageOracleContract) GetActivePreimages(ctx context.Context, blockHa
 			Countered:          meta.countered(),
 		})
 	}
-
 	return proposals, nil
 }
 

op-challenger/game/fault/contracts/oracle_test.go

@@ -105,12 +105,47 @@ func TestPreimageOracleContract_Squeeze(t *testing.T) {
 }
 
 func TestGetActivePreimages(t *testing.T) {
-	stubRpc, oracle := setupPreimageOracleTest(t)
 	blockHash := common.Hash{0xaa}
+	_, oracle, proposals := setupPreimageOracleTestWithProposals(t, batching.BlockByHash(blockHash))
+	preimages, err := oracle.GetActivePreimages(context.Background(), blockHash)
+	require.NoError(t, err)
+	require.Equal(t, proposals, preimages)
+}
+
+func TestGetProposalMetadata(t *testing.T) {
+	blockHash := common.Hash{0xaa}
+	block := batching.BlockByHash(blockHash)
+	stubRpc, oracle, proposals := setupPreimageOracleTestWithProposals(t, block)
+	preimages, err := oracle.GetProposalMetadata(
+		context.Background(),
+		block,
+		proposals[0].LargePreimageIdent,
+		proposals[1].LargePreimageIdent,
+		proposals[2].LargePreimageIdent,
+	)
+	require.NoError(t, err)
+	require.Equal(t, proposals, preimages)
+
+	// Fetching a proposal that doesn't exist should return an empty metadata object.
+	ident := gameTypes.LargePreimageIdent{Claimant: common.Address{0x12}, UUID: big.NewInt(123)}
+	meta := new(metadata)
+	stubRpc.SetResponse(
+		oracleAddr,
+		methodProposalMetadata,
+		block,
+		[]interface{}{ident.Claimant, ident.UUID},
+		[]interface{}{meta})
+	preimages, err = oracle.GetProposalMetadata(context.Background(), batching.BlockByHash(blockHash), ident)
+	require.NoError(t, err)
+	require.Equal(t, []gameTypes.LargePreimageMetaData{{LargePreimageIdent: ident}}, preimages)
+}
+
+func setupPreimageOracleTestWithProposals(t *testing.T, block batching.Block) (*batchingTest.AbiBasedRpc, *PreimageOracleContract, []gameTypes.LargePreimageMetaData) {
+	stubRpc, oracle := setupPreimageOracleTest(t)
 	stubRpc.SetResponse(
 		oracleAddr,
 		methodProposalCount,
-		batching.BlockByHash(blockHash),
+		block,
 		[]interface{}{},
 		[]interface{}{big.NewInt(3)})
 
@@ -150,41 +185,36 @@ func TestGetActivePreimages(t *testing.T) {
 		BytesProcessed:  233,
 		Countered:       false,
 	}
-	expectGetProposals(stubRpc, batching.BlockByHash(blockHash), preimage1, preimage2, preimage3)
-	preimages, err := oracle.GetActivePreimages(context.Background(), blockHash)
-	require.NoError(t, err)
-	require.Equal(t, []gameTypes.LargePreimageMetaData{preimage1, preimage2, preimage3}, preimages)
-}
 
-func expectGetProposals(stubRpc *batchingTest.AbiBasedRpc, block batching.Block, proposals ...gameTypes.LargePreimageMetaData) {
+	proposals := []gameTypes.LargePreimageMetaData{preimage1, preimage2, preimage3}
+
 	for i, proposal := range proposals {
-		expectGetProposal(stubRpc, block, int64(i), proposal)
+		stubRpc.SetResponse(
+			oracleAddr,
+			methodProposals,
+			block,
+			[]interface{}{big.NewInt(int64(i))},
+			[]interface{}{
+				proposal.Claimant,
+				proposal.UUID,
+			})
+		meta := new(metadata)
+		meta.setTimestamp(proposal.Timestamp)
+		meta.setPartOffset(proposal.PartOffset)
+		meta.setClaimedSize(proposal.ClaimedSize)
+		meta.setBlocksProcessed(proposal.BlocksProcessed)
+		meta.setBytesProcessed(proposal.BytesProcessed)
+		meta.setCountered(proposal.Countered)
+		stubRpc.SetResponse(
+			oracleAddr,
+			methodProposalMetadata,
+			block,
+			[]interface{}{proposal.Claimant, proposal.UUID},
+			[]interface{}{meta})
 	}
-}
 
-func expectGetProposal(stubRpc *batchingTest.AbiBasedRpc, block batching.Block, idx int64, proposal gameTypes.LargePreimageMetaData) {
-	stubRpc.SetResponse(
-		oracleAddr,
-		methodProposals,
-		block,
-		[]interface{}{big.NewInt(idx)},
-		[]interface{}{
-			proposal.Claimant,
-			proposal.UUID,
-		})
-	meta := new(metadata)
-	meta.setTimestamp(proposal.Timestamp)
-	meta.setPartOffset(proposal.PartOffset)
-	meta.setClaimedSize(proposal.ClaimedSize)
-	meta.setBlocksProcessed(proposal.BlocksProcessed)
-	meta.setBytesProcessed(proposal.BytesProcessed)
-	meta.setCountered(proposal.Countered)
-	stubRpc.SetResponse(
-		oracleAddr,
-		methodProposalMetadata,
-		block,
-		[]interface{}{proposal.Claimant, proposal.UUID},
-		[]interface{}{meta})
+	return stubRpc, oracle, proposals
+
 }
 
 func setupPreimageOracleTest(t *testing.T) (*batchingTest.AbiBasedRpc, *PreimageOracleContract) {

op-challenger/game/fault/preimages/direct_test.go

@@ -112,7 +112,7 @@ func (s *mockTxMgr) Send(_ context.Context, _ txmgr.TxCandidate) (*ethtypes.Rece
 	if s.statusFail {
 		return &ethtypes.Receipt{Status: ethtypes.ReceiptStatusFailed}, nil
 	}
-	return &ethtypes.Receipt{}, nil
+	return &ethtypes.Receipt{Status: ethtypes.ReceiptStatusSuccessful}, nil
 }
 
 func (s *mockTxMgr) BlockNumber(_ context.Context) (uint64, error) { return 0, nil }

op-challenger/game/fault/preimages/large.go

@@ -11,6 +11,8 @@ import (
 
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/keccak/matrix"
+	gameTypes "github.com/ethereum-optimism/optimism/op-challenger/game/types"
+	"github.com/ethereum-optimism/optimism/op-service/sources/batching"
 	"github.com/ethereum-optimism/optimism/op-service/txmgr"
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -54,41 +56,39 @@ func NewLargePreimageUploader(logger log.Logger, txMgr txmgr.TxManager, contract
 }
 
 func (p *LargePreimageUploader) UploadPreimage(ctx context.Context, parent uint64, data *types.PreimageOracleData) error {
-	// Split the preimage data into chunks of size [MaxChunkSize] (except the last chunk).
-	stateMatrix := matrix.NewStateMatrix()
-	chunk := make([]byte, 0, MaxChunkSize)
-	calls := []Chunk{}
-	commitments := make([][32]byte, 0, MaxLeafsPerChunk)
-	in := bytes.NewReader(data.OracleData)
-	for i := 0; ; i++ {
-		// Absorb the next preimage chunk leaf and run the keccak permutation.
-		leaf, err := stateMatrix.AbsorbNextLeaf(in)
-		chunk = append(chunk, leaf...)
-		commitments = append(commitments, stateMatrix.StateCommitment())
-		// SAFETY: the last leaf will always return an [io.EOF] error from [AbsorbNextLeaf].
-		if errors.Is(err, io.EOF) {
-			calls = append(calls, Chunk{chunk, commitments[:], true})
-			break
-		}
-		if err != nil {
-			return fmt.Errorf("failed to absorb leaf: %w", err)
-		}
-
-		// Only create a call if the chunk is full.
-		if len(chunk) >= MaxChunkSize {
-			calls = append(calls, Chunk{chunk, commitments[:], false})
-			chunk = make([]byte, 0, MaxChunkSize)
-			commitments = make([][32]byte, 0, MaxLeafsPerChunk)
-		}
+	chunks, err := p.splitChunks(data)
+	if err != nil {
+		return fmt.Errorf("failed to split preimage into chunks for data with oracle offset %d: %w", data.OracleOffset, err)
 	}
 
 	uuid := p.newUUID(data)
-	err := p.initLargePreimage(ctx, uuid, data.OracleOffset, uint32(len(data.OracleData)))
+
+	// Fetch the current metadata for this preimage data, if it exists.
+	ident := gameTypes.LargePreimageIdent{Claimant: p.txMgr.From(), UUID: uuid}
+	metadata, err := p.contract.GetProposalMetadata(ctx, batching.BlockLatest, ident)
 	if err != nil {
-		return fmt.Errorf("failed to initialize large preimage with uuid: %s: %w", uuid, err)
+		return fmt.Errorf("failed to get pre-image oracle metadata: %w", err)
+	}
+
+	// The proposal is not initialized if the queried metadata has a claimed size of 0.
+	if len(metadata) == 1 && metadata[0].ClaimedSize == 0 {
+		err = p.initLargePreimage(ctx, uuid, data.OracleOffset, uint32(len(data.OracleData)))
+		if err != nil {
+			return fmt.Errorf("failed to initialize large preimage with uuid: %s: %w", uuid, err)
+		}
 	}
 
-	err = p.addLargePreimageLeafs(ctx, uuid, calls)
+	// Filter out any chunks that have already been uploaded to the Preimage Oracle.
+	if len(metadata) > 0 {
+		numSkip := metadata[0].BytesProcessed / MaxChunkSize
+		chunks = chunks[numSkip:]
+		// If the timestamp is non-zero, the preimage has been finalized.
+		if metadata[0].Timestamp != 0 {
+			chunks = chunks[len(chunks):]
+		}
+	}
+
+	err = p.addLargePreimageLeafs(ctx, uuid, chunks)
 	if err != nil {
 		return fmt.Errorf("failed to add leaves to large preimage with uuid: %s: %w", uuid, err)
 	}
@@ -111,6 +111,37 @@ func (p *LargePreimageUploader) newUUID(data *types.PreimageOracleData) *big.Int
 	return hash.Big()
 }
 
+// splitChunks splits the preimage data into chunks of size [MaxChunkSize] (except the last chunk).
+func (p *LargePreimageUploader) splitChunks(data *types.PreimageOracleData) ([]Chunk, error) {
+	stateMatrix := matrix.NewStateMatrix()
+	chunk := make([]byte, 0, MaxChunkSize)
+	chunks := []Chunk{}
+	commitments := make([][32]byte, 0, MaxLeafsPerChunk)
+	in := bytes.NewReader(data.OracleData)
+	for i := 0; ; i++ {
+		// Absorb the next preimage chunk leaf and run the keccak permutation.
+		leaf, err := stateMatrix.AbsorbNextLeaf(in)
+		chunk = append(chunk, leaf...)
+		commitments = append(commitments, stateMatrix.StateCommitment())
+		// SAFETY: the last leaf will always return an [io.EOF] error from [AbsorbNextLeaf].
+		if errors.Is(err, io.EOF) {
+			chunks = append(chunks, Chunk{chunk, commitments[:], true})
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("failed to absorb leaf: %w", err)
+		}
+
+		// Only create a call if the chunk is full.
+		if len(chunk) >= MaxChunkSize {
+			chunks = append(chunks, Chunk{chunk, commitments[:], false})
+			chunk = make([]byte, 0, MaxChunkSize)
+			commitments = make([][32]byte, 0, MaxLeafsPerChunk)
+		}
+	}
+	return chunks, nil
+}
+
 // initLargePreimage initializes the large preimage proposal.
 // This method *must* be called before adding any leaves.
 func (p *LargePreimageUploader) initLargePreimage(ctx context.Context, uuid *big.Int, partOffset uint32, claimedSize uint32) error {

op-challenger/game/fault/preimages/large_test.go

@@ -9,6 +9,8 @@ import (
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/contracts"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/keccak/matrix"
+	gameTypes "github.com/ethereum-optimism/optimism/op-challenger/game/types"
+	"github.com/ethereum-optimism/optimism/op-service/sources/batching"
 	"github.com/ethereum-optimism/optimism/op-service/testlog"
 	"github.com/ethereum-optimism/optimism/op-service/txmgr"
 	"github.com/ethereum/go-ethereum/common"
@@ -68,7 +70,8 @@ func TestLargePreimageUploader_UploadPreimage(t *testing.T) {
 	t.Run("InitFails", func(t *testing.T) {
 		oracle, _, contract := newTestLargePreimageUploader(t)
 		contract.initFails = true
-		err := oracle.UploadPreimage(context.Background(), 0, &types.PreimageOracleData{})
+		data := mockPreimageOracleData()
+		err := oracle.UploadPreimage(context.Background(), 0, &data)
 		require.ErrorIs(t, err, mockInitLPPError)
 		require.Equal(t, 1, contract.initCalls)
 	})
@@ -76,29 +79,92 @@ func TestLargePreimageUploader_UploadPreimage(t *testing.T) {
 	t.Run("AddLeavesFails", func(t *testing.T) {
 		oracle, _, contract := newTestLargePreimageUploader(t)
 		contract.addFails = true
-		err := oracle.UploadPreimage(context.Background(), 0, &types.PreimageOracleData{})
+		data := mockPreimageOracleData()
+		err := oracle.UploadPreimage(context.Background(), 0, &data)
 		require.ErrorIs(t, err, mockAddLeavesError)
 		require.Equal(t, 1, contract.addCalls)
 	})
 
-	t.Run("Success", func(t *testing.T) {
-		fullLeaf := make([]byte, matrix.LeafSize)
-		for i := 0; i < matrix.LeafSize; i++ {
-			fullLeaf[i] = byte(i)
-		}
+	t.Run("AlreadyInitialized", func(t *testing.T) {
 		oracle, _, contract := newTestLargePreimageUploader(t)
-		data := types.PreimageOracleData{
-			OracleData: append(fullLeaf, fullLeaf...),
-		}
+		data := mockPreimageOracleData()
+		contract.initialized = true
+		contract.claimedSize = uint32(len(data.OracleData))
+		err := oracle.UploadPreimage(context.Background(), 0, &data)
+		require.Equal(t, 0, contract.initCalls)
+		require.Equal(t, 6, contract.addCalls)
+		// TODO(client-pod#467): fix this to not error. See LargePreimageUploader.UploadPreimage.
+		require.ErrorIs(t, err, errNotSupported)
+	})
+
+	t.Run("NoBytesProcessed", func(t *testing.T) {
+		oracle, _, contract := newTestLargePreimageUploader(t)
+		data := mockPreimageOracleData()
 		err := oracle.UploadPreimage(context.Background(), 0, &data)
 		require.Equal(t, 1, contract.initCalls)
-		require.Equal(t, 1, contract.addCalls)
+		require.Equal(t, 6, contract.addCalls)
 		require.Equal(t, data.OracleData, contract.addData)
-		// TODO(proofs#467): fix this to not error. See LargePreimageUploader.UploadPreimage.
+		// TODO(client-pod#467): fix this to not error. See LargePreimageUploader.UploadPreimage.
+		require.ErrorIs(t, err, errNotSupported)
+	})
+
+	t.Run("PartialBytesProcessed", func(t *testing.T) {
+		oracle, _, contract := newTestLargePreimageUploader(t)
+		data := mockPreimageOracleData()
+		contract.bytesProcessed = 3 * MaxChunkSize
+		contract.claimedSize = uint32(len(data.OracleData))
+		err := oracle.UploadPreimage(context.Background(), 0, &data)
+		require.Equal(t, 0, contract.initCalls)
+		require.Equal(t, 3, contract.addCalls)
+		require.Equal(t, data.OracleData[contract.bytesProcessed:], contract.addData)
+		// TODO(client-pod#467): fix this to not error. See LargePreimageUploader.UploadPreimage.
+		require.ErrorIs(t, err, errNotSupported)
+	})
+
+	t.Run("LastLeafNotProcessed", func(t *testing.T) {
+		oracle, _, contract := newTestLargePreimageUploader(t)
+		data := mockPreimageOracleData()
+		contract.bytesProcessed = 5 * MaxChunkSize
+		contract.claimedSize = uint32(len(data.OracleData))
+		err := oracle.UploadPreimage(context.Background(), 0, &data)
+		require.Equal(t, 0, contract.initCalls)
+		require.Equal(t, 1, contract.addCalls)
+		require.Equal(t, data.OracleData[contract.bytesProcessed:], contract.addData)
+		// TODO(client-pod#467): fix this to not error. See LargePreimageUploader.UploadPreimage.
+		require.ErrorIs(t, err, errNotSupported)
+	})
+
+	t.Run("AllBytesProcessed", func(t *testing.T) {
+		oracle, _, contract := newTestLargePreimageUploader(t)
+		data := mockPreimageOracleData()
+		contract.bytesProcessed = 5*MaxChunkSize + 1
+		contract.timestamp = 123
+		contract.claimedSize = uint32(len(data.OracleData))
+		err := oracle.UploadPreimage(context.Background(), 0, &data)
+		require.Equal(t, 0, contract.initCalls)
+		require.Equal(t, 0, contract.addCalls)
+		require.Empty(t, contract.addData)
+		// TODO(client-pod#467): fix this to not error. See LargePreimageUploader.UploadPreimage.
 		require.ErrorIs(t, err, errNotSupported)
 	})
 }
 
+func mockPreimageOracleData() types.PreimageOracleData {
+	fullLeaf := make([]byte, matrix.LeafSize)
+	for i := 0; i < matrix.LeafSize; i++ {
+		fullLeaf[i] = byte(i)
+	}
+	oracleData := make([]byte, 5*MaxLeafsPerChunk)
+	for i := 0; i < 5*MaxLeafsPerChunk; i++ {
+		oracleData = append(oracleData, fullLeaf...)
+	}
+	// Add a single byte to the end to make sure the last leaf is not processed.
+	oracleData = append(oracleData, byte(1))
+	return types.PreimageOracleData{
+		OracleData: oracleData,
+	}
+}
+
 func newTestLargePreimageUploader(t *testing.T) (*LargePreimageUploader, *mockTxMgr, *mockPreimageOracleContract) {
 	logger := testlog.Logger(t, log.LvlError)
 	txMgr := &mockTxMgr{}
@@ -109,11 +175,15 @@ func newTestLargePreimageUploader(t *testing.T) (*LargePreimageUploader, *mockTx
 }
 
 type mockPreimageOracleContract struct {
-	initCalls int
-	initFails bool
-	addCalls  int
-	addFails  bool
-	addData   []byte
+	initCalls      int
+	initFails      bool
+	initialized    bool
+	claimedSize    uint32
+	bytesProcessed int
+	timestamp      uint64
+	addCalls       int
+	addFails       bool
+	addData        []byte
 }
 
 func (s *mockPreimageOracleContract) InitLargePreimage(_ *big.Int, _ uint32, _ uint32) (txmgr.TxCandidate, error) {
@@ -134,3 +204,18 @@ func (s *mockPreimageOracleContract) AddLeaves(_ *big.Int, input []byte, _ [][32
 func (s *mockPreimageOracleContract) Squeeze(_ common.Address, _ *big.Int, _ *matrix.StateMatrix, _ contracts.Leaf, _ contracts.MerkleProof, _ contracts.Leaf, _ contracts.MerkleProof) (txmgr.TxCandidate, error) {
 	return txmgr.TxCandidate{}, nil
 }
+func (s *mockPreimageOracleContract) GetProposalMetadata(_ context.Context, _ batching.Block, idents ...gameTypes.LargePreimageIdent) ([]gameTypes.LargePreimageMetaData, error) {
+	if s.initialized || s.bytesProcessed > 0 {
+		metadata := make([]gameTypes.LargePreimageMetaData, 0)
+		for _, ident := range idents {
+			metadata = append(metadata, gameTypes.LargePreimageMetaData{
+				LargePreimageIdent: ident,
+				ClaimedSize:        s.claimedSize,
+				BytesProcessed:     uint32(s.bytesProcessed),
+				Timestamp:          s.timestamp,
+			})
+		}
+		return metadata, nil
+	}
+	return []gameTypes.LargePreimageMetaData{{LargePreimageIdent: idents[0]}}, nil
+}

op-challenger/game/fault/preimages/types.go

@@ -8,6 +8,8 @@ import (
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/contracts"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/keccak/matrix"
+	gameTypes "github.com/ethereum-optimism/optimism/op-challenger/game/types"
+	"github.com/ethereum-optimism/optimism/op-service/sources/batching"
 	"github.com/ethereum-optimism/optimism/op-service/txmgr"
 	"github.com/ethereum/go-ethereum/common"
 )
@@ -25,4 +27,5 @@ type PreimageOracleContract interface {
 	InitLargePreimage(uuid *big.Int, partOffset uint32, claimedSize uint32) (txmgr.TxCandidate, error)
 	AddLeaves(uuid *big.Int, input []byte, commitments [][32]byte, finalize bool) (txmgr.TxCandidate, error)
 	Squeeze(claimant common.Address, uuid *big.Int, stateMatrix *matrix.StateMatrix, preState contracts.Leaf, preStateProof contracts.MerkleProof, postState contracts.Leaf, postStateProof contracts.MerkleProof) (txmgr.TxCandidate, error)
+	GetProposalMetadata(ctx context.Context, block batching.Block, idents ...gameTypes.LargePreimageIdent) ([]gameTypes.LargePreimageMetaData, error)
 }