feat(op-challenger): large preimage uploader add leaf support (#9062)

op-challenger/game/fault/contracts/oracle.go

@@ -98,18 +98,9 @@ func (c *PreimageOracleContract) InitLargePreimage(uuid *big.Int, partOffset uin
 	return call.ToTxCandidate()
 }
 
-func (c *PreimageOracleContract) AddLeaves(uuid *big.Int, leaves []Leaf, finalize bool) ([]txmgr.TxCandidate, error) {
-	var txs []txmgr.TxCandidate
-	for _, leaf := range leaves {
-		commitments := [][32]byte{([32]byte)(leaf.StateCommitment.Bytes())}
-		call := c.contract.Call(methodAddLeavesLPP, uuid, leaf.Input[:], commitments, finalize)
-		tx, err := call.ToTxCandidate()
-		if err != nil {
-			return nil, err
-		}
-		txs = append(txs, tx)
-	}
-	return txs, nil
+func (c *PreimageOracleContract) AddLeaves(uuid *big.Int, input []byte, commitments [][32]byte, finalize bool) (txmgr.TxCandidate, error) {
+	call := c.contract.Call(methodAddLeavesLPP, uuid, input, commitments, finalize)
+	return call.ToTxCandidate()
 }
 
 func (c *PreimageOracleContract) Squeeze(

op-challenger/game/fault/contracts/oracle_test.go

@@ -56,23 +56,19 @@ func TestPreimageOracleContract_AddLeaves(t *testing.T) {
 	stubRpc, oracle := setupPreimageOracleTest(t)
 
 	uuid := big.NewInt(123)
-	leaves := []Leaf{{
-		Input:           [136]byte{0x12},
-		Index:           big.NewInt(123),
-		StateCommitment: common.Hash{0x34},
-	}}
+	input := []byte{0x12}
+	commitments := [][32]byte{{0x34}}
 	finalize := true
 	stubRpc.SetResponse(oracleAddr, methodAddLeavesLPP, batching.BlockLatest, []interface{}{
 		uuid,
-		leaves[0].Input[:],
-		[][32]byte{([32]byte)(leaves[0].StateCommitment.Bytes())},
+		input,
+		commitments,
 		finalize,
 	}, nil)
 
-	txs, err := oracle.AddLeaves(uuid, leaves, finalize)
+	tx, err := oracle.AddLeaves(uuid, input, commitments, finalize)
 	require.NoError(t, err)
-	require.Len(t, txs, 1)
-	stubRpc.VerifyTxCandidate(txs[0])
+	stubRpc.VerifyTxCandidate(tx)
 }
 
 func TestPreimageOracleContract_Squeeze(t *testing.T) {

op-challenger/game/fault/preimages/large.go

 package preimages
 
 import (
+	"bytes"
 	"context"
 	"crypto/rand"
 	"errors"
 	"fmt"
+	"io"
 	"math/big"
 
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	"github.com/ethereum-optimism/optimism/op-challenger/game/keccak/matrix"
 	"github.com/ethereum-optimism/optimism/op-service/txmgr"
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
@@ -17,6 +20,24 @@ var errNotSupported = errors.New("not supported")
 
 var _ PreimageUploader = (*LargePreimageUploader)(nil)
 
+// MaxLeafsPerChunk is the maximum number of leafs per chunk.
+const MaxLeafsPerChunk = 300
+
+// MaxChunkSize is the maximum size of a preimage chunk in bytes.
+// Notice, the max chunk size must be a multiple of the leaf size.
+// The max chunk size is roughly 0.04MB to avoid memory expansion.
+const MaxChunkSize = MaxLeafsPerChunk * matrix.LeafSize
+
+// Chunk is a contigous segment of preimage data.
+type Chunk struct {
+	// Input is the preimage data.
+	Input []byte
+	// Commitments are the keccak commitments for each leaf in the chunk.
+	Commitments [][32]byte
+	// Finalize indicates whether the chunk is the final chunk.
+	Finalize bool
+}
+
 // LargePreimageUploader handles uploading large preimages by
 // streaming the merkleized preimage to the PreimageOracle contract,
 // tightly packed across multiple transactions.
@@ -32,23 +53,50 @@ func NewLargePreimageUploader(logger log.Logger, txMgr txmgr.TxManager, contract
 }
 
 func (p *LargePreimageUploader) UploadPreimage(ctx context.Context, parent uint64, data *types.PreimageOracleData) error {
-	// todo(proofs#467): generate the full preimage
-	// todo(proofs#467): run the preimage through the keccak permutation, hashing
-	//                   the intermediate state matrix after each block is applied.
-	// todo(proofs#467): split up the preimage into chunks and submit the preimages
-	//                   and state commitments to the preimage oracle contract using
-	//                   `PreimageOracle.addLeavesLPP` (`_finalize` = false).
+	// Split the preimage data into chunks of size [MaxChunkSize] (except the last chunk).
+	stateMatrix := matrix.NewStateMatrix()
+	chunk := make([]byte, 0, MaxChunkSize)
+	calls := []Chunk{}
+	commitments := make([][32]byte, 0, MaxLeafsPerChunk)
+	in := bytes.NewReader(data.OracleData)
+	for i := 0; ; i++ {
+		// Absorb the next preimage chunk leaf and run the keccak permutation.
+		leaf, err := stateMatrix.AbsorbNextLeaf(in)
+		chunk = append(chunk, leaf...)
+		commitments = append(commitments, stateMatrix.StateCommitment())
+		// SAFETY: the last leaf will always return an [io.EOF] error from [AbsorbNextLeaf].
+		if errors.Is(err, io.EOF) {
+			calls = append(calls, Chunk{chunk, commitments[:], true})
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("failed to absorb leaf: %w", err)
+		}
+
+		// Only create a call if the chunk is full.
+		if len(chunk) >= MaxChunkSize {
+			calls = append(calls, Chunk{chunk, commitments[:], false})
+			chunk = make([]byte, 0, MaxChunkSize)
+			commitments = make([][32]byte, 0, MaxLeafsPerChunk)
+		}
+	}
 
 	// TODO(client-pod#473): The UUID must be deterministic so the challenger can resume uploads.
 	uuid, err := p.newUUID()
 	if err != nil {
 		return fmt.Errorf("failed to generate UUID: %w", err)
 	}
+
 	err = p.initLargePreimage(ctx, uuid, data.OracleOffset, uint32(len(data.OracleData)))
 	if err != nil {
 		return fmt.Errorf("failed to initialize large preimage with uuid: %s: %w", uuid, err)
 	}
 
+	err = p.addLargePreimageLeafs(ctx, uuid, calls)
+	if err != nil {
+		return fmt.Errorf("failed to add leaves to large preimage with uuid: %s: %w", uuid, err)
+	}
+
 	// todo(proofs#467): track the challenge period starting once the full preimage is posted.
 	// todo(proofs#467): once the challenge period is over, call `squeezeLPP` on the preimage oracle contract.
 
@@ -74,6 +122,34 @@ func (p *LargePreimageUploader) initLargePreimage(ctx context.Context, uuid *big
 	return nil
 }
 
+// addLargePreimageLeafs adds leafs to the large preimage proposal.
+// This method **must** be called after calling [initLargePreimage].
+// SAFETY: submits transactions in a [Queue] for latency while preserving submission order.
+func (p *LargePreimageUploader) addLargePreimageLeafs(ctx context.Context, uuid *big.Int, chunks []Chunk) error {
+	queue := txmgr.NewQueue[int](ctx, p.txMgr, 10)
+	receiptChs := make([]chan txmgr.TxReceipt[int], len(chunks))
+	for i, chunk := range chunks {
+		tx, err := p.contract.AddLeaves(uuid, chunk.Input, chunk.Commitments, chunk.Finalize)
+		if err != nil {
+			return fmt.Errorf("failed to create pre-image oracle tx: %w", err)
+		}
+		receiptChs[i] = make(chan txmgr.TxReceipt[int], 1)
+		queue.Send(i, tx, receiptChs[i])
+	}
+	for _, receiptCh := range receiptChs {
+		receipt := <-receiptCh
+		if receipt.Err != nil {
+			return receipt.Err
+		}
+		if receipt.Receipt.Status == ethtypes.ReceiptStatusFailed {
+			p.log.Error("LargePreimageUploader add leafs tx successfully published but reverted", "tx_hash", receipt.Receipt.TxHash)
+		} else {
+			p.log.Debug("LargePreimageUploader add leafs tx successfully published", "tx_hash", receipt.Receipt.TxHash)
+		}
+	}
+	return nil
+}
+
 // sendTxAndWait sends a transaction through the [txmgr] and waits for a receipt.
 // This sets the tx GasLimit to 0, performing gas estimation online through the [txmgr].
 func (p *LargePreimageUploader) sendTxAndWait(ctx context.Context, candidate txmgr.TxCandidate) error {

op-challenger/game/fault/preimages/large_test.go

@@ -2,6 +2,7 @@ package preimages
 
 import (
 	"context"
+	"errors"
 	"math/big"
 	"testing"
 
@@ -15,6 +16,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+var mockAddLeavesError = errors.New("mock add leaves error")
+
 func TestLargePreimageUploader_UploadPreimage(t *testing.T) {
 	t.Run("InitFails", func(t *testing.T) {
 		oracle, _, contract := newTestLargePreimageUploader(t)
@@ -24,10 +27,27 @@ func TestLargePreimageUploader_UploadPreimage(t *testing.T) {
 		require.Equal(t, 1, contract.initCalls)
 	})
 
-	t.Run("Success", func(t *testing.T) {
+	t.Run("AddLeavesFails", func(t *testing.T) {
 		oracle, _, contract := newTestLargePreimageUploader(t)
+		contract.addFails = true
 		err := oracle.UploadPreimage(context.Background(), 0, &types.PreimageOracleData{})
+		require.ErrorIs(t, err, mockAddLeavesError)
+		require.Equal(t, 1, contract.addCalls)
+	})
+
+	t.Run("Success", func(t *testing.T) {
+		fullLeaf := make([]byte, matrix.LeafSize)
+		for i := 0; i < matrix.LeafSize; i++ {
+			fullLeaf[i] = byte(i)
+		}
+		oracle, _, contract := newTestLargePreimageUploader(t)
+		data := types.PreimageOracleData{
+			OracleData: append(fullLeaf, fullLeaf...),
+		}
+		err := oracle.UploadPreimage(context.Background(), 0, &data)
 		require.Equal(t, 1, contract.initCalls)
+		require.Equal(t, 1, contract.addCalls)
+		require.Equal(t, data.OracleData, contract.addData)
 		// TODO(proofs#467): fix this to not error. See LargePreimageUploader.UploadPreimage.
 		require.ErrorIs(t, err, errNotSupported)
 	})
@@ -36,13 +56,18 @@ func TestLargePreimageUploader_UploadPreimage(t *testing.T) {
 func newTestLargePreimageUploader(t *testing.T) (*LargePreimageUploader, *mockTxMgr, *mockPreimageOracleContract) {
 	logger := testlog.Logger(t, log.LvlError)
 	txMgr := &mockTxMgr{}
-	contract := &mockPreimageOracleContract{}
+	contract := &mockPreimageOracleContract{
+		addData: make([]byte, 0),
+	}
 	return NewLargePreimageUploader(logger, txMgr, contract), txMgr, contract
 }
 
 type mockPreimageOracleContract struct {
 	initCalls int
 	initFails bool
+	addCalls  int
+	addFails  bool
+	addData   []byte
 }
 
 func (s *mockPreimageOracleContract) InitLargePreimage(_ *big.Int, _ uint32, _ uint32) (txmgr.TxCandidate, error) {
@@ -52,8 +77,13 @@ func (s *mockPreimageOracleContract) InitLargePreimage(_ *big.Int, _ uint32, _ u
 	}
 	return txmgr.TxCandidate{}, nil
 }
-func (s *mockPreimageOracleContract) AddLeaves(_ *big.Int, _ []contracts.Leaf, _ bool) ([]txmgr.TxCandidate, error) {
-	return []txmgr.TxCandidate{}, nil
+func (s *mockPreimageOracleContract) AddLeaves(_ *big.Int, input []byte, _ [][32]byte, _ bool) (txmgr.TxCandidate, error) {
+	s.addCalls++
+	s.addData = append(s.addData, input...)
+	if s.addFails {
+		return txmgr.TxCandidate{}, mockAddLeavesError
+	}
+	return txmgr.TxCandidate{}, nil
 }
 func (s *mockPreimageOracleContract) Squeeze(_ common.Address, _ *big.Int, _ *matrix.StateMatrix, _ contracts.Leaf, _ contracts.MerkleProof, _ contracts.Leaf, _ contracts.MerkleProof) (txmgr.TxCandidate, error) {
 	return txmgr.TxCandidate{}, nil

op-challenger/game/fault/preimages/types.go

@@ -23,6 +23,6 @@ type PreimageUploader interface {
 // PreimageOracleContract is the interface for interacting with the PreimageOracle contract.
 type PreimageOracleContract interface {
 	InitLargePreimage(uuid *big.Int, partOffset uint32, claimedSize uint32) (txmgr.TxCandidate, error)
-	AddLeaves(uuid *big.Int, leaves []contracts.Leaf, finalize bool) ([]txmgr.TxCandidate, error)
+	AddLeaves(uuid *big.Int, input []byte, commitments [][32]byte, finalize bool) (txmgr.TxCandidate, error)
 	Squeeze(claimant common.Address, uuid *big.Int, stateMatrix *matrix.StateMatrix, preState contracts.Leaf, preStateProof contracts.MerkleProof, postState contracts.Leaf, postStateProof contracts.MerkleProof) (txmgr.TxCandidate, error)
 }

op-challenger/game/keccak/matrix/matrix.go

@@ -43,7 +43,7 @@ func (d *StateMatrix) PackState() []byte {
 
 // AbsorbNextLeaf reads up to [LeafSize] bytes from in and absorbs them into the state matrix.
 // If EOF is reached while reading, the state matrix is finalized and [io.EOF] is returned.
-func (d *StateMatrix) AbsorbNextLeaf(in io.Reader) error {
+func (d *StateMatrix) AbsorbNextLeaf(in io.Reader) ([]byte, error) {
 	data := make([]byte, LeafSize)
 	read := 0
 	final := false
@@ -53,15 +53,16 @@ func (d *StateMatrix) AbsorbNextLeaf(in io.Reader) error {
 			final = true
 			break
 		} else if err != nil {
-			return err
+			return nil, err
 		}
 		read += n
 	}
-	d.AbsorbLeaf(data[:read], final)
+	leafData := data[:read]
+	d.AbsorbLeaf(leafData, final)
 	if final {
-		return io.EOF
+		return leafData, io.EOF
 	}
-	return nil
+	return leafData, nil
 }
 
 // AbsorbLeaf absorbs the specified data into the keccak sponge.

op-challenger/game/keccak/matrix/matrix_test.go

@@ -89,7 +89,7 @@ func TestReferenceCommitmentsFromReader(t *testing.T) {
 			commitments := []common.Hash{s.StateCommitment()}
 			in := bytes.NewReader(test.Input)
 			for {
-				err := s.AbsorbNextLeaf(in)
+				_, err := s.AbsorbNextLeaf(in)
 				if errors.Is(err, io.EOF) {
 					commitments = append(commitments, s.StateCommitment())
 					break
@@ -106,6 +106,61 @@ func TestReferenceCommitmentsFromReader(t *testing.T) {
 	}
 }
 
+func TestMatrix_AbsorbNextLeaf(t *testing.T) {
+	fullLeaf := make([]byte, LeafSize)
+	for i := 0; i < LeafSize; i++ {
+		fullLeaf[i] = byte(i)
+	}
+	tests := []struct {
+		name  string
+		input []byte
+		leafs [][]byte
+		errs  []error
+	}{
+		{
+			name:  "empty",
+			input: []byte{},
+			leafs: [][]byte{{}},
+			errs:  []error{io.EOF},
+		},
+		{
+			name:  "single",
+			input: fullLeaf,
+			leafs: [][]byte{fullLeaf},
+			errs:  []error{io.EOF},
+		},
+		{
+			name:  "single-overflow",
+			input: append(fullLeaf, byte(9)),
+			leafs: [][]byte{fullLeaf, {byte(9)}},
+			errs:  []error{nil, io.EOF},
+		},
+		{
+			name:  "double",
+			input: append(fullLeaf, fullLeaf...),
+			leafs: [][]byte{fullLeaf, fullLeaf},
+			errs:  []error{nil, io.EOF},
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			state := NewStateMatrix()
+			in := bytes.NewReader(test.input)
+			for i, leaf := range test.leafs {
+				buf, err := state.AbsorbNextLeaf(in)
+				if errors.Is(err, io.EOF) {
+					require.Equal(t, test.errs[i], err)
+					break
+				}
+				require.NoError(t, err)
+				require.Equal(t, leaf, buf)
+			}
+		})
+	}
+}
+
 func FuzzKeccak(f *testing.F) {
 	f.Fuzz(func(t *testing.T, number, time uint64, data []byte) {
 		s := NewStateMatrix()