op-challenger: Support posting blob preimages (#9376)

* op-challenger: Load required data for blob preimages

* op-challenger: Support posting blob preimages to the oracle contract.

* op-challenger: Use actual oracle data when computing proof.

* op-challenger: Test rejecting invalid blob value

* Unextract method - it wasn't required.

* op-challenger: Load commitment and required field element from the key preimage rather than the hint
op-program: Store the preimage of the field element key so it is available to the challenger

* op-challenger: Improve testing. Use geth KZG utils to avoid creating a new kzg context.

op-challenger/game/fault/contracts/oracle.go

@@ -25,6 +25,7 @@ const (
 	methodSqueezeLPP                = "squeezeLPP"
 	methodLoadKeccak256PreimagePart = "loadKeccak256PreimagePart"
 	methodLoadSha256PreimagePart    = "loadSha256PreimagePart"
+	methodLoadBlobPreimagePart      = "loadBlobPreimagePart"
 	methodProposalCount             = "proposalCount"
 	methodProposals                 = "proposals"
 	methodProposalMetadata          = "proposalMetadata"
@@ -93,6 +94,14 @@ func (c *PreimageOracleContract) AddGlobalDataTx(data *types.PreimageOracleData)
 	case preimage.Sha256KeyType:
 		call := c.contract.Call(methodLoadSha256PreimagePart, new(big.Int).SetUint64(uint64(data.OracleOffset)), data.GetPreimageWithoutSize())
 		return call.ToTxCandidate()
+	case preimage.BlobKeyType:
+		call := c.contract.Call(methodLoadBlobPreimagePart,
+			new(big.Int).SetUint64(data.BlobFieldIndex),
+			new(big.Int).SetBytes(data.GetPreimageWithoutSize()),
+			data.BlobCommitment,
+			data.BlobProof,
+			new(big.Int).SetUint64(uint64(data.OracleOffset)))
+		return call.ToTxCandidate()
 	default:
 		return txmgr.TxCandidate{}, fmt.Errorf("%w: %v", ErrUnsupportedKeyType, keyType)
 	}

op-challenger/game/fault/contracts/oracle_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math"
 	"math/big"
+	"math/rand"
 	"testing"
 
 	"github.com/ethereum-optimism/optimism/op-bindings/bindings"
@@ -14,6 +15,7 @@ import (
 	preimage "github.com/ethereum-optimism/optimism/op-preimage"
 	"github.com/ethereum-optimism/optimism/op-service/sources/batching"
 	batchingTest "github.com/ethereum-optimism/optimism/op-service/sources/batching/test"
+	"github.com/ethereum-optimism/optimism/op-service/testutils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/stretchr/testify/require"
 )
@@ -51,6 +53,23 @@ func TestPreimageOracleContract_AddGlobalDataTx(t *testing.T) {
 		require.NoError(t, err)
 		stubRpc.VerifyTxCandidate(tx)
 	})
+
+	t.Run("Blob", func(t *testing.T) {
+		stubRpc, oracle := setupPreimageOracleTest(t)
+		fieldData := testutils.RandomData(rand.New(rand.NewSource(23)), 32)
+		data := types.NewPreimageOracleData(common.Hash{byte(preimage.BlobKeyType), 0xcc}.Bytes(), fieldData, uint32(545))
+		stubRpc.SetResponse(oracleAddr, methodLoadBlobPreimagePart, batching.BlockLatest, []interface{}{
+			new(big.Int).SetUint64(data.BlobFieldIndex),
+			new(big.Int).SetBytes(data.GetPreimageWithoutSize()),
+			data.BlobCommitment,
+			data.BlobProof,
+			new(big.Int).SetUint64(uint64(data.OracleOffset)),
+		}, nil)
+
+		tx, err := oracle.AddGlobalDataTx(data)
+		require.NoError(t, err)
+		stubRpc.VerifyTxCandidate(tx)
+	})
 }
 
 func TestPreimageOracleContract_ChallengePeriod(t *testing.T) {

op-challenger/game/fault/trace/cannon/executor.go

@@ -85,7 +85,7 @@ func (e *Executor) generateProof(ctx context.Context, dir string, begin uint64,
 		return fmt.Errorf("find starting snapshot: %w", err)
 	}
 	proofDir := filepath.Join(dir, proofsDir)
-	dataDir := filepath.Join(dir, preimagesDir)
+	dataDir := preimageDir(dir)
 	lastGeneratedState := filepath.Join(dir, finalState)
 	args := []string{
 		"run",
@@ -141,6 +141,10 @@ func (e *Executor) generateProof(ctx context.Context, dir string, begin uint64,
 	return err
 }
 
+func preimageDir(dir string) string {
+	return filepath.Join(dir, preimagesDir)
+}
+
 func runCmd(ctx context.Context, l log.Logger, binary string, args ...string) error {
 	cmd := exec.CommandContext(ctx, binary, args...)
 	stdOut := oplog.NewWriter(l, log.LevelInfo)

op-challenger/game/fault/trace/cannon/preimage.go

+package cannon
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+
+	gokzg4844 "github.com/crate-crypto/go-kzg-4844"
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	preimage "github.com/ethereum-optimism/optimism/op-preimage"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/crypto/kzg4844"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+const (
+	fieldElemKeyLength = 80
+	commitmentLength   = 48
+)
+
+var (
+	ErrInvalidScalarValue     = errors.New("invalid scalar value")
+	ErrInvalidBlobKeyPreimage = errors.New("invalid blob key preimage")
+)
+
+type preimageSource func(key common.Hash) ([]byte, error)
+
+type preimageLoader struct {
+	getPreimage preimageSource
+}
+
+func newPreimageLoader(getPreimage preimageSource) *preimageLoader {
+	return &preimageLoader{
+		getPreimage: getPreimage,
+	}
+}
+
+func (l *preimageLoader) LoadPreimage(proof *proofData) (*types.PreimageOracleData, error) {
+	if len(proof.OracleKey) == 0 {
+		return nil, nil
+	}
+	switch preimage.KeyType(proof.OracleKey[0]) {
+	case preimage.BlobKeyType:
+		return l.loadBlobPreimage(proof)
+	default:
+		return types.NewPreimageOracleData(proof.OracleKey, proof.OracleValue, proof.OracleOffset), nil
+	}
+}
+
+func (l *preimageLoader) loadBlobPreimage(proof *proofData) (*types.PreimageOracleData, error) {
+	if len(proof.OracleValue) != gokzg4844.SerializedScalarSize {
+		return nil, fmt.Errorf("%w, expected length %v but was %v", ErrInvalidScalarValue, gokzg4844.SerializedScalarSize, len(proof.OracleValue))
+	}
+
+	// The key for a blob field element is a keccak hash of commitment++fieldElementIndex.
+	// First retrieve the preimage of the key as a keccak hash so we have the commitment and required field element
+	inputsKey := preimage.Keccak256Key(proof.OracleKey).PreimageKey()
+	inputs, err := l.getPreimage(inputsKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get key preimage: %w", err)
+	}
+	if len(inputs) != fieldElemKeyLength {
+		return nil, fmt.Errorf("%w, expected length %v but was %v", ErrInvalidBlobKeyPreimage, fieldElemKeyLength, len(inputs))
+	}
+	commitment := inputs[:commitmentLength]
+	requiredFieldElement := binary.BigEndian.Uint64(inputs[72:])
+
+	// Now, reconstruct the full blob by loading the 4096 field elements.
+	blob := eth.Blob{}
+	fieldElemKey := make([]byte, fieldElemKeyLength)
+	copy(fieldElemKey[:commitmentLength], commitment)
+	for i := 0; i < params.BlobTxFieldElementsPerBlob; i++ {
+		binary.BigEndian.PutUint64(fieldElemKey[72:], uint64(i))
+		key := preimage.BlobKey(crypto.Keccak256(fieldElemKey)).PreimageKey()
+		fieldElement, err := l.getPreimage(key)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load field element %v with key %v:  %w", i, common.Hash(key), err)
+		}
+		copy(blob[i<<5:(i+1)<<5], fieldElement[:])
+	}
+
+	// Sanity check the blob data matches the commitment
+	blobCommitment, err := blob.ComputeKZGCommitment()
+	if err != nil || !bytes.Equal(blobCommitment[:], commitment[:]) {
+		return nil, fmt.Errorf("invalid blob commitment: %w", err)
+	}
+
+	// Compute the KZG proof for the required field element
+	kzgProof, _, err := kzg4844.ComputeProof(kzg4844.Blob(blob), kzg4844.Point(proof.OracleValue))
+	if err != nil {
+		return nil, fmt.Errorf("failed to compute kzg proof: %w", err)
+	}
+
+	return types.NewPreimageOracleBlobData(proof.OracleKey, proof.OracleValue, proof.OracleOffset, requiredFieldElement, commitment, kzgProof[:]), nil
+}

op-challenger/game/fault/trace/cannon/preimage_test.go

+package cannon
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+	"testing"
+
+	"github.com/consensys/gnark-crypto/ecc/bls12-381/fr"
+	gokzg4844 "github.com/crate-crypto/go-kzg-4844"
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	preimage "github.com/ethereum-optimism/optimism/op-preimage"
+	"github.com/ethereum-optimism/optimism/op-program/host/kvstore"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/crypto/kzg4844"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPreimageLoader_NoPreimage(t *testing.T) {
+	loader := newPreimageLoader(kvstore.NewMemKV().Get)
+	actual, err := loader.LoadPreimage(&proofData{})
+	require.NoError(t, err)
+	require.Nil(t, actual)
+}
+
+func TestPreimageLoader_LocalPreimage(t *testing.T) {
+	loader := newPreimageLoader(kvstore.NewMemKV().Get)
+	proof := &proofData{
+		OracleKey:    common.Hash{byte(preimage.LocalKeyType), 0xaa, 0xbb}.Bytes(),
+		OracleValue:  nil,
+		OracleOffset: 4,
+	}
+	actual, err := loader.LoadPreimage(proof)
+	require.NoError(t, err)
+	expected := types.NewPreimageOracleData(proof.OracleKey, nil, proof.OracleOffset)
+	require.Equal(t, expected, actual)
+	require.True(t, actual.IsLocal)
+}
+
+func TestPreimageLoader_SimpleTypes(t *testing.T) {
+	tests := []preimage.KeyType{
+		preimage.Keccak256KeyType,
+		preimage.Sha256KeyType,
+	}
+	for _, keyType := range tests {
+		keyType := keyType
+		t.Run(fmt.Sprintf("type-%v", keyType), func(t *testing.T) {
+			loader := newPreimageLoader(kvstore.NewMemKV().Get)
+			proof := &proofData{
+				OracleKey:    common.Hash{byte(keyType), 0xaa, 0xbb}.Bytes(),
+				OracleValue:  []byte{1, 2, 3, 4, 5, 6},
+				OracleOffset: 3,
+			}
+			actual, err := loader.LoadPreimage(proof)
+			require.NoError(t, err)
+			expected := types.NewPreimageOracleData(proof.OracleKey, proof.OracleValue, proof.OracleOffset)
+			require.Equal(t, expected, actual)
+		})
+	}
+}
+
+func TestPreimageLoader_BlobPreimage(t *testing.T) {
+	blob := testBlob()
+	commitment, err := kzg4844.BlobToCommitment(kzg4844.Blob(blob))
+	require.NoError(t, err)
+
+	fieldIndex := uint64(24)
+	elementData := blob[fieldIndex<<5 : (fieldIndex+1)<<5]
+	kzgProof, _, err := kzg4844.ComputeProof(kzg4844.Blob(blob), kzg4844.Point(elementData))
+	require.NoError(t, err)
+
+	keyBuf := make([]byte, 80)
+	copy(keyBuf[:48], commitment[:])
+	binary.BigEndian.PutUint64(keyBuf[72:], fieldIndex)
+	key := preimage.BlobKey(crypto.Keccak256Hash(keyBuf)).PreimageKey()
+
+	proof := &proofData{
+		OracleKey:    key[:],
+		OracleValue:  elementData,
+		OracleOffset: 4,
+	}
+
+	t.Run("RejectInvalidValueLength", func(t *testing.T) {
+		kv := kvstore.NewMemKV()
+		loader := newPreimageLoader(kv.Get)
+		proof := &proofData{
+			OracleKey:    proof.OracleKey,
+			OracleValue:  []byte{1, 2, 3},
+			OracleOffset: proof.OracleOffset,
+		}
+		_, err := loader.LoadPreimage(proof)
+		require.ErrorIs(t, err, ErrInvalidScalarValue)
+	})
+
+	t.Run("NoKeyPreimage", func(t *testing.T) {
+		kv := kvstore.NewMemKV()
+		loader := newPreimageLoader(kv.Get)
+		proof := &proofData{
+			OracleKey:    common.Hash{byte(preimage.BlobKeyType), 0xaf}.Bytes(),
+			OracleValue:  proof.OracleValue,
+			OracleOffset: proof.OracleOffset,
+		}
+		_, err := loader.LoadPreimage(proof)
+		require.ErrorIs(t, err, kvstore.ErrNotFound)
+	})
+
+	t.Run("InvalidKeyPreimage", func(t *testing.T) {
+		kv := kvstore.NewMemKV()
+		loader := newPreimageLoader(kv.Get)
+		proof := &proofData{
+			OracleKey:    common.Hash{byte(preimage.BlobKeyType), 0xad}.Bytes(),
+			OracleValue:  proof.OracleValue,
+			OracleOffset: proof.OracleOffset,
+		}
+		require.NoError(t, kv.Put(preimage.Keccak256Key(proof.OracleKey).PreimageKey(), []byte{1, 2}))
+		_, err := loader.LoadPreimage(proof)
+		require.ErrorIs(t, err, ErrInvalidBlobKeyPreimage)
+	})
+
+	t.Run("MissingBlobs", func(t *testing.T) {
+		kv := kvstore.NewMemKV()
+		loader := newPreimageLoader(kv.Get)
+		proof := &proofData{
+			OracleKey:    common.Hash{byte(preimage.BlobKeyType), 0xae}.Bytes(),
+			OracleValue:  proof.OracleValue,
+			OracleOffset: proof.OracleOffset,
+		}
+		require.NoError(t, kv.Put(preimage.Keccak256Key(proof.OracleKey).PreimageKey(), keyBuf))
+		_, err := loader.LoadPreimage(proof)
+		require.ErrorIs(t, err, kvstore.ErrNotFound)
+	})
+
+	t.Run("Valid", func(t *testing.T) {
+		kv := kvstore.NewMemKV()
+		loader := newPreimageLoader(kv.Get)
+		storeBlob(t, kv, gokzg4844.KZGCommitment(commitment), blob)
+		actual, err := loader.LoadPreimage(proof)
+		require.NoError(t, err)
+		expected := types.NewPreimageOracleBlobData(proof.OracleKey, proof.OracleValue, proof.OracleOffset, fieldIndex, commitment[:], kzgProof[:])
+		require.Equal(t, expected, actual)
+		require.False(t, actual.IsLocal)
+	})
+}
+
+// Returns a serialized random field element in big-endian
+func fieldElement(val uint64) [32]byte {
+	r := fr.NewElement(val)
+	return gokzg4844.SerializeScalar(r)
+}
+
+func testBlob() gokzg4844.Blob {
+	var blob gokzg4844.Blob
+	bytesPerBlob := gokzg4844.ScalarsPerBlob * gokzg4844.SerializedScalarSize
+	for i := 0; i < bytesPerBlob; i += gokzg4844.SerializedScalarSize {
+		fieldElementBytes := fieldElement(uint64(i))
+		copy(blob[i:i+gokzg4844.SerializedScalarSize], fieldElementBytes[:])
+	}
+	return blob
+}
+
+func storeBlob(t *testing.T, kv kvstore.KV, commitment gokzg4844.KZGCommitment, blob gokzg4844.Blob) {
+	// Pre-store versioned hash preimage (commitment)
+	key := preimage.Sha256Key(sha256.Sum256(commitment[:]))
+	err := kv.Put(key.PreimageKey(), commitment[:])
+	require.NoError(t, err, "Failed to store versioned hash preimage in kvstore")
+
+	// Pre-store blob field elements
+	blobKeyBuf := make([]byte, 80)
+	copy(blobKeyBuf[:48], commitment[:])
+	for i := 0; i < params.BlobTxFieldElementsPerBlob; i++ {
+		binary.BigEndian.PutUint64(blobKeyBuf[72:], uint64(i))
+		feKey := crypto.Keccak256Hash(blobKeyBuf)
+		err := kv.Put(preimage.Keccak256Key(feKey).PreimageKey(), blobKeyBuf)
+		require.NoError(t, err)
+
+		err = kv.Put(preimage.BlobKey(feKey).PreimageKey(), blob[i<<5:(i+1)<<5])
+		require.NoError(t, err, "Failed to store field element preimage in kvstore")
+	}
+}

op-challenger/game/fault/trace/cannon/provider.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/ethereum-optimism/optimism/op-challenger/config"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	"github.com/ethereum-optimism/optimism/op-program/host/kvstore"
 	"github.com/ethereum-optimism/optimism/op-service/ioutil"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -45,11 +46,12 @@ type ProofGenerator interface {
 }
 
 type CannonTraceProvider struct {
-	logger    log.Logger
-	dir       string
-	prestate  string
-	generator ProofGenerator
-	gameDepth types.Depth
+	logger         log.Logger
+	dir            string
+	prestate       string
+	generator      ProofGenerator
+	gameDepth      types.Depth
+	preimageLoader *preimageLoader
 
 	// lastStep stores the last step in the actual trace if known. 0 indicates unknown.
 	// Cached as an optimisation to avoid repeatedly attempting to execute beyond the end of the trace.
@@ -58,11 +60,12 @@ type CannonTraceProvider struct {
 
 func NewTraceProvider(logger log.Logger, m CannonMetricer, cfg *config.Config, localInputs LocalGameInputs, dir string, gameDepth types.Depth) *CannonTraceProvider {
 	return &CannonTraceProvider{
-		logger:    logger,
-		dir:       dir,
-		prestate:  cfg.CannonAbsolutePreState,
-		generator: NewExecutor(logger, m, cfg, localInputs),
-		gameDepth: gameDepth,
+		logger:         logger,
+		dir:            dir,
+		prestate:       cfg.CannonAbsolutePreState,
+		generator:      NewExecutor(logger, m, cfg, localInputs),
+		gameDepth:      gameDepth,
+		preimageLoader: newPreimageLoader(kvstore.NewDiskKV(preimageDir(dir)).Get),
 	}
 }
 
@@ -104,9 +107,9 @@ func (p *CannonTraceProvider) GetStepData(ctx context.Context, pos types.Positio
 	if data == nil {
 		return nil, nil, nil, errors.New("proof missing proof data")
 	}
-	var oracleData *types.PreimageOracleData
-	if len(proof.OracleKey) > 0 {
-		oracleData = types.NewPreimageOracleData(proof.OracleKey, proof.OracleValue, proof.OracleOffset)
+	oracleData, err := p.preimageLoader.LoadPreimage(proof)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to load preimage: %w", err)
 	}
 	return value, data, oracleData, nil
 }

op-challenger/game/fault/types/types.go

@@ -34,6 +34,11 @@ type PreimageOracleData struct {
 	OracleKey    []byte
 	oracleData   []byte
 	OracleOffset uint32
+
+	// 4844 blob data
+	BlobFieldIndex uint64
+	BlobCommitment []byte
+	BlobProof      []byte
 }
 
 // GetIdent returns the ident for the preimage oracle data.
@@ -61,6 +66,18 @@ func NewPreimageOracleData(key []byte, data []byte, offset uint32) *PreimageOrac
 	}
 }
 
+func NewPreimageOracleBlobData(key []byte, data []byte, offset uint32, fieldIndex uint64, commitment []byte, proof []byte) *PreimageOracleData {
+	return &PreimageOracleData{
+		IsLocal:        false,
+		OracleKey:      key,
+		oracleData:     data,
+		OracleOffset:   offset,
+		BlobFieldIndex: fieldIndex,
+		BlobCommitment: commitment,
+		BlobProof:      proof,
+	}
+}
+
 // StepCallData encapsulates the data needed to perform a step.
 type StepCallData struct {
 	ClaimIndex uint64

op-program/host/kvstore/mem.go

 package kvstore
 
 import (
+	"slices"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -23,7 +24,7 @@ func NewMemKV() *MemKV {
 func (m *MemKV) Put(k common.Hash, v []byte) error {
 	m.Lock()
 	defer m.Unlock()
-	m.m[k] = v
+	m.m[k] = slices.Clone(v)
 	return nil
 }
 
@@ -34,5 +35,5 @@ func (m *MemKV) Get(k common.Hash) ([]byte, error) {
 	if !ok {
 		return nil, ErrNotFound
 	}
-	return v, nil
+	return slices.Clone(v), nil
 }

op-program/host/prefetcher/prefetcher.go

@@ -157,6 +157,9 @@ func (p *Prefetcher) prefetch(ctx context.Context, hint string) error {
 		for i := 0; i < params.BlobTxFieldElementsPerBlob; i++ {
 			binary.BigEndian.PutUint64(blobKey[72:], uint64(i))
 			blobKeyHash := crypto.Keccak256Hash(blobKey)
+			if err := p.kvStore.Put(preimage.Keccak256Key(blobKeyHash).PreimageKey(), blobKey); err != nil {
+				return err
+			}
 			if err = p.kvStore.Put(preimage.BlobKey(blobKeyHash).PreimageKey(), sidecar.Blob[i<<5:(i+1)<<5]); err != nil {
 				return err
 			}

op-program/host/prefetcher/prefetcher_test.go

@@ -214,6 +214,20 @@ func TestFetchL1Blob(t *testing.T) {
 
 		blobs := oracle.GetBlob(l1Ref, blobHash)
 		require.EqualValues(t, blobs[:], blob[:])
+
+		// Check that the preimages of field element keys are also stored
+		// This makes it possible for the challenger to extract the commitment and required field from the
+		// oracle key rather than needing the hint data.
+
+		fieldElemKey := make([]byte, 80)
+		copy(fieldElemKey[:48], commitment[:])
+		for i := 0; i < params.BlobTxFieldElementsPerBlob; i++ {
+			binary.BigEndian.PutUint64(fieldElemKey[72:], uint64(i))
+			key := preimage.Keccak256Key(crypto.Keccak256(fieldElemKey)).PreimageKey()
+			actual, err := prefetcher.kvStore.Get(key)
+			require.NoError(t, err)
+			require.Equal(t, fieldElemKey, actual)
+		}
 	})
 }