feat(op-program): L1 4844 blob fetcher in client (#9098)

* Add blob preimage type

* 4844 blob fetcher

key type

* use params package constants

* Add l1 ref timestamp to commitment hint

* single preimage hint

add log

* Stub oracle

* Pass in blobs fetcher

* Encode hex in string hint route

* Panic if blob reconstruction was faulty

* @refcell review

* rebase

op-preimage/iface.go

@@ -34,6 +34,12 @@ const (
 	LocalKeyType KeyType = 1
 	// Keccak256KeyType is for keccak256 pre-images, for any global shared pre-images.
 	Keccak256KeyType KeyType = 2
+	// GlobalGenericKeyType is a reseved key type for generic global data.
+	GlobalGenericKeyType KeyType = 3
+	// Sha256KeyType is for sha256 pre-images, for any global shared pre-images.
+	Sha256KeyType KeyType = 4
+	// BlobKeyType is for blob point pre-images.
+	BlobKeyType KeyType = 5
 )
 
 // LocalIndexKey is a key local to the program, indexing a special program input.
@@ -62,6 +68,40 @@ func (k Keccak256Key) TerminalString() string {
 	return "0x" + hex.EncodeToString(k[:])
 }
 
+// Sha256Key wraps a sha256 hash to use it as a typed pre-image key.hash
+type Sha256Key [32]byte
+
+func (k Sha256Key) PreimageKey() (out [32]byte) {
+	out = k
+	out[0] = byte(Sha256KeyType)
+	return
+}
+
+func (k Sha256Key) String() string {
+	return "0x" + hex.EncodeToString(k[:])
+}
+
+func (k Sha256Key) TerminalString() string {
+	return "0x" + hex.EncodeToString(k[:])
+}
+
+// BlobKey is the hash of a blob commitment and `z` value to use as a preimage key for `y`.
+type BlobKey [32]byte
+
+func (k BlobKey) PreimageKey() (out [32]byte) {
+	out = k
+	out[0] = byte(BlobKeyType)
+	return
+}
+
+func (k BlobKey) String() string {
+	return "0x" + hex.EncodeToString(k[:])
+}
+
+func (k BlobKey) TerminalString() string {
+	return "0x" + hex.EncodeToString(k[:])
+}
+
 // Hint is an interface to enable any program type to function as a hint,
 // when passed to the Hinter interface, returning a string representation
 // of what data the host should prepare pre-images for.

op-preimage/iface_test.go

+package preimage
+
+import (
+	"encoding/binary"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPreimageKeyTypes(t *testing.T) {
+	t.Run("LocalIndexKey", func(t *testing.T) {
+		actual := LocalIndexKey(0xFFFFFFFF)
+
+		// PreimageKey encoding
+		expected := [32]byte{}
+		expected[0] = byte(LocalKeyType)
+		binary.BigEndian.PutUint64(expected[24:], 0xFFFFFFFF)
+		require.Equal(t, expected, actual.PreimageKey())
+	})
+
+	t.Run("Keccak256Key", func(t *testing.T) {
+		fauxHash := [32]byte{}
+		fauxHash[31] = 0xFF
+		actual := Keccak256Key(fauxHash)
+
+		// PreimageKey encoding
+		expected := [32]byte{}
+		expected[0] = byte(Keccak256KeyType)
+		expected[31] = 0xFF
+		require.Equal(t, expected, actual.PreimageKey())
+
+		// String encoding
+		require.Equal(t, "0x00000000000000000000000000000000000000000000000000000000000000ff", actual.String())
+	})
+
+	t.Run("Sha256Key", func(t *testing.T) {
+		fauxHash := [32]byte{}
+		fauxHash[31] = 0xFF
+		actual := Sha256Key(fauxHash)
+
+		// PreimageKey encoding
+		expected := [32]byte{}
+		expected[0] = byte(Sha256KeyType)
+		expected[31] = 0xFF
+		require.Equal(t, expected, actual.PreimageKey())
+
+		// String encoding
+		require.Equal(t, "0x00000000000000000000000000000000000000000000000000000000000000ff", actual.String())
+	})
+
+	t.Run("BlobKey", func(t *testing.T) {
+		fauxHash := [32]byte{}
+		fauxHash[31] = 0xFF
+		actual := BlobKey(fauxHash)
+
+		// PreimageKey encoding
+		expected := [32]byte{}
+		expected[0] = byte(BlobKeyType)
+		expected[31] = 0xFF
+		require.Equal(t, expected, actual.PreimageKey())
+
+		// String encoding
+		require.Equal(t, "0x00000000000000000000000000000000000000000000000000000000000000ff", actual.String())
+	})
+}

op-program/client/driver/driver.go

@@ -14,9 +14,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 )
 
-var (
-	ErrClaimNotValid = errors.New("invalid claim")
-)
+var ErrClaimNotValid = errors.New("invalid claim")
 
 type Derivation interface {
 	Step(ctx context.Context) error
@@ -39,9 +37,9 @@ type Driver struct {
 	targetBlockNum uint64
 }
 
-func NewDriver(logger log.Logger, cfg *rollup.Config, l1Source derive.L1Fetcher, l2Source L2Source, targetBlockNum uint64) *Driver {
+func NewDriver(logger log.Logger, cfg *rollup.Config, l1Source derive.L1Fetcher, l1BlobsSource derive.L1BlobsFetcher, l2Source L2Source, targetBlockNum uint64) *Driver {
 	engine := derive.NewEngineController(l2Source, logger, metrics.NoopMetrics, cfg, sync.CLSync)
-	pipeline := derive.NewDerivationPipeline(logger, cfg, l1Source, nil, l2Source, engine, metrics.NoopMetrics, &sync.Config{})
+	pipeline := derive.NewDerivationPipeline(logger, cfg, l1Source, l1BlobsSource, l2Source, engine, metrics.NoopMetrics, &sync.Config{})
 	pipeline.Reset()
 	return &Driver{
 		logger:         logger,

op-program/client/l1/blob_fetcher.go

+package l1
+
+import (
+	"context"
+	"errors"
+
+	"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+var InvalidHashesLengthError = errors.New("invalid hashes length")
+
+type BlobFetcher struct {
+	logger log.Logger
+	oracle Oracle
+}
+
+var _ = (*derive.L1BlobsFetcher)(nil)
+
+func NewBlobFetcher(logger log.Logger, oracle Oracle) *BlobFetcher {
+	return &BlobFetcher{
+		logger: logger,
+		oracle: oracle,
+	}
+}
+
+// GetBlobs fetches blobs that were confirmed in the given L1 block with the given indexed blob hashes.
+func (b *BlobFetcher) GetBlobs(ctx context.Context, ref eth.L1BlockRef, hashes []eth.IndexedBlobHash) ([]*eth.Blob, error) {
+	blobs := make([]*eth.Blob, len(hashes))
+	for i := 0; i < len(hashes); i++ {
+		b.logger.Info("Fetching blob", "l1_ref", ref.Hash, "blob_versioned_hash", hashes[i].Hash, "index", hashes[i].Index)
+		blobs[i] = b.oracle.GetBlob(ref, hashes[i])
+	}
+	return blobs, nil
+}

op-program/client/l1/cache.go

 package l1
 
 import (
+	"encoding/binary"
+
 	"github.com/hashicorp/golang-lru/v2/simplelru"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
 
 	"github.com/ethereum-optimism/optimism/op-service/eth"
 )
@@ -18,17 +21,20 @@ type CachingOracle struct {
 	blocks *simplelru.LRU[common.Hash, eth.BlockInfo]
 	txs    *simplelru.LRU[common.Hash, types.Transactions]
 	rcpts  *simplelru.LRU[common.Hash, types.Receipts]
+	blobs  *simplelru.LRU[common.Hash, *eth.Blob]
 }
 
 func NewCachingOracle(oracle Oracle) *CachingOracle {
 	blockLRU, _ := simplelru.NewLRU[common.Hash, eth.BlockInfo](cacheSize, nil)
 	txsLRU, _ := simplelru.NewLRU[common.Hash, types.Transactions](cacheSize, nil)
 	rcptsLRU, _ := simplelru.NewLRU[common.Hash, types.Receipts](cacheSize, nil)
+	blobsLRU, _ := simplelru.NewLRU[common.Hash, *eth.Blob](cacheSize, nil)
 	return &CachingOracle{
 		oracle: oracle,
 		blocks: blockLRU,
 		txs:    txsLRU,
 		rcpts:  rcptsLRU,
+		blobs:  blobsLRU,
 	}
 }
 
@@ -63,3 +69,20 @@ func (o *CachingOracle) ReceiptsByBlockHash(blockHash common.Hash) (eth.BlockInf
 	o.rcpts.Add(blockHash, rcpts)
 	return block, rcpts
 }
+
+func (o *CachingOracle) GetBlob(ref eth.L1BlockRef, blobHash eth.IndexedBlobHash) *eth.Blob {
+	// Create a 32 byte hash key by hashing `blobHash.Hash ++ ref.Time ++ blobHash.Index`
+	hashBuf := make([]byte, 48)
+	copy(hashBuf[0:32], blobHash.Hash[:])
+	binary.BigEndian.PutUint64(hashBuf[32:], ref.Time)
+	binary.BigEndian.PutUint64(hashBuf[40:], blobHash.Index)
+	cacheKey := crypto.Keccak256Hash(hashBuf)
+
+	blob, ok := o.blobs.Get(cacheKey)
+	if ok {
+		return blob
+	}
+	blob = o.oracle.GetBlob(ref, blobHash)
+	o.blobs.Add(cacheKey, blob)
+	return blob
+}

op-program/client/l1/cache_test.go

@@ -71,3 +71,23 @@ func TestCachingOracle_ReceiptsByBlockHash(t *testing.T) {
 	require.Equal(t, eth.BlockToInfo(block), actualBlock)
 	require.EqualValues(t, rcpts, actualRcpts)
 }
+
+func TestCachingOracle_GetBlobs(t *testing.T) {
+	stub := test.NewStubOracle(t)
+	oracle := NewCachingOracle(stub)
+
+	l1BlockRef := eth.L1BlockRef{Time: 0}
+	indexedBlobHash := eth.IndexedBlobHash{Hash: [32]byte{0xFA, 0xCA, 0xDE}, Index: 0}
+	blob := eth.Blob{0xFF}
+
+	// Initial call retrieves from the stub
+	stub.Blobs[l1BlockRef] = make(map[eth.IndexedBlobHash]*eth.Blob)
+	stub.Blobs[l1BlockRef][indexedBlobHash] = &blob
+	actualBlob := oracle.GetBlob(l1BlockRef, indexedBlobHash)
+	require.Equal(t, &blob, actualBlob)
+
+	// Later calls should retrieve from cache
+	delete(stub.Blobs[l1BlockRef], indexedBlobHash)
+	actualBlob = oracle.GetBlob(l1BlockRef, indexedBlobHash)
+	require.Equal(t, &blob, actualBlob)
+}

op-program/client/l1/hints.go

@@ -2,6 +2,7 @@ package l1
 
 import (
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 
 	preimage "github.com/ethereum-optimism/optimism/op-preimage"
 )
@@ -10,6 +11,7 @@ const (
 	HintL1BlockHeader  = "l1-block-header"
 	HintL1Transactions = "l1-transactions"
 	HintL1Receipts     = "l1-receipts"
+	HintL1Blob         = "l1-blob"
 )
 
 type BlockHeaderHint common.Hash
@@ -35,3 +37,11 @@ var _ preimage.Hint = ReceiptsHint{}
 func (l ReceiptsHint) Hint() string {
 	return HintL1Receipts + " " + (common.Hash)(l).String()
 }
+
+type BlobHint []byte
+
+var _ preimage.Hint = BlobHint{}
+
+func (l BlobHint) Hint() string {
+	return HintL1Blob + " " + hexutil.Encode(l)
+}

op-program/client/l1/oracle.go

 package l1
 
 import (
+	"bytes"
+	"encoding/binary"
 	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 
 	preimage "github.com/ethereum-optimism/optimism/op-preimage"
@@ -21,6 +25,9 @@ type Oracle interface {
 
 	// ReceiptsByBlockHash retrieves the receipts from the block with the given hash.
 	ReceiptsByBlockHash(blockHash common.Hash) (eth.BlockInfo, types.Receipts)
+
+	// GetBlobField retrieves the field element at the given index from the blob with the given hash.
+	GetBlob(ref eth.L1BlockRef, blobHash eth.IndexedBlobHash) *eth.Blob
 }
 
 // PreimageOracle implements Oracle using by interfacing with the pure preimage.Oracle
@@ -86,3 +93,31 @@ func (p *PreimageOracle) ReceiptsByBlockHash(blockHash common.Hash) (eth.BlockIn
 
 	return info, receipts
 }
+
+func (p *PreimageOracle) GetBlob(ref eth.L1BlockRef, blobHash eth.IndexedBlobHash) *eth.Blob {
+	// Send a hint for the blob commitment & blob field elements.
+	blobReqMeta := make([]byte, 16)
+	binary.BigEndian.PutUint64(blobReqMeta[0:8], blobHash.Index)
+	binary.BigEndian.PutUint64(blobReqMeta[8:16], ref.Time)
+	p.hint.Hint(BlobHint(append(blobHash.Hash[:], blobReqMeta...)))
+
+	commitment := p.oracle.Get(preimage.Sha256Key(blobHash.Hash))
+
+	// Reconstruct the full blob from the 4096 field elements.
+	blob := eth.Blob{}
+	fieldElemKey := make([]byte, 80)
+	copy(fieldElemKey[:48], commitment)
+	for i := 0; i < params.BlobTxFieldElementsPerBlob; i++ {
+		binary.BigEndian.PutUint64(fieldElemKey[72:], uint64(i))
+		fieldElement := p.oracle.Get(preimage.BlobKey(crypto.Keccak256(fieldElemKey)))
+
+		copy(blob[i<<5:(i+1)<<5], fieldElement[:])
+	}
+
+	blobCommitment, err := blob.ComputeKZGCommitment()
+	if err != nil || !bytes.Equal(blobCommitment[:], commitment[:]) {
+		panic(fmt.Errorf("invalid blob commitment: %w", err))
+	}
+
+	return &blob
+}

op-program/client/l1/test/stub_oracle.go

@@ -19,6 +19,9 @@ type StubOracle struct {
 
 	// Rcpts maps Block hash to receipts
 	Rcpts map[common.Hash]types.Receipts
+
+	// Blobs maps indexed blob hash to l1 block ref to blob
+	Blobs map[eth.L1BlockRef]map[eth.IndexedBlobHash]*eth.Blob
 }
 
 func NewStubOracle(t *testing.T) *StubOracle {
@@ -27,8 +30,10 @@ func NewStubOracle(t *testing.T) *StubOracle {
 		Blocks: make(map[common.Hash]eth.BlockInfo),
 		Txs:    make(map[common.Hash]types.Transactions),
 		Rcpts:  make(map[common.Hash]types.Receipts),
+		Blobs:  make(map[eth.L1BlockRef]map[eth.IndexedBlobHash]*eth.Blob),
 	}
 }
+
 func (o StubOracle) HeaderByBlockHash(blockHash common.Hash) eth.BlockInfo {
 	info, ok := o.Blocks[blockHash]
 	if !ok {
@@ -52,3 +57,15 @@ func (o StubOracle) ReceiptsByBlockHash(blockHash common.Hash) (eth.BlockInfo, t
 	}
 	return o.HeaderByBlockHash(blockHash), rcpts
 }
+
+func (o StubOracle) GetBlob(ref eth.L1BlockRef, blobHash eth.IndexedBlobHash) *eth.Blob {
+	blobMap, ok := o.Blobs[ref]
+	if !ok {
+		o.t.Fatalf("unknown blob ref %s", ref)
+	}
+	blob, ok := blobMap[blobHash]
+	if !ok {
+		o.t.Fatalf("unknown blob hash %s %d", blobHash.Hash, blobHash.Index)
+	}
+	return blob
+}

op-program/client/program.go

@@ -40,7 +40,6 @@ func Main(logger log.Logger) {
 
 // RunProgram executes the Program, while attached to an IO based pre-image oracle, to be served by a host.
 func RunProgram(logger log.Logger, preimageOracle io.ReadWriter, preimageHinter io.ReadWriter) error {
-
 	pClient := preimage.NewOracleClient(preimageOracle)
 	hClient := preimage.NewHintWriter(preimageHinter)
 	l1PreimageOracle := l1.NewCachingOracle(l1.NewPreimageOracle(pClient, hClient))
@@ -64,6 +63,7 @@ func RunProgram(logger log.Logger, preimageOracle io.ReadWriter, preimageHinter
 // runDerivation executes the L2 state transition, given a minimal interface to retrieve data.
 func runDerivation(logger log.Logger, cfg *rollup.Config, l2Cfg *params.ChainConfig, l1Head common.Hash, l2OutputRoot common.Hash, l2Claim common.Hash, l2ClaimBlockNum uint64, l1Oracle l1.Oracle, l2Oracle l2.Oracle) error {
 	l1Source := l1.NewOracleL1Client(logger, l1Oracle, l1Head)
+	l1BlobsSource := l1.NewBlobFetcher(logger, l1Oracle)
 	engineBackend, err := l2.NewOracleBackedL2Chain(logger, l2Oracle, l2Cfg, l2OutputRoot)
 	if err != nil {
 		return fmt.Errorf("failed to create oracle-backed L2 chain: %w", err)
@@ -71,7 +71,7 @@ func runDerivation(logger log.Logger, cfg *rollup.Config, l2Cfg *params.ChainCon
 	l2Source := l2.NewOracleEngine(cfg, logger, engineBackend)
 
 	logger.Info("Starting derivation")
-	d := cldr.NewDriver(logger, cfg, l1Source, l2Source, l2ClaimBlockNum)
+	d := cldr.NewDriver(logger, cfg, l1Source, l1BlobsSource, l2Source, l2ClaimBlockNum)
 	for {
 		if err = d.Step(context.Background()); errors.Is(err, io.EOF) {
 			break