op-program: Implement oracle backed database

Adds separate oracle method to access code by hash to support the way Geth now adds a prefix to the hash when storing new code values in the database.

op-program/cmd/main.go

 package main
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os"
@@ -92,8 +93,9 @@ func FaultProofProgram(logger log.Logger, cfg *config.Config) error {
 		return errors.New("offline mode not supported")
 	}
 
+	ctx := context.Background()
 	logger.Info("Connecting to L2 node", "l2", cfg.L2URL)
-	_, err := l2.NewFetchingL2Oracle(logger, cfg.L2URL)
+	_, err := l2.NewFetchingL2Oracle(ctx, logger, cfg.L2URL)
 	if err != nil {
 		return fmt.Errorf("connect l2 oracle: %w", err)
 	}

op-program/l2/db.go

+package l2
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+)
+
+var codePrefixedKeyLength = common.HashLength + len(rawdb.CodePrefix)
+
+var (
+	ErrInvalidKeyLength = errors.New("pre-images must be identified by 32-byte hash keys")
+)
+
+type OracleKeyValueStore struct {
+	db     ethdb.KeyValueStore
+	oracle StateOracle
+}
+
+func NewOracleBackedDB(oracle StateOracle) *OracleKeyValueStore {
+	return &OracleKeyValueStore{
+		db:     memorydb.New(),
+		oracle: oracle,
+	}
+}
+
+func (o *OracleKeyValueStore) Get(key []byte) ([]byte, error) {
+	has, err := o.db.Has(key)
+	if err != nil {
+		return nil, fmt.Errorf("checking in-memory db: %w", err)
+	}
+	if has {
+		return o.db.Get(key)
+	}
+
+	if len(key) == codePrefixedKeyLength && bytes.HasPrefix(key, rawdb.CodePrefix) {
+		key = key[len(rawdb.CodePrefix):]
+		return o.oracle.CodeByHash(*(*[common.HashLength]byte)(key))
+	}
+	if len(key) != common.HashLength {
+		return nil, ErrInvalidKeyLength
+	}
+	return o.oracle.NodeByHash(*(*[common.HashLength]byte)(key))
+}
+
+func (o *OracleKeyValueStore) NewBatch() ethdb.Batch {
+	return o.db.NewBatch()
+}
+
+func (o *OracleKeyValueStore) NewBatchWithSize(size int) ethdb.Batch {
+	return o.db.NewBatchWithSize(size)
+}
+
+func (o *OracleKeyValueStore) Put(key []byte, value []byte) error {
+	return o.db.Put(key, value)
+}
+
+func (o *OracleKeyValueStore) Close() error {
+	return nil
+}
+
+// Remaining methods are unused when accessing the state for block processing so leaving unimplemented.
+
+func (o *OracleKeyValueStore) Has(key []byte) (bool, error) {
+	panic("not supported")
+}
+
+func (o *OracleKeyValueStore) Delete(key []byte) error {
+	panic("not supported")
+}
+
+func (o *OracleKeyValueStore) Stat(property string) (string, error) {
+	panic("not supported")
+}
+
+func (o *OracleKeyValueStore) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
+	panic("not supported")
+}
+
+func (o *OracleKeyValueStore) Compact(start []byte, limit []byte) error {
+	panic("not supported")
+}
+
+func (o *OracleKeyValueStore) NewSnapshot() (ethdb.Snapshot, error) {
+	panic("not supported")
+}

op-program/l2/db_test.go

+package l2
+
+import (
+	"fmt"
+	"math/big"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	userAccount    = common.HexToAddress("0x1111")
+	codeAccount    = common.HexToAddress("0x2222")
+	unknownAccount = common.HexToAddress("0x3333")
+)
+
+// Should implement the KeyValueStore API
+var _ ethdb.KeyValueStore = (*OracleKeyValueStore)(nil)
+
+func TestGet(t *testing.T) {
+	t.Run("UnknownKey", func(t *testing.T) {
+		oracle := newStubStateOracle()
+		db := NewOracleBackedDB(oracle)
+		val, err := db.Get(common.Hash{}.Bytes())
+		require.Error(t, err)
+		require.Nil(t, val)
+	})
+
+	t.Run("IncorrectLengthKey", func(t *testing.T) {
+		oracle := newStubStateOracle()
+		db := NewOracleBackedDB(oracle)
+		val, err := db.Get([]byte{1, 2, 3})
+		require.ErrorIs(t, err, ErrInvalidKeyLength)
+		require.Nil(t, val)
+	})
+
+	t.Run("KeyWithCodePrefix", func(t *testing.T) {
+		oracle := newStubStateOracle()
+		db := NewOracleBackedDB(oracle)
+		key := common.HexToHash("0x12345678")
+		prefixedKey := append(rawdb.CodePrefix, key.Bytes()...)
+
+		expected := []byte{1, 2, 3}
+		oracle.code[key] = expected
+		val, err := db.Get(prefixedKey)
+
+		require.NoError(t, err)
+		require.Equal(t, expected, val)
+	})
+
+	t.Run("NormalKeyThatHappensToStartWithCodePrefix", func(t *testing.T) {
+		oracle := newStubStateOracle()
+		db := NewOracleBackedDB(oracle)
+		key := make([]byte, common.HashLength)
+		copy(rawdb.CodePrefix, key)
+		println(key[0])
+		expected := []byte{1, 2, 3}
+		oracle.data[common.BytesToHash(key)] = expected
+		val, err := db.Get(key)
+
+		require.NoError(t, err)
+		require.Equal(t, expected, val)
+	})
+
+	t.Run("KnownKey", func(t *testing.T) {
+		key := common.HexToHash("0xAA4488")
+		expected := []byte{2, 6, 3, 8}
+		oracle := newStubStateOracle()
+		oracle.data[key] = expected
+		db := NewOracleBackedDB(oracle)
+		val, err := db.Get(key.Bytes())
+		require.NoError(t, err)
+		require.Equal(t, expected, val)
+	})
+}
+
+func TestPut(t *testing.T) {
+	t.Run("NewKey", func(t *testing.T) {
+		oracle := newStubStateOracle()
+		db := NewOracleBackedDB(oracle)
+		key := common.HexToHash("0xAA4488")
+		value := []byte{2, 6, 3, 8}
+		err := db.Put(key.Bytes(), value)
+		require.NoError(t, err)
+
+		actual, err := db.Get(key.Bytes())
+		require.NoError(t, err)
+		require.Equal(t, value, actual)
+	})
+	t.Run("ReplaceKey", func(t *testing.T) {
+		oracle := newStubStateOracle()
+		db := NewOracleBackedDB(oracle)
+		key := common.HexToHash("0xAA4488")
+		value1 := []byte{2, 6, 3, 8}
+		value2 := []byte{1, 2, 3}
+		err := db.Put(key.Bytes(), value1)
+		require.NoError(t, err)
+		err = db.Put(key.Bytes(), value2)
+		require.NoError(t, err)
+
+		actual, err := db.Get(key.Bytes())
+		require.NoError(t, err)
+		require.Equal(t, value2, actual)
+	})
+}
+
+func TestSupportsStateDBOperations(t *testing.T) {
+	l2Genesis := createGenesis()
+	realDb := rawdb.NewDatabase(memorydb.New())
+	genesisBlock := l2Genesis.MustCommit(realDb)
+
+	loader := &kvStateOracle{
+		source: realDb,
+	}
+	assertStateDataAvailable(t, NewOracleBackedDB(loader), l2Genesis, genesisBlock)
+}
+
+func TestUpdateState(t *testing.T) {
+	l2Genesis := createGenesis()
+	oracle := newStubStateOracle()
+	db := rawdb.NewDatabase(NewOracleBackedDB(oracle))
+
+	genesisBlock := l2Genesis.MustCommit(db)
+	assertStateDataAvailable(t, db, l2Genesis, genesisBlock)
+
+	statedb, err := state.New(genesisBlock.Root(), state.NewDatabase(rawdb.NewDatabase(db)), nil)
+	require.NoError(t, err)
+	statedb.SetBalance(userAccount, big.NewInt(50))
+	require.Equal(t, big.NewInt(50), statedb.GetBalance(userAccount))
+	statedb.SetNonce(userAccount, uint64(5))
+	require.Equal(t, uint64(5), statedb.GetNonce(userAccount))
+
+	statedb.SetBalance(unknownAccount, big.NewInt(60))
+	require.Equal(t, big.NewInt(60), statedb.GetBalance(unknownAccount))
+	statedb.SetCode(codeAccount, []byte{1})
+	require.Equal(t, []byte{1}, statedb.GetCode(codeAccount))
+
+	// Changes should be available under the new state root after committing
+	newRoot, err := statedb.Commit(false)
+	require.NoError(t, err)
+	err = statedb.Database().TrieDB().Commit(newRoot, true)
+	require.NoError(t, err)
+
+	statedb, err = state.New(newRoot, state.NewDatabase(rawdb.NewDatabase(db)), nil)
+	require.NoError(t, err)
+	require.Equal(t, big.NewInt(50), statedb.GetBalance(userAccount))
+	require.Equal(t, uint64(5), statedb.GetNonce(userAccount))
+	require.Equal(t, big.NewInt(60), statedb.GetBalance(unknownAccount))
+	require.Equal(t, []byte{1}, statedb.GetCode(codeAccount))
+}
+
+func createGenesis() *core.Genesis {
+	l2Genesis := &core.Genesis{
+		Config:     &params.ChainConfig{},
+		Difficulty: common.Big0,
+		ParentHash: common.Hash{},
+		BaseFee:    big.NewInt(7),
+		Alloc: map[common.Address]core.GenesisAccount{
+			userAccount: {
+				Balance: big.NewInt(1_000_000_000_000_000_000),
+				Nonce:   10,
+			},
+			codeAccount: {
+				Balance: big.NewInt(100),
+				Code:    []byte{5, 7, 8, 3, 4},
+				Storage: map[common.Hash]common.Hash{
+					common.HexToHash("0x01"): common.HexToHash("0x11"),
+					common.HexToHash("0x02"): common.HexToHash("0x12"),
+					common.HexToHash("0x03"): common.HexToHash("0x13"),
+				},
+			},
+		},
+	}
+	return l2Genesis
+}
+
+func assertStateDataAvailable(t *testing.T, db ethdb.KeyValueStore, l2Genesis *core.Genesis, genesisBlock *types.Block) {
+	statedb, err := state.New(genesisBlock.Root(), state.NewDatabase(rawdb.NewDatabase(db)), nil)
+	require.NoError(t, err)
+
+	for address, account := range l2Genesis.Alloc {
+		require.Equal(t, account.Balance, statedb.GetBalance(address))
+		require.Equal(t, account.Nonce, statedb.GetNonce(address))
+		require.Equal(t, common.BytesToHash(crypto.Keccak256(account.Code)), statedb.GetCodeHash(address))
+		require.Equal(t, account.Code, statedb.GetCode(address))
+		for key, value := range account.Storage {
+			require.Equal(t, value, statedb.GetState(address, key))
+		}
+	}
+	require.Equal(t, common.Hash{}, statedb.GetState(codeAccount, common.HexToHash("0x99")), "retrieve unset storage key")
+	require.Equal(t, common.Big0, statedb.GetBalance(unknownAccount), "unset account balance")
+	require.Equal(t, uint64(0), statedb.GetNonce(unknownAccount), "unset account balance")
+	require.Nil(t, statedb.GetCode(unknownAccount), "unset account code")
+	require.Equal(t, common.Hash{}, statedb.GetCodeHash(unknownAccount), "unset account code hash")
+}
+
+func newStubStateOracle() *stubStateOracle {
+	return &stubStateOracle{
+		data: make(map[common.Hash][]byte),
+		code: make(map[common.Hash][]byte),
+	}
+}
+
+type stubStateOracle struct {
+	data map[common.Hash][]byte
+	code map[common.Hash][]byte
+}
+
+func (o *stubStateOracle) NodeByHash(nodeHash common.Hash) ([]byte, error) {
+	data, ok := o.data[nodeHash]
+	if !ok {
+		return nil, fmt.Errorf("no value for node %v", nodeHash)
+	}
+	return data, nil
+}
+
+func (o *stubStateOracle) CodeByHash(hash common.Hash) ([]byte, error) {
+	data, ok := o.code[hash]
+	if !ok {
+		return nil, fmt.Errorf("no value for code %v", hash)
+	}
+	return data, nil
+}
+
+// kvStateOracle loads data from a source ethdb.KeyValueStore
+type kvStateOracle struct {
+	source ethdb.KeyValueStore
+}
+
+func (o *kvStateOracle) NodeByHash(nodeHash common.Hash) ([]byte, error) {
+	return o.source.Get(nodeHash.Bytes())
+}
+
+func (o *kvStateOracle) CodeByHash(hash common.Hash) ([]byte, error) {
+	return rawdb.ReadCode(o.source, hash), nil
+}

op-program/l2/fetcher.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/log"
@@ -21,35 +22,52 @@ type CallContext interface {
 }
 
 type FetchingL2Oracle struct {
+	ctx         context.Context
 	logger      log.Logger
 	blockSource BlockSource
 	callContext CallContext
 }
 
-func NewFetchingL2Oracle(logger log.Logger, l2Url string) (*FetchingL2Oracle, error) {
+func NewFetchingL2Oracle(ctx context.Context, logger log.Logger, l2Url string) (*FetchingL2Oracle, error) {
 	rpcClient, err := rpc.Dial(l2Url)
 	if err != nil {
 		return nil, err
 	}
 	ethClient := ethclient.NewClient(rpcClient)
 	return &FetchingL2Oracle{
+		ctx:         ctx,
 		logger:      logger,
 		blockSource: ethClient,
 		callContext: rpcClient,
 	}, nil
 }
 
-func (s FetchingL2Oracle) NodeByHash(ctx context.Context, nodeHash common.Hash) ([]byte, error) {
+func (o *FetchingL2Oracle) NodeByHash(hash common.Hash) ([]byte, error) {
+	// MPT nodes are stored as the hash of the node (with no prefix)
+	return o.dbGet(hash.Bytes())
+}
+
+func (o *FetchingL2Oracle) CodeByHash(hash common.Hash) ([]byte, error) {
+	// First try retrieving with the new code prefix
+	code, err := o.dbGet(append(rawdb.CodePrefix, hash.Bytes()...))
+	if err != nil {
+		// Fallback to the legacy un-prefixed version
+		return o.dbGet(hash.Bytes())
+	}
+	return code, nil
+}
+
+func (o *FetchingL2Oracle) dbGet(key []byte) ([]byte, error) {
 	var node hexutil.Bytes
-	err := s.callContext.CallContext(ctx, &node, "debug_dbGet", nodeHash.Hex())
+	err := o.callContext.CallContext(o.ctx, &node, "debug_dbGet", hexutil.Encode(key))
 	if err != nil {
-		return nil, fmt.Errorf("fetch node %s: %w", nodeHash.Hex(), err)
+		return nil, fmt.Errorf("fetch node %s: %w", hexutil.Encode(key), err)
 	}
 	return node, nil
 }
 
-func (s FetchingL2Oracle) BlockByHash(ctx context.Context, blockHash common.Hash) (*types.Block, error) {
-	block, err := s.blockSource.BlockByHash(ctx, blockHash)
+func (o *FetchingL2Oracle) BlockByHash(blockHash common.Hash) (*types.Block, error) {
+	block, err := o.blockSource.BlockByHash(o.ctx, blockHash)
 	if err != nil {
 		return nil, fmt.Errorf("fetch block %s: %w", blockHash.Hex(), err)
 	}

op-program/l2/fetcher_test.go

@@ -12,11 +12,15 @@ import (
 	"github.com/ethereum-optimism/optimism/op-node/testutils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/stretchr/testify/require"
 )
 
+// Require the fetching oracle to implement StateOracle
+var _ StateOracle = (*FetchingL2Oracle)(nil)
+
 type callContextRequest struct {
 	ctx    context.Context
 	method string
@@ -50,7 +54,6 @@ func (c *stubCallContext) CallContext(ctx context.Context, result any, method st
 func TestNodeByHash(t *testing.T) {
 	rng := rand.New(rand.NewSource(1234))
 	hash := testutils.RandomHash(rng)
-	ctx := context.Background()
 
 	t.Run("Error", func(t *testing.T) {
 		stub := &stubCallContext{
@@ -58,7 +61,7 @@ func TestNodeByHash(t *testing.T) {
 		}
 		fetcher := newFetcher(nil, stub)
 
-		node, err := fetcher.NodeByHash(ctx, hash)
+		node, err := fetcher.NodeByHash(hash)
 		require.ErrorIs(t, err, stub.nextErr)
 		require.Nil(t, node)
 	})
@@ -70,7 +73,7 @@ func TestNodeByHash(t *testing.T) {
 		}
 		fetcher := newFetcher(nil, stub)
 
-		node, err := fetcher.NodeByHash(ctx, hash)
+		node, err := fetcher.NodeByHash(hash)
 		require.NoError(t, err)
 		require.EqualValues(t, expected, node)
 	})
@@ -81,7 +84,7 @@ func TestNodeByHash(t *testing.T) {
 		}
 		fetcher := newFetcher(nil, stub)
 
-		_, _ = fetcher.NodeByHash(ctx, hash)
+		_, _ = fetcher.NodeByHash(hash)
 		require.Len(t, stub.requests, 1, "should make single request")
 		req := stub.requests[0]
 		require.Equal(t, "debug_dbGet", req.method)
@@ -89,6 +92,67 @@ func TestNodeByHash(t *testing.T) {
 	})
 }
 
+func TestCodeByHash(t *testing.T) {
+	rng := rand.New(rand.NewSource(1234))
+	hash := testutils.RandomHash(rng)
+
+	t.Run("Error", func(t *testing.T) {
+		stub := &stubCallContext{
+			nextErr: errors.New("oops"),
+		}
+		fetcher := newFetcher(nil, stub)
+
+		node, err := fetcher.CodeByHash(hash)
+		require.ErrorIs(t, err, stub.nextErr)
+		require.Nil(t, node)
+	})
+
+	t.Run("Success", func(t *testing.T) {
+		expected := (hexutil.Bytes)([]byte{12, 34})
+		stub := &stubCallContext{
+			nextResult: expected,
+		}
+		fetcher := newFetcher(nil, stub)
+
+		node, err := fetcher.CodeByHash(hash)
+		require.NoError(t, err)
+		require.EqualValues(t, expected, node)
+	})
+
+	t.Run("RequestArgs", func(t *testing.T) {
+		stub := &stubCallContext{
+			nextResult: (hexutil.Bytes)([]byte{12, 34}),
+		}
+		fetcher := newFetcher(nil, stub)
+
+		_, _ = fetcher.CodeByHash(hash)
+		require.Len(t, stub.requests, 1, "should make single request")
+		req := stub.requests[0]
+		require.Equal(t, "debug_dbGet", req.method)
+		codeDbKey := append(rawdb.CodePrefix, hash.Bytes()...)
+		require.Equal(t, []interface{}{hexutil.Encode(codeDbKey)}, req.args)
+	})
+
+	t.Run("FallbackToUnprefixed", func(t *testing.T) {
+		stub := &stubCallContext{
+			nextErr: errors.New("not found"),
+		}
+		fetcher := newFetcher(nil, stub)
+
+		_, _ = fetcher.CodeByHash(hash)
+		require.Len(t, stub.requests, 2, "should request with and without prefix")
+		req := stub.requests[0]
+		require.Equal(t, "debug_dbGet", req.method)
+		codeDbKey := append(rawdb.CodePrefix, hash.Bytes()...)
+		require.Equal(t, []interface{}{hexutil.Encode(codeDbKey)}, req.args)
+
+		req = stub.requests[1]
+		require.Equal(t, "debug_dbGet", req.method)
+		codeDbKey = hash.Bytes()
+		require.Equal(t, []interface{}{hexutil.Encode(codeDbKey)}, req.args)
+	})
+}
+
 type blockRequest struct {
 	ctx       context.Context
 	blockHash common.Hash
@@ -111,14 +175,13 @@ func (s *stubBlockSource) BlockByHash(ctx context.Context, blockHash common.Hash
 func TestBlockByHash(t *testing.T) {
 	rng := rand.New(rand.NewSource(1234))
 	hash := testutils.RandomHash(rng)
-	ctx := context.Background()
 
 	t.Run("Success", func(t *testing.T) {
 		block, _ := testutils.RandomBlock(rng, 1)
 		stub := &stubBlockSource{nextResult: block}
 		fetcher := newFetcher(stub, nil)
 
-		res, err := fetcher.BlockByHash(ctx, hash)
+		res, err := fetcher.BlockByHash(hash)
 		require.NoError(t, err)
 		require.Same(t, block, res)
 	})
@@ -127,7 +190,7 @@ func TestBlockByHash(t *testing.T) {
 		stub := &stubBlockSource{nextErr: errors.New("boom")}
 		fetcher := newFetcher(stub, nil)
 
-		res, err := fetcher.BlockByHash(ctx, hash)
+		res, err := fetcher.BlockByHash(hash)
 		require.ErrorIs(t, err, stub.nextErr)
 		require.Nil(t, res)
 	})
@@ -136,7 +199,7 @@ func TestBlockByHash(t *testing.T) {
 		stub := &stubBlockSource{}
 		fetcher := newFetcher(stub, nil)
 
-		_, _ = fetcher.BlockByHash(ctx, hash)
+		_, _ = fetcher.BlockByHash(hash)
 
 		require.Len(t, stub.requests, 1, "should make single request")
 		req := stub.requests[0]

op-program/l2/oracle.go

+package l2
+
+import "github.com/ethereum/go-ethereum/common"
+
+// StateOracle defines the high-level API used to retrieve L2 state data pre-images
+// The returned data is always the preimage of the requested hash.
+type StateOracle interface {
+	// NodeByHash retrieves the merkle-patricia trie node pre-image for a given hash.
+	// Trie nodes may be from the world state trie or any account storage trie.
+	// Contract code is not stored as part of the trie and must be retrieved via CodeByHash
+	// Returns an error if the pre-image is unavailable.
+	NodeByHash(nodeHash common.Hash) ([]byte, error)
+
+	// CodeByHash retrieves the contract code pre-image for a given hash.
+	// codeHash should be retrieved from the world state account for a contract.
+	// Returns an error if the pre-image is unavailable.
+	CodeByHash(codeHash common.Hash) ([]byte, error)
+}