Skip to content

N
nebula
Commit 373c5aa3 authored by protolambda's avatar protolambda
Browse files
Options

op-program: prefetcher now backed with rpc source interface

parent a7e24c93
Hide whitespace changes
Inline Side-by-side
Showing with 148 additions and 78 deletions
op-program/host/cmd/main.go
View file @ 373c5aa3
......@@ -8,7 +8,9 @@ import (
"os"
"github.com/ethereum-optimism/optimism/op-node/chaincfg"
"github.com/ethereum-optimism/optimism/op-node/client"
"github.com/ethereum-optimism/optimism/op-node/eth"
"github.com/ethereum-optimism/optimism/op-node/sources"
cldr "github.com/ethereum-optimism/optimism/op-program/client/driver"
"github.com/ethereum-optimism/optimism/op-program/host/config"
"github.com/ethereum-optimism/optimism/op-program/host/flags"
......@@ -97,6 +99,11 @@ func setupLogging(ctx *cli.Context) (log.Logger, error) {
return logger, nil
}
type L2Source struct {
*sources.L2Client
*sources.DebugClient
}
// FaultProofProgram is the programmatic entry-point for the fault proof program
func FaultProofProgram(logger log.Logger, cfg *config.Config) error {
cfg.Rollup.LogDescription(logger, chaincfg.L2ChainIDToNetworkName)
......@@ -108,18 +115,32 @@ func FaultProofProgram(logger log.Logger, cfg *config.Config) error {
kv := kvstore.NewMemKV()
logger.Info("Connecting to L1 node", "l1", cfg.L1URL)
l1Fetcher, err := l1.NewFetchingOracle(ctx, logger, cfg)
l1RPC, err := client.NewRPC(ctx, logger, cfg.L1URL)
if err != nil {
return fmt.Errorf("connect l1 fetcher: %w", err)
return fmt.Errorf("failed to setup L1 RPC: %w", err)
}
logger.Info("Connecting to L2 node", "l2", cfg.L2URL)
l2Fetcher, err := l2.NewFetchingOracle(ctx, logger, cfg)
l2RPC, err := client.NewRPC(ctx, logger, cfg.L2URL)
if err != nil {
return fmt.Errorf("failed to setup L2 RPC: %w", err)
}
l1ClCfg := sources.L1ClientDefaultConfig(cfg.Rollup, cfg.L1TrustRPC, cfg.L1RPCKind)
l2ClCfg := sources.L2ClientDefaultConfig(cfg.Rollup, true)
l1Cl, err := sources.NewL1Client(l1RPC, logger, nil, l1ClCfg)
if err != nil {
return fmt.Errorf("failed to create L1 client: %w", err)
}
l2Cl, err := sources.NewL2Client(l2RPC, logger, nil, l2ClCfg)
if err != nil {
return fmt.Errorf("connect l2 fetcher: %w", err)
return fmt.Errorf("failed to create L2 client: %w", err)
}
prefetch := prefetcher.NewPrefetcher(l1Fetcher, l2Fetcher, kv)
preimageOracle := asOracleFn(prefetch)
l2DebugCl := &L2Source{L2Client: l2Cl, DebugClient: sources.NewDebugClient(l2RPC.CallContext)}
logger.Info("Setting up pre-fetcher")
prefetch := prefetcher.NewPrefetcher(l1Cl, l2DebugCl, kv)
preimageOracle := asOracleFn(ctx, prefetch)
hinter := asHinter(prefetch)
l1Source := l1.NewSource(logger, preimageOracle, hinter, cfg.L1Head)
......@@ -145,9 +166,9 @@ func FaultProofProgram(logger log.Logger, cfg *config.Config) error {
return nil
}
func asOracleFn(prefetcher *prefetcher.Prefetcher) preimage.OracleFn {
func asOracleFn(ctx context.Context, prefetcher *prefetcher.Prefetcher) preimage.OracleFn {
return func(key preimage.Key) []byte {
pre, err := prefetcher.GetPreimage(key.PreimageKey())
pre, err := prefetcher.GetPreimage(ctx, key.PreimageKey())
if err != nil {
panic(fmt.Errorf("preimage unavailable for key %v: %w", key, err))
}
......
op-program/host/prefetcher/prefetcher.go
View file @ 373c5aa3
package prefetcher
import (
"context"
"errors"
"fmt"
"strings"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum-optimism/optimism/op-node/eth"
"github.com/ethereum-optimism/optimism/op-program/client/l1"
"github.com/ethereum-optimism/optimism/op-program/client/l2"
"github.com/ethereum-optimism/optimism/op-program/client/mpt"
"github.com/ethereum-optimism/optimism/op-program/host/kvstore"
"github.com/ethereum-optimism/optimism/op-program/preimage"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
)
type L1Source interface {
InfoByHash(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, error)
InfoAndTxsByHash(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, types.Transactions, error)
FetchReceipts(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, types.Receipts, error)
}
type L2Source interface {
InfoAndTxsByHash(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, types.Transactions, error)
NodeByHash(ctx context.Context, hash common.Hash) ([]byte, error)
CodeByHash(ctx context.Context, hash common.Hash) ([]byte, error)
}
type Prefetcher struct {
l1Fetcher l1.Oracle
l2Fetcher l2.Oracle
l1Fetcher L1Source
l2Fetcher L2Source
lastHint string
kvStore kvstore.KV
}
func NewPrefetcher(l1Fetcher l1.Oracle, l2Fetcher l2.Oracle, kvStore kvstore.KV) *Prefetcher {
func NewPrefetcher(l1Fetcher L1Source, l2Fetcher L2Source, kvStore kvstore.KV) *Prefetcher {
return &Prefetcher{
l1Fetcher: l1Fetcher,
l2Fetcher: l2Fetcher,
......@@ -38,12 +51,12 @@ func (p *Prefetcher) Hint(hint string) error {
return nil
}
func (p *Prefetcher) GetPreimage(key common.Hash) ([]byte, error) {
func (p *Prefetcher) GetPreimage(ctx context.Context, key common.Hash) ([]byte, error) {
pre, err := p.kvStore.Get(key)
if errors.Is(err, kvstore.ErrNotFound) && p.lastHint != "" {
hint := p.lastHint
p.lastHint = ""
if err := p.prefetch(hint); err != nil {
if err := p.prefetch(ctx, hint); err != nil {
return nil, fmt.Errorf("prefetch failed: %w", err)
}
// Should now be available
......@@ -52,52 +65,72 @@ func (p *Prefetcher) GetPreimage(key common.Hash) ([]byte, error) {
return pre, err
}
func (p *Prefetcher) prefetch(hint string) error {
func (p *Prefetcher) prefetch(ctx context.Context, hint string) error {
hintType, hash, err := parseHint(hint)
if err != nil {
return err
}
switch hintType {
case l1.HintL1BlockHeader:
header := p.l1Fetcher.HeaderByBlockHash(hash)
header, err := p.l1Fetcher.InfoByHash(ctx, hash)
if err != nil {
return fmt.Errorf("failed to fetch L1 block %s header: %w", hash, err)
}
data, err := header.HeaderRLP()
if err != nil {
return fmt.Errorf("marshall header: %w", err)
}
return p.kvStore.Put(preimage.Keccak256Key(hash).PreimageKey(), data)
case l1.HintL1Transactions:
_, txs := p.l1Fetcher.TransactionsByBlockHash(hash)
_, txs, err := p.l1Fetcher.InfoAndTxsByHash(ctx, hash)
if err != nil {
return fmt.Errorf("failed to fetch L1 block %s txs: %w", hash, err)
}
return p.storeTransactions(txs)
case l1.HintL1Receipts:
_, rcpts := p.l1Fetcher.ReceiptsByBlockHash(hash)
opaqueRcpts, err := eth.EncodeReceipts(rcpts)
_, receipts, err := p.l1Fetcher.FetchReceipts(ctx, hash)
if err != nil {
return err
return fmt.Errorf("failed to fetch L1 block %s receipts: %w", hash, err)
}
return p.storeTrieNodes(opaqueRcpts)
return p.storeReceipts(receipts)
case l2.HintL2BlockHeader:
// Pre-fetch both block and transactions
block := p.l2Fetcher.BlockByHash(hash)
data, err := rlp.EncodeToBytes(block.Header())
header, txs, err := p.l2Fetcher.InfoAndTxsByHash(ctx, hash)
if err != nil {
return fmt.Errorf("marshall header: %w", err)
return fmt.Errorf("failed to fetch L2 block %s: %w", hash, err)
}
data, err := header.HeaderRLP()
if err != nil {
return fmt.Errorf("failed to encode header to RLP: %w", err)
}
err = p.kvStore.Put(preimage.Keccak256Key(hash).PreimageKey(), data)
if err != nil {
return err
}
return p.storeTransactions(block.Transactions())
return p.storeTransactions(txs)
case l2.HintL2StateNode:
node := p.l2Fetcher.NodeByHash(hash)
node, err := p.l2Fetcher.NodeByHash(ctx, hash)
if err != nil {
return fmt.Errorf("failed to fetch L2 state node %s: %w", hash, err)
}
return p.kvStore.Put(preimage.Keccak256Key(hash).PreimageKey(), node)
case l2.HintL2Code:
code := p.l2Fetcher.CodeByHash(hash)
code, err := p.l2Fetcher.CodeByHash(ctx, hash)
if err != nil {
return fmt.Errorf("failed to fetch L2 contract code %s: %w", hash, err)
}
return p.kvStore.Put(preimage.Keccak256Key(hash).PreimageKey(), code)
}
return fmt.Errorf("unknown hint type: %v", hintType)
}
func (p *Prefetcher) storeReceipts(receipts types.Receipts) error {
opaqueReceipts, err := eth.EncodeReceipts(receipts)
if err != nil {
return err
}
return p.storeTrieNodes(opaqueReceipts)
}
func (p *Prefetcher) storeTransactions(txs types.Transactions) error {
opaqueTxs, err := eth.EncodeTransactions(txs)
if err != nil {
......
op-program/host/prefetcher/prefetcher_test.go
View file @ 373c5aa3
package prefetcher
import (
"context"
"math/rand"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/stretchr/testify/require"
"github.com/ethereum-optimism/optimism/op-node/eth"
"github.com/ethereum-optimism/optimism/op-node/testutils"
"github.com/ethereum-optimism/optimism/op-program/client/l1"
l1test "github.com/ethereum-optimism/optimism/op-program/client/l1/test"
"github.com/ethereum-optimism/optimism/op-program/client/l2"
l2test "github.com/ethereum-optimism/optimism/op-program/client/l2/test"
"github.com/ethereum-optimism/optimism/op-program/client/mpt"
"github.com/ethereum-optimism/optimism/op-program/host/kvstore"
"github.com/ethereum-optimism/optimism/op-program/preimage"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/stretchr/testify/require"
)
func TestNoHint(t *testing.T) {
t.Run("NotFound", func(t *testing.T) {
prefetcher, _, _, _, _ := createPrefetcher(t)
res, err := prefetcher.GetPreimage(common.Hash{0xab})
prefetcher, _, _, _ := createPrefetcher(t)
res, err := prefetcher.GetPreimage(context.Background(), common.Hash{0xab})
require.ErrorIs(t, err, kvstore.ErrNotFound)
require.Nil(t, res)
})
t.Run("Exists", func(t *testing.T) {
prefetcher, _, _, _, kv := createPrefetcher(t)
prefetcher, _, _, kv := createPrefetcher(t)
data := []byte{1, 2, 3}
hash := crypto.Keccak256Hash(data)
require.NoError(t, kv.Put(hash, data))
res, err := prefetcher.GetPreimage(hash)
res, err := prefetcher.GetPreimage(context.Background(), hash)
require.NoError(t, err)
require.Equal(t, res, data)
})
......@@ -49,7 +49,7 @@ func TestFetchL1BlockHeader(t *testing.T) {
require.NoError(t, err)
t.Run("AlreadyKnown", func(t *testing.T) {
prefetcher, _, _, _, kv := createPrefetcher(t)
prefetcher, _, _, kv := createPrefetcher(t)
storeBlock(t, kv, block, rcpts)
oracle := l1.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
......@@ -58,10 +58,12 @@ func TestFetchL1BlockHeader(t *testing.T) {
})
t.Run("Unknown", func(t *testing.T) {
prefetcher, l1Oracle, _, _, _ := createPrefetcher(t)
l1Oracle.Blocks[hash] = eth.HeaderBlockInfo(block.Header())
prefetcher, l1Cl, _, _ := createPrefetcher(t)
l1Cl.ExpectInfoByHash(hash, eth.HeaderBlockInfo(block.Header()), nil)
defer l1Cl.AssertExpectations(t)
require.NoError(t, prefetcher.Hint(l1.BlockHeaderHint(hash).Hint()))
result, err := prefetcher.GetPreimage(key)
result, err := prefetcher.GetPreimage(context.Background(), key)
require.NoError(t, err)
require.Equal(t, pre, result)
})
......@@ -73,7 +75,7 @@ func TestFetchL1Transactions(t *testing.T) {
hash := block.Hash()
t.Run("AlreadyKnown", func(t *testing.T) {
prefetcher, _, _, _, kv := createPrefetcher(t)
prefetcher, _, _, kv := createPrefetcher(t)
storeBlock(t, kv, block, rcpts)
......@@ -85,9 +87,10 @@ func TestFetchL1Transactions(t *testing.T) {
})
t.Run("Unknown", func(t *testing.T) {
prefetcher, l1Oracle, _, _, _ := createPrefetcher(t)
l1Oracle.Blocks[hash] = eth.BlockToInfo(block)
l1Oracle.Txs[hash] = block.Transactions()
prefetcher, l1Cl, _, _ := createPrefetcher(t)
l1Cl.ExpectInfoByHash(hash, eth.BlockToInfo(block), nil)
l1Cl.ExpectInfoAndTxsByHash(hash, eth.BlockToInfo(block), block.Transactions(), nil)
defer l1Cl.AssertExpectations(t)
oracle := l1.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
header, txs := oracle.TransactionsByBlockHash(hash)
......@@ -102,8 +105,7 @@ func TestFetchL1Receipts(t *testing.T) {
hash := block.Hash()
t.Run("AlreadyKnown", func(t *testing.T) {
prefetcher, _, _, _, kv := createPrefetcher(t)
prefetcher, _, _, kv := createPrefetcher(t)
storeBlock(t, kv, block, receipts)
// Check the data is available (note the oracle does not know about the block, only the kvstore does)
......@@ -114,10 +116,11 @@ func TestFetchL1Receipts(t *testing.T) {
})
t.Run("Unknown", func(t *testing.T) {
prefetcher, l1Oracle, _, _, _ := createPrefetcher(t)
l1Oracle.Blocks[hash] = eth.BlockToInfo(block)
l1Oracle.Txs[hash] = block.Transactions()
l1Oracle.Rcpts[hash] = receipts
prefetcher, l1Cl, _, _ := createPrefetcher(t)
l1Cl.ExpectInfoByHash(hash, eth.BlockToInfo(block), nil)
l1Cl.ExpectInfoAndTxsByHash(hash, eth.BlockToInfo(block), block.Transactions(), nil)
l1Cl.ExpectFetchReceipts(hash, eth.BlockToInfo(block), receipts, nil)
defer l1Cl.AssertExpectations(t)
oracle := l1.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
header, actualReceipts := oracle.ReceiptsByBlockHash(hash)
......@@ -132,7 +135,7 @@ func TestFetchL2Block(t *testing.T) {
hash := block.Hash()
t.Run("AlreadyKnown", func(t *testing.T) {
prefetcher, _, _, _, kv := createPrefetcher(t)
prefetcher, _, _, kv := createPrefetcher(t)
storeBlock(t, kv, block, rcpts)
oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
......@@ -142,8 +145,9 @@ func TestFetchL2Block(t *testing.T) {
})
t.Run("Unknown", func(t *testing.T) {
prefetcher, _, l2BlockOracle, _, _ := createPrefetcher(t)
l2BlockOracle.Blocks[hash] = block
prefetcher, _, l2Cl, _ := createPrefetcher(t)
l2Cl.ExpectInfoAndTxsByHash(hash, eth.BlockToInfo(block), block.Transactions(), nil)
defer l2Cl.MockL2Client.AssertExpectations(t)
oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
result := oracle.BlockByHash(hash)
......@@ -159,7 +163,7 @@ func TestFetchL2Node(t *testing.T) {
key := preimage.Keccak256Key(hash).PreimageKey()
t.Run("AlreadyKnown", func(t *testing.T) {
prefetcher, _, _, _, kv := createPrefetcher(t)
prefetcher, _, _, kv := createPrefetcher(t)
require.NoError(t, kv.Put(key, node))
oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
......@@ -168,8 +172,9 @@ func TestFetchL2Node(t *testing.T) {
})
t.Run("Unknown", func(t *testing.T) {
prefetcher, _, _, l2StateOracle, _ := createPrefetcher(t)
l2StateOracle.Data[hash] = node
prefetcher, _, l2Cl, _ := createPrefetcher(t)
l2Cl.ExpectNodeByHash(hash, node, nil)
defer l2Cl.MockDebugClient.AssertExpectations(t)
oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
result := oracle.NodeByHash(hash)
......@@ -184,7 +189,7 @@ func TestFetchL2Code(t *testing.T) {
key := preimage.Keccak256Key(hash).PreimageKey()
t.Run("AlreadyKnown", func(t *testing.T) {
prefetcher, _, _, _, kv := createPrefetcher(t)
prefetcher, _, _, kv := createPrefetcher(t)
require.NoError(t, kv.Put(key, code))
oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
......@@ -193,8 +198,9 @@ func TestFetchL2Code(t *testing.T) {
})
t.Run("Unknown", func(t *testing.T) {
prefetcher, _, _, l2StateOracle, _ := createPrefetcher(t)
l2StateOracle.Code[hash] = code
prefetcher, _, l2Cl, _ := createPrefetcher(t)
l2Cl.ExpectCodeByHash(hash, code, nil)
defer l2Cl.MockDebugClient.AssertExpectations(t)
oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
result := oracle.CodeByHash(hash)
......@@ -203,7 +209,7 @@ func TestFetchL2Code(t *testing.T) {
}
func TestBadHints(t *testing.T) {
prefetcher, _, _, _, kv := createPrefetcher(t)
prefetcher, _, _, kv := createPrefetcher(t)
hash := common.Hash{0xad}
t.Run("NoSpace", func(t *testing.T) {
......@@ -211,7 +217,7 @@ func TestBadHints(t *testing.T) {
require.NoError(t, prefetcher.Hint(l1.HintL1BlockHeader))
// But it will fail to prefetch when the pre-image isn't available
pre, err := prefetcher.GetPreimage(hash)
pre, err := prefetcher.GetPreimage(context.Background(), hash)
require.ErrorContains(t, err, "unsupported hint")
require.Nil(t, pre)
})
......@@ -221,7 +227,7 @@ func TestBadHints(t *testing.T) {
require.NoError(t, prefetcher.Hint(l1.HintL1BlockHeader+" asdfsadf"))
// But it will fail to prefetch when the pre-image isn't available
pre, err := prefetcher.GetPreimage(hash)
pre, err := prefetcher.GetPreimage(context.Background(), hash)
require.ErrorContains(t, err, "invalid hash")
require.Nil(t, pre)
})
......@@ -231,7 +237,7 @@ func TestBadHints(t *testing.T) {
require.NoError(t, prefetcher.Hint("unknown "+hash.Hex()))
// But it will fail to prefetch when the pre-image isn't available
pre, err := prefetcher.GetPreimage(hash)
pre, err := prefetcher.GetPreimage(context.Background(), hash)
require.ErrorContains(t, err, "unknown hint type")
require.Nil(t, pre)
})
......@@ -245,18 +251,28 @@ func TestBadHints(t *testing.T) {
// Hint is invalid
require.NoError(t, prefetcher.Hint("asdfsadf"))
// But fetching the key fails because prefetching isn't required
pre, err := prefetcher.GetPreimage(hash)
pre, err := prefetcher.GetPreimage(context.Background(), hash)
require.NoError(t, err)
require.Equal(t, value, pre)
})
}
func createPrefetcher(t *testing.T) (*Prefetcher, *l1test.StubOracle, *l2test.StubBlockOracle, *l2test.StubStateOracle, kvstore.KV) {
type l2Client struct {
*testutils.MockL2Client
*testutils.MockDebugClient
}
func createPrefetcher(t *testing.T) (*Prefetcher, *testutils.MockL1Source, *l2Client, kvstore.KV) {
kv := kvstore.NewMemKV()
l1Oracle := l1test.NewStubOracle(t)
l2BlockOracle, l2StateOracle := l2test.NewStubOracle(t)
prefetcher := NewPrefetcher(l1Oracle, l2BlockOracle, kv)
return prefetcher, l1Oracle, l2BlockOracle, l2StateOracle, kv
l1Source := new(testutils.MockL1Source)
l2Source := &l2Client{
MockL2Client: new(testutils.MockL2Client),
MockDebugClient: new(testutils.MockDebugClient),
}
prefetcher := NewPrefetcher(l1Source, l2Source, kv)
return prefetcher, l1Source, l2Source, kv
}
func storeBlock(t *testing.T, kv kvstore.KV, block *types.Block, receipts types.Receipts) {
......@@ -284,7 +300,7 @@ func storeBlock(t *testing.T, kv kvstore.KV, block *types.Block, receipts types.
func asOracleFn(t *testing.T, prefetcher *Prefetcher) preimage.OracleFn {
return func(key preimage.Key) []byte {
pre, err := prefetcher.GetPreimage(key.PreimageKey())
pre, err := prefetcher.GetPreimage(context.Background(), key.PreimageKey())
require.NoError(t, err)
return pre
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment