op-program: add experimental L2 source flag to enable execution witnesses (#12558)

* feat: add experimental L2 source flag

* Move L2 experimental client methods to eth client

* Fix missing return

* Improve config handling for experimental features

* Add l2 experimental test

op-e2e/actions/proofs/helpers/env.go

@@ -195,7 +195,7 @@ func (env *L2FaultProofEnv) RunFaultProofProgram(t helpers.Testing, l2ClaimBlock
 			l2RPC := env.Engine.RPCClient()
 			l2Client, err := host.NewL2Client(l2RPC, env.log, nil, &host.L2ClientConfig{L2ClientConfig: l2ClCfg, L2Head: cfg.L2Head})
 			require.NoError(t, err, "failed to create L2 client")
-			l2DebugCl := &host.L2Source{L2Client: l2Client, DebugClient: sources.NewDebugClient(l2RPC.CallContext)}
+			l2DebugCl := host.NewL2SourceWithClient(logger, l2Client, sources.NewDebugClient(l2RPC.CallContext))
 
 			return prefetcher.NewPrefetcher(logger, l1Cl, l1BlobFetcher, l2DebugCl, kv), nil
 		})

op-program/host/cmd/main_test.go

@@ -274,6 +274,19 @@ func TestL2Claim(t *testing.T) {
 	})
 }
 
+func TestL2Experimental(t *testing.T) {
+	t.Run("DefaultEmpty", func(t *testing.T) {
+		cfg := configForArgs(t, addRequiredArgs())
+		require.Equal(t, cfg.L2ExperimentalURL, "")
+	})
+
+	t.Run("Valid", func(t *testing.T) {
+		expected := "https://example.com:8545"
+		cfg := configForArgs(t, replaceRequiredArg("--l2.experimental", expected))
+		require.EqualValues(t, expected, cfg.L2ExperimentalURL)
+	})
+}
+
 func TestL2BlockNumber(t *testing.T) {
 	t.Run("Required", func(t *testing.T) {
 		verifyArgsInvalid(t, "flag l2.blocknumber is required", addRequiredArgsExcept("--l2.blocknumber"))

op-program/host/config/config.go

@@ -56,7 +56,11 @@ type Config struct {
 	L2Head common.Hash
 	// L2OutputRoot is the agreed L2 output root to start derivation from
 	L2OutputRoot common.Hash
-	L2URL        string
+	// L2URL is the URL of the L2 node to fetch L2 data from, this is the canonical URL for L2 data
+	// This URL is used as a fallback for L2ExperimentalURL if the experimental URL fails or cannot retrieve the desired data
+	L2URL string
+	// L2ExperimentalURL is the URL of the L2 node (non hash db archival node, for example, reth archival node) to fetch L2 data from
+	L2ExperimentalURL string
 	// L2Claim is the claimed L2 output root to verify
 	L2Claim common.Hash
 	// L2ClaimBlockNumber is the block number the claimed L2 output root is from
@@ -218,6 +222,7 @@ func NewConfigFromCLI(log log.Logger, ctx *cli.Context) (*Config, error) {
 		DataDir:             ctx.String(flags.DataDir.Name),
 		DataFormat:          dbFormat,
 		L2URL:               ctx.String(flags.L2NodeAddr.Name),
+		L2ExperimentalURL:   ctx.String(flags.L2NodeExperimentalAddr.Name),
 		L2ChainConfig:       l2ChainConfig,
 		L2Head:              l2Head,
 		L2OutputRoot:        l2OutputRoot,

op-program/host/flags/flags.go

@@ -47,6 +47,11 @@ var (
 		Usage:   "Address of L2 JSON-RPC endpoint to use (eth and debug namespace required)",
 		EnvVars: prefixEnvVars("L2_RPC"),
 	}
+	L2NodeExperimentalAddr = &cli.StringFlag{
+		Name:    "l2.experimental",
+		Usage:   "Address of L2 JSON-RPC endpoint to use for experimental features (debug_executionWitness)",
+		EnvVars: prefixEnvVars("L2_RPC_EXPERIMENTAL_RPC"),
+	}
 	L1Head = &cli.StringFlag{
 		Name:    "l1.head",
 		Usage:   "Hash of the L1 head block. Derivation stops after this block is processed.",
@@ -131,6 +136,7 @@ var programFlags = []cli.Flag{
 	DataDir,
 	DataFormat,
 	L2NodeAddr,
+	L2NodeExperimentalAddr,
 	L2GenesisPath,
 	L1NodeAddr,
 	L1BeaconAddr,

op-program/host/host.go

@@ -24,11 +24,6 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 )
 
-type L2Source struct {
-	*L2Client
-	*sources.DebugClient
-}
-
 type Prefetcher interface {
 	Hint(hint string) error
 	GetPreimage(ctx context.Context, key common.Hash) ([]byte, error)
@@ -235,26 +230,23 @@ func makeDefaultPrefetcher(ctx context.Context, logger log.Logger, kv kvstore.KV
 		return nil, fmt.Errorf("failed to setup L1 RPC: %w", err)
 	}
 
-	logger.Info("Connecting to L2 node", "l2", cfg.L2URL)
-	l2RPC, err := client.NewRPC(ctx, logger, cfg.L2URL, client.WithDialAttempts(10))
-	if err != nil {
-		return nil, fmt.Errorf("failed to setup L2 RPC: %w", err)
-	}
-
 	l1ClCfg := sources.L1ClientDefaultConfig(cfg.Rollup, cfg.L1TrustRPC, cfg.L1RPCKind)
-	l2ClCfg := sources.L2ClientDefaultConfig(cfg.Rollup, true)
 	l1Cl, err := sources.NewL1Client(l1RPC, logger, nil, l1ClCfg)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create L1 client: %w", err)
 	}
+
+	logger.Info("Connecting to L1 beacon", "l1", cfg.L1BeaconURL)
 	l1Beacon := sources.NewBeaconHTTPClient(client.NewBasicHTTPClient(cfg.L1BeaconURL, logger))
 	l1BlobFetcher := sources.NewL1BeaconClient(l1Beacon, sources.L1BeaconClientConfig{FetchAllSidecars: false})
-	l2Cl, err := NewL2Client(l2RPC, logger, nil, &L2ClientConfig{L2ClientConfig: l2ClCfg, L2Head: cfg.L2Head})
+
+	logger.Info("Initializing L2 clients")
+	l2Client, err := NewL2Source(ctx, logger, cfg)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create L2 client: %w", err)
+		return nil, fmt.Errorf("failed to create L2 source: %w", err)
 	}
-	l2DebugCl := &L2Source{L2Client: l2Cl, DebugClient: sources.NewDebugClient(l2RPC.CallContext)}
-	return prefetcher.NewPrefetcher(logger, l1Cl, l1BlobFetcher, l2DebugCl, kv), nil
+
+	return prefetcher.NewPrefetcher(logger, l1Cl, l1BlobFetcher, l2Client, kv), nil
 }
 
 func routeHints(logger log.Logger, hHostRW io.ReadWriter, hinter preimage.HintHandler) chan error {

op-program/host/l2_source.go

+package host
+
+import (
+	"context"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-program/host/config"
+	"github.com/ethereum-optimism/optimism/op-program/host/prefetcher"
+	"github.com/ethereum-optimism/optimism/op-service/client"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/ethereum-optimism/optimism/op-service/sources"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// L2Source is a source of L2 data, it abstracts away the details of how to fetch L2 data between canonical and experimental sources.
+// It also tracks metrics for each of the apis. Once experimental sources are stable, this will only route to the "experimental" source.
+type L2Source struct {
+	logger log.Logger
+
+	// canonical source, used as a fallback if experimental source is enabled but fails
+	// the underlying node should be a geth hash scheme archival node.
+	canonicalEthClient   *L2Client
+	canonicalDebugClient *sources.DebugClient
+
+	// experimental source, used as the primary source if enabled
+	experimentalClient *L2Client
+}
+
+var _ prefetcher.L2Source = &L2Source{}
+
+// NewL2SourceWithClient creates a new L2 source with the given client as the canonical client.
+// This doesn't configure the experimental source, but is useful for testing.
+func NewL2SourceWithClient(logger log.Logger, canonicalL2Client *L2Client, canonicalDebugClient *sources.DebugClient) *L2Source {
+	source := &L2Source{
+		logger:               logger,
+		canonicalEthClient:   canonicalL2Client,
+		canonicalDebugClient: canonicalDebugClient,
+	}
+
+	return source
+}
+
+func NewL2Source(ctx context.Context, logger log.Logger, config *config.Config) (*L2Source, error) {
+	logger.Info("Connecting to canonical L2 source", "url", config.L2URL)
+
+	// eth_getProof calls are expensive and takes time, so we use a longer timeout
+	canonicalL2RPC, err := client.NewRPC(ctx, logger, config.L2URL, client.WithDialAttempts(10), client.WithCallTimeout(5*time.Minute))
+	if err != nil {
+		return nil, err
+	}
+	canonicalDebugClient := sources.NewDebugClient(canonicalL2RPC.CallContext)
+
+	canonicalL2ClientCfg := sources.L2ClientDefaultConfig(config.Rollup, true)
+	canonicalL2Client, err := NewL2Client(canonicalL2RPC, logger, nil, &L2ClientConfig{L2ClientConfig: canonicalL2ClientCfg, L2Head: config.L2Head})
+	if err != nil {
+		return nil, err
+	}
+
+	source := NewL2SourceWithClient(logger, canonicalL2Client, canonicalDebugClient)
+
+	if len(config.L2ExperimentalURL) == 0 {
+		return source, nil
+	}
+
+	logger.Info("Connecting to experimental L2 source", "url", config.L2ExperimentalURL)
+	// debug_executionWitness calls are expensive and takes time, so we use a longer timeout
+	experimentalRPC, err := client.NewRPC(ctx, logger, config.L2ExperimentalURL, client.WithDialAttempts(10), client.WithCallTimeout(5*time.Minute))
+	if err != nil {
+		return nil, err
+	}
+	experimentalL2ClientCfg := sources.L2ClientDefaultConfig(config.Rollup, true)
+	experimentalL2Client, err := NewL2Client(experimentalRPC, logger, nil, &L2ClientConfig{L2ClientConfig: experimentalL2ClientCfg, L2Head: config.L2Head})
+	if err != nil {
+		return nil, err
+	}
+
+	source.experimentalClient = experimentalL2Client
+
+	return source, nil
+}
+
+func (l *L2Source) ExperimentalEnabled() bool {
+	return l.experimentalClient != nil
+}
+
+// CodeByHash implements prefetcher.L2Source.
+func (l *L2Source) CodeByHash(ctx context.Context, hash common.Hash) ([]byte, error) {
+	if l.ExperimentalEnabled() {
+		// This means experimental source was not able to retrieve relevant information from eth_getProof or debug_executionWitness
+		// We should fall back to the canonical source, and log a warning, and record a metric
+		l.logger.Warn("Experimental source failed to retrieve code by hash, falling back to canonical source", "hash", hash)
+	}
+	return l.canonicalDebugClient.CodeByHash(ctx, hash)
+}
+
+// NodeByHash implements prefetcher.L2Source.
+func (l *L2Source) NodeByHash(ctx context.Context, hash common.Hash) ([]byte, error) {
+	if l.ExperimentalEnabled() {
+		// This means experimental source was not able to retrieve relevant information from eth_getProof or debug_executionWitness
+		// We should fall back to the canonical source, and log a warning, and record a metric
+		l.logger.Warn("Experimental source failed to retrieve node by hash, falling back to canonical source", "hash", hash)
+	}
+	return l.canonicalDebugClient.NodeByHash(ctx, hash)
+}
+
+// InfoAndTxsByHash implements prefetcher.L2Source.
+func (l *L2Source) InfoAndTxsByHash(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, types.Transactions, error) {
+	if l.ExperimentalEnabled() {
+		return l.experimentalClient.InfoAndTxsByHash(ctx, blockHash)
+	}
+	return l.canonicalEthClient.InfoAndTxsByHash(ctx, blockHash)
+}
+
+// OutputByRoot implements prefetcher.L2Source.
+func (l *L2Source) OutputByRoot(ctx context.Context, root common.Hash) (eth.Output, error) {
+	if l.ExperimentalEnabled() {
+		return l.experimentalClient.OutputByRoot(ctx, root)
+	}
+	return l.canonicalEthClient.OutputByRoot(ctx, root)
+}
+
+// ExecutionWitness implements prefetcher.L2Source.
+func (l *L2Source) ExecutionWitness(ctx context.Context, blockNum uint64) (*eth.ExecutionWitness, error) {
+	if !l.ExperimentalEnabled() {
+		l.logger.Error("Experimental source is not enabled, cannot fetch execution witness", "blockNum", blockNum)
+		return nil, prefetcher.ErrExperimentalPrefetchDisabled
+	}
+
+	// log errors, but return standard error so we know to retry with legacy source
+	witness, err := l.experimentalClient.ExecutionWitness(ctx, blockNum)
+	if err != nil {
+		l.logger.Error("Failed to fetch execution witness from experimental source", "blockNum", blockNum, "err", err)
+		return nil, prefetcher.ErrExperimentalPrefetchFailed
+	}
+	return witness, nil
+}
+
+// GetProof implements prefetcher.L2Source.
+func (l *L2Source) GetProof(ctx context.Context, address common.Address, storage []common.Hash, blockTag string) (*eth.AccountResult, error) {
+	if l.ExperimentalEnabled() {
+		return l.experimentalClient.GetProof(ctx, address, storage, blockTag)
+	}
+	proof, err := l.canonicalEthClient.GetProof(ctx, address, storage, blockTag)
+	if err != nil {
+		l.logger.Error("Failed to fetch proof from canonical source", "address", address, "storage", storage, "blockTag", blockTag, "err", err)
+		return nil, prefetcher.ErrExperimentalPrefetchFailed
+	}
+	return proof, nil
+}

op-program/host/prefetcher/prefetcher.go

@@ -28,6 +28,11 @@ var (
 	precompileFailure = [1]byte{0}
 )
 
+var (
+	ErrExperimentalPrefetchFailed   = errors.New("experimental prefetch failed")
+	ErrExperimentalPrefetchDisabled = errors.New("experimental prefetch disabled")
+)
+
 var acceleratedPrecompiles = []common.Address{
 	common.BytesToAddress([]byte{0x1}),  // ecrecover
 	common.BytesToAddress([]byte{0x8}),  // bn256Pairing

op-service/eth/execution_witness.go

+package eth
+
+import "github.com/ethereum/go-ethereum/common/hexutil"
+
+type ExecutionWitness struct {
+	Keys  map[string]hexutil.Bytes `json:"keys"`
+	Codes map[string]hexutil.Bytes `json:"codes"`
+	State map[string]hexutil.Bytes `json:"state"`
+}

op-service/sources/eth_client.go

@@ -315,6 +315,22 @@ func (s *EthClient) FetchReceipts(ctx context.Context, blockHash common.Hash) (e
 	return info, receipts, nil
 }
 
+// ExecutionWitness fetches execution witness data for a block number.
+func (s *EthClient) ExecutionWitness(ctx context.Context, blockNum uint64) (*eth.ExecutionWitness, error) {
+	var witness *eth.ExecutionWitness
+
+	err := s.client.CallContext(ctx, &witness, "debug_executionWitness", hexutil.EncodeUint64(blockNum), true)
+	if err != nil {
+		return nil, err
+	}
+
+	if witness == nil {
+		return nil, ethereum.NotFound
+	}
+
+	return witness, nil
+}
+
 // GetProof returns an account proof result, with any optional requested storage proofs.
 // The retrieval does sanity-check that storage proofs for the expected keys are present in the response,
 // but does not verify the result. Call accountResult.Verify(stateRoot) to verify the result.