Merge branch 'develop' into clabby/ci/fix-ctb-tests

.circleci/config.yml

@@ -60,7 +60,7 @@ commands:
 jobs:
   yarn-monorepo:
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     resource_class: large
     steps:
       - checkout
@@ -317,7 +317,7 @@ jobs:
 
   contracts-bedrock-tests:
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     resource_class: large
     steps:
       - checkout
@@ -341,7 +341,7 @@ jobs:
 
   contracts-bedrock-checks:
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     steps:
       - checkout
       - attach_workspace: { at: "." }
@@ -401,7 +401,7 @@ jobs:
 
   contracts-bedrock-slither:
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     resource_class: large
     steps:
       - checkout
@@ -421,7 +421,7 @@ jobs:
 
   contracts-bedrock-validate-spaces:
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     steps:
       - checkout
       - attach_workspace: { at: "." }
@@ -438,7 +438,7 @@ jobs:
 
   bedrock-echidna-build:
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     steps:
       - checkout
       - attach_workspace: { at: "." }
@@ -456,7 +456,7 @@ jobs:
 
   bedrock-echidna-run:
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     parameters:
       echidna_target:
         description: Which echidna fuzz contract to run
@@ -483,7 +483,7 @@ jobs:
 
   op-bindings-build:
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     resource_class: medium
     steps:
       - checkout
@@ -512,7 +512,7 @@ jobs:
         description: Coverage flag name
         type: string
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     resource_class: large
     steps:
       - checkout
@@ -558,7 +558,7 @@ jobs:
 
   fuzz-op-node:
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     steps:
       - checkout
       - check-changed:
@@ -570,7 +570,7 @@ jobs:
 
   depcheck:
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     steps:
       - checkout
       - attach_workspace: { at: "." }
@@ -632,7 +632,7 @@ jobs:
         description: Go Module Name
         type: string
     docker:
-      - image: ethereumoptimism/ci-builder:latest # only used to enable codecov.
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest # only used to enable codecov.
     resource_class: xlarge
     steps:
       - checkout
@@ -660,7 +660,7 @@ jobs:
         description: If the op-e2e package should use HTTP clients
         type: string
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     resource_class: xlarge
     steps:
       - checkout
@@ -699,7 +699,7 @@ jobs:
         type: string
         default: this-package-does-not-exist
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
       - image: cimg/postgres:14.1
     steps:
       - checkout
@@ -728,7 +728,7 @@ jobs:
 
   geth-tests:
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     steps:
       - checkout
       - check-changed:
@@ -948,7 +948,7 @@ jobs:
 
   go-mod-tidy:
     docker:
-      - image: ethereumoptimism/ci-builder:latest
+      - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
     steps:
       - checkout
       - run:
@@ -1397,3 +1397,18 @@ workflows:
             - oplabs-gcr-release
           requires:
             - hold
+  release-ci-builder:
+    jobs:
+      - docker-publish:
+          name: ci-builder-docker-publish
+          filters:
+            tags:
+              only: /^ci-builder\/v.*/
+            branches:
+              ignore: /.*/
+          docker_file: ./ops/docker/ci-builder/Dockerfile
+          docker_name: ci-builder
+          docker_tags: <<pipeline.git.revision>>,latest
+          docker_context: ./ops/docker/ci-builder
+          context:
+            - oplabs-gcr
\ No newline at end of file

.github/workflows/release.yml

@@ -27,7 +27,6 @@ jobs:
       op-exporter: ${{ steps.packages.outputs.op-exporter }}
       l2geth-exporter: ${{ steps.packages.outputs.l2geth-exporter }}
       batch-submitter-service: ${{ steps.packages.outputs.batch-submitter-service }}
-      ci-builder: ${{ steps.packages.outputs.ci-builder }}
       foundry: ${{ steps.packages.outputs.foundry }}
       endpoint-monitor: ${{ steps.packages.outputs.endpoint-monitor }}
 
@@ -159,32 +158,6 @@ jobs:
           push: true
           tags: ethereumoptimism/hardhat-node:${{ needs.release.outputs.hardhat-node }},ethereumoptimism/hardhat-node:latest
 
-  ci-builder:
-    name: Publish ci-builder ${{ needs.release.outputs.ci-builder }}
-    needs: release
-    if: needs.release.outputs.ci-builder != ''
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN_SECRET }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Publish ci-builder
-        uses: docker/build-push-action@v2
-        with:
-          context: ./ops/docker/ci-builder
-          file: ./ops/docker/ci-builder/Dockerfile
-          push: true
-          tags: ethereumoptimism/ci-builder:${{ needs.release.outputs.ci-builder }},ethereumoptimism/ci-builder:latest
-
   foundry:
     name: Publish foundry ${{ needs.release.outputs.foundry }}
     needs: release

op-chain-ops/cmd/withdrawals/main.go

@@ -22,11 +22,14 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/eth/tracers"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
 )
 
 // abiTrue represents the storage representation of the boolean
@@ -116,6 +119,10 @@ func main() {
 				Value: "bad-withdrawals.json",
 				Usage: "Path to write JSON file of bad withdrawals to manually inspect",
 			},
+			&cli.StringFlag{
+				Name:  "storage-out",
+				Usage: "Path to write text file of L2ToL1MessagePasser storage",
+			},
 		},
 		Action: func(ctx *cli.Context) error {
 			clients, err := util.NewClients(ctx)
@@ -163,10 +170,11 @@ func main() {
 			}
 
 			outfile := ctx.String("bad-withdrawals-out")
-			f, err := os.OpenFile(outfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o755)
+			f, err := os.OpenFile(outfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
 			if err != nil {
 				return err
 			}
+			defer f.Close()
 
 			// create a transactor
 			opts, err := newTransactor(ctx)
@@ -177,6 +185,28 @@ func main() {
 			// Need this to compare in event parsing
 			l1StandardBridgeAddress := common.HexToAddress(ctx.String("l1-standard-bridge-address"))
 
+			if storageOutfile := ctx.String("storage-out"); storageOutfile != "" {
+				ff, err := os.OpenFile(storageOutfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
+				if err != nil {
+					return err
+				}
+				defer ff.Close()
+
+				log.Info("Fetching storage for L2ToL1MessagePasser")
+				if storageRange, err := callStorageRange(clients, predeploys.L2ToL1MessagePasserAddr); err != nil {
+					log.Info("error getting storage range", "err", err)
+				} else {
+					str := ""
+					for key, value := range storageRange {
+						str += fmt.Sprintf("%s: %s\n", key.Hex(), value.Hex())
+					}
+					_, err = ff.WriteString(str)
+					if err != nil {
+						return err
+					}
+				}
+			}
+
 			// iterate over all of the withdrawals and submit them
 			for i, wd := range wds {
 				log.Info("Processing withdrawal", "index", i)
@@ -234,7 +264,7 @@ func main() {
 				// successful messages can be skipped, received messages failed
 				// their execution and should be replayed
 				if isSuccessNew {
-					log.Info("Message already relayed", "index", i, "hash", hash, "slot", slot)
+					log.Info("Message already relayed", "index", i, "hash", hash.Hex(), "slot", slot.Hex())
 					continue
 				}
 
@@ -248,7 +278,7 @@ func main() {
 
 				// the value should be set to a boolean in storage
 				if !bytes.Equal(storageValue, abiTrue.Bytes()) {
-					return fmt.Errorf("storage slot %x not found in state", slot)
+					return fmt.Errorf("storage slot %x not found in state", slot.Hex())
 				}
 
 				legacySlot, err := wd.StorageSlot()
@@ -443,10 +473,48 @@ func callTrace(c *util.Clients, receipt *types.Receipt) (callFrame, error) {
 		Tracer: &tracer,
 	}
 	err := c.L1RpcClient.Call(&finalizationTrace, "debug_traceTransaction", receipt.TxHash, traceConfig)
-	if err != nil {
 	return finalizationTrace, err
+}
+
+func callStorageRangeAt(
+	client *rpc.Client,
+	blockHash common.Hash,
+	txIndex int,
+	addr common.Address,
+	keyStart hexutil.Bytes,
+	maxResult int,
+) (*eth.StorageRangeResult, error) {
+	var storageRange *eth.StorageRangeResult
+	err := client.Call(&storageRange, "debug_storageRangeAt", blockHash, txIndex, addr, keyStart, maxResult)
+	return storageRange, err
+}
+
+func callStorageRange(c *util.Clients, addr common.Address) (state.Storage, error) {
+	header, err := c.L2Client.HeaderByNumber(context.Background(), nil)
+	if err != nil {
+		return nil, err
 	}
-	return finalizationTrace, err
+	hash := header.Hash()
+	keyStart := hexutil.Bytes(common.Hash{}.Bytes())
+	maxResult := 1000
+
+	ret := make(state.Storage)
+
+	for {
+		result, err := callStorageRangeAt(c.L2RpcClient, hash, 0, addr, keyStart, maxResult)
+		if err != nil {
+			return nil, err
+		}
+		for key, value := range result.Storage {
+			ret[key] = value.Value
+		}
+		if result.NextKey == nil {
+			break
+		} else {
+			keyStart = hexutil.Bytes(result.NextKey.Bytes())
+		}
+	}
+	return ret, nil
 }
 
 // handleFinalizeETHWithdrawal will ensure that the calldata is correct
@@ -709,10 +777,14 @@ func newWithdrawals(ctx *cli.Context, l1ChainID *big.Int) ([]*crossdomain.Legacy
 	witnessFile := ctx.String("witness-file")
 
 	log.Debug("Migration data", "ovm-path", ovmMsgs, "evm-messages", evmMsgs, "witness-file", witnessFile)
-	ovmMessages, err := crossdomain.NewSentMessageFromJSON(ovmMsgs)
+	var ovmMessages []*crossdomain.SentMessage
+	var err error
+	if ovmMsgs != "" {
+		ovmMessages, err = crossdomain.NewSentMessageFromJSON(ovmMsgs)
 		if err != nil {
 			return nil, err
 		}
+	}
 
 	// use empty ovmMessages if its not mainnet. The mainnet messages are
 	// committed to in git.

op-chain-ops/crossdomain/migrate.go

@@ -96,17 +96,12 @@ func MigrateWithdrawal(withdrawal *LegacyWithdrawal, l1CrossDomainMessenger *com
 	return w, nil
 }
 
+// MigrateWithdrawalGasLimit computes the gas limit for the migrated withdrawal.
 func MigrateWithdrawalGasLimit(data []byte) uint64 {
-	// Compute the cost of the calldata
-	dataCost := uint64(0)
-	for _, b := range data {
-		if b == 0 {
-			dataCost += params.TxDataZeroGas
-		} else {
-			dataCost += params.TxDataNonZeroGasEIP2028
-		}
-	}
-
+	// Compute the upper bound on the gas limit. This could be more
+	// accurate if individual 0 bytes and non zero bytes were accounted
+	// for.
+	dataCost := uint64(len(data)) * params.TxDataNonZeroGasEIP2028
 	// Set the outer gas limit. This cannot be zero
 	gasLimit := dataCost + 200_000
 	// Cap the gas limit to be 25 million to prevent creating withdrawals

op-chain-ops/crossdomain/migrate_test.go

@@ -71,15 +71,15 @@ func TestMigrateWithdrawalGasLimit(t *testing.T) {
 		},
 		{
 			input:  []byte{0xff, 0x00},
-			output: 200_000 + 16 + 4,
+			output: 200_000 + 16 + 16,
 		},
 		{
 			input:  []byte{0x00},
-			output: 200_000 + 4,
+			output: 200_000 + 16,
 		},
 		{
 			input:  []byte{0x00, 0x00, 0x00},
-			output: 200_000 + 4 + 4 + 4,
+			output: 200_000 + 16 + 16 + 16,
 		},
 	}
 

op-e2e/system_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/slices"
 
 	"github.com/ethereum-optimism/optimism/op-bindings/bindings"
 	"github.com/ethereum-optimism/optimism/op-bindings/predeploys"
@@ -36,6 +37,7 @@ import (
 	"github.com/ethereum-optimism/optimism/op-node/sources"
 	"github.com/ethereum-optimism/optimism/op-node/testlog"
 	"github.com/ethereum-optimism/optimism/op-node/withdrawals"
+	"github.com/ethereum-optimism/optimism/op-service/backoff"
 	oppprof "github.com/ethereum-optimism/optimism/op-service/pprof"
 )
 
@@ -619,6 +621,24 @@ func TestSystemMockP2P(t *testing.T) {
 	// Enable the sequencer now that everyone is ready to receive payloads.
 	rollupRPCClient, err := rpc.DialContext(context.Background(), sys.RollupNodes["sequencer"].HTTPEndpoint())
 	require.Nil(t, err)
+
+	verifierPeerID := sys.RollupNodes["verifier"].P2P().Host().ID()
+	check := func() bool {
+		sequencerBlocksTopicPeers := sys.RollupNodes["sequencer"].P2P().GossipOut().BlocksTopicPeers()
+		return slices.Contains[peer.ID](sequencerBlocksTopicPeers, verifierPeerID)
+	}
+
+	// poll to see if the verifier node is connected & meshed on gossip.
+	// Without this verifier, we shouldn't start sending blocks around, or we'll miss them and fail the test.
+	backOffStrategy := backoff.Exponential()
+	for i := 0; i < 10; i++ {
+		if check() {
+			break
+		}
+		time.Sleep(backOffStrategy.Duration(i))
+	}
+	require.True(t, check(), "verifier must be meshed with sequencer for gossip test to proceed")
+
 	require.NoError(t, rollupRPCClient.Call(nil, "admin_startSequencer", sys.L2GenesisCfg.ToBlock().Hash()))
 
 	l2Seq := sys.Clients["sequencer"]

op-node/rollup/sync/start.go

@@ -110,7 +110,7 @@ func FindL2Heads(ctx context.Context, cfg *rollup.Config, l1 L1Chain, l2 L2Chain
 	}
 
 	lgr.Info("Loaded current L2 heads", "unsafe", result.Unsafe, "safe", result.Safe, "finalized", result.Finalized,
-		"unsafe_origin", result.Unsafe.L1Origin, "unsafe_origin", result.Safe.L1Origin)
+		"unsafe_origin", result.Unsafe.L1Origin, "safe_origin", result.Safe.L1Origin)
 
 	// Remember original unsafe block to determine reorg depth
 	prevUnsafe := result.Unsafe
@@ -207,7 +207,7 @@ func FindL2Heads(ctx context.Context, cfg *rollup.Config, l1 L1Chain, l2 L2Chain
 		// Don't traverse further than the finalized head to find a safe head
 		if n.Number == result.Finalized.Number {
 			lgr.Info("Hit finalized L2 head, returning immediately", "unsafe", result.Unsafe, "safe", result.Safe,
-				"finalized", result.Finalized, "unsafe_origin", result.Unsafe.L1Origin, "unsafe_origin", result.Safe.L1Origin)
+				"finalized", result.Finalized, "unsafe_origin", result.Unsafe.L1Origin, "safe_origin", result.Safe.L1Origin)
 			result.Safe = n
 			return result, nil
 		}

op-node/sources/debug_client.go

+package sources
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+)
+
+type DebugClient struct {
+	callContext CallContextFn
+}
+
+func NewDebugClient(callContext CallContextFn) *DebugClient {
+	return &DebugClient{callContext}
+}
+
+func (o *DebugClient) NodeByHash(ctx context.Context, hash common.Hash) ([]byte, error) {
+	// MPT nodes are stored as the hash of the node (with no prefix)
+	node, err := o.dbGet(ctx, hash[:])
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve state MPT node: %w", err)
+	}
+	return node, nil
+}
+
+func (o *DebugClient) CodeByHash(ctx context.Context, hash common.Hash) ([]byte, error) {
+	// First try retrieving with the new code prefix
+	code, err := o.dbGet(ctx, append(append(make([]byte, 0), rawdb.CodePrefix...), hash[:]...))
+	if err != nil {
+		// Fallback to the legacy un-prefixed version
+		code, err = o.dbGet(ctx, hash[:])
+		if err != nil {
+			return nil, fmt.Errorf("failed to retrieve contract code, using new and legacy keys, with codehash %s: %w", hash, err)
+		}
+	}
+	return code, nil
+}
+
+func (o *DebugClient) dbGet(ctx context.Context, key []byte) ([]byte, error) {
+	var node hexutil.Bytes
+	err := o.callContext(ctx, &node, "debug_dbGet", hexutil.Encode(key))
+	if err != nil {
+		return nil, fmt.Errorf("fetch error %x: %w", key, err)
+	}
+	return node, nil
+}

op-node/testutils/mock_debug_client.go

+package testutils
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/stretchr/testify/mock"
+)
+
+type MockDebugClient struct {
+	mock.Mock
+}
+
+func (m *MockDebugClient) ExpectNodeByHash(hash common.Hash, res []byte, err error) {
+	m.Mock.On("NodeByHash", hash).Once().Return(res, &err)
+}
+
+func (m *MockDebugClient) NodeByHash(ctx context.Context, hash common.Hash) ([]byte, error) {
+	out := m.Mock.MethodCalled("NodeByHash", hash)
+	return out[0].([]byte), *out[1].(*error)
+}
+
+func (m *MockDebugClient) ExpectCodeByHash(hash common.Hash, res []byte, err error) {
+	m.Mock.On("CodeByHash", hash).Once().Return(res, &err)
+}
+
+func (m *MockDebugClient) CodeByHash(ctx context.Context, hash common.Hash) ([]byte, error) {
+	out := m.Mock.MethodCalled("CodeByHash", hash)
+	return out[0].([]byte), *out[1].(*error)
+}

op-program/Makefile

@@ -9,7 +9,7 @@ LDFLAGSSTRING +=-X github.com/ethereum-optimism/optimism/op-program/version.Meta
 LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
 
 op-program:
-	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/op-program ./cmd/main.go
+	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/op-program ./host/cmd/main.go
 
 clean:
 	rm -rf bin

op-program/client/l1/hints.go

@@ -6,12 +6,18 @@ import (
 	"github.com/ethereum-optimism/optimism/op-program/preimage"
 )
 
+const (
+	HintL1BlockHeader  = "l1-block-header"
+	HintL1Transactions = "l1-transactions"
+	HintL1Receipts     = "l1-receipts"
+)
+
 type BlockHeaderHint common.Hash
 
 var _ preimage.Hint = BlockHeaderHint{}
 
 func (l BlockHeaderHint) Hint() string {
-	return "l1-block-header " + (common.Hash)(l).String()
+	return HintL1BlockHeader + " " + (common.Hash)(l).String()
 }
 
 type TransactionsHint common.Hash
@@ -19,7 +25,7 @@ type TransactionsHint common.Hash
 var _ preimage.Hint = TransactionsHint{}
 
 func (l TransactionsHint) Hint() string {
-	return "l1-transactions " + (common.Hash)(l).String()
+	return HintL1Transactions + " " + (common.Hash)(l).String()
 }
 
 type ReceiptsHint common.Hash
@@ -27,5 +33,5 @@ type ReceiptsHint common.Hash
 var _ preimage.Hint = ReceiptsHint{}
 
 func (l ReceiptsHint) Hint() string {
-	return "l1-receipts " + (common.Hash)(l).String()
+	return HintL1Receipts + " " + (common.Hash)(l).String()
 }

op-program/client/l2/hints.go

@@ -6,12 +6,19 @@ import (
 	"github.com/ethereum-optimism/optimism/op-program/preimage"
 )
 
+const (
+	HintL2BlockHeader  = "l2-block-header"
+	HintL2Transactions = "l2-transactions"
+	HintL2Code         = "l2-code"
+	HintL2StateNode    = "l2-state-node"
+)
+
 type BlockHeaderHint common.Hash
 
 var _ preimage.Hint = BlockHeaderHint{}
 
 func (l BlockHeaderHint) Hint() string {
-	return "l2-block-header " + (common.Hash)(l).String()
+	return HintL2BlockHeader + " " + (common.Hash)(l).String()
 }
 
 type TransactionsHint common.Hash
@@ -19,7 +26,7 @@ type TransactionsHint common.Hash
 var _ preimage.Hint = TransactionsHint{}
 
 func (l TransactionsHint) Hint() string {
-	return "l2-transactions " + (common.Hash)(l).String()
+	return HintL2Transactions + " " + (common.Hash)(l).String()
 }
 
 type CodeHint common.Hash
@@ -27,7 +34,7 @@ type CodeHint common.Hash
 var _ preimage.Hint = CodeHint{}
 
 func (l CodeHint) Hint() string {
-	return "l2-code " + (common.Hash)(l).String()
+	return HintL2Code + " " + (common.Hash)(l).String()
 }
 
 type StateNodeHint common.Hash
@@ -35,5 +42,5 @@ type StateNodeHint common.Hash
 var _ preimage.Hint = StateNodeHint{}
 
 func (l StateNodeHint) Hint() string {
-	return "l2-state-node " + (common.Hash)(l).String()
+	return HintL2StateNode + " " + (common.Hash)(l).String()
 }

op-program/host/cmd/main.go

@@ -6,17 +6,20 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"time"
 
 	"github.com/ethereum-optimism/optimism/op-node/chaincfg"
+	"github.com/ethereum-optimism/optimism/op-node/client"
 	"github.com/ethereum-optimism/optimism/op-node/eth"
-	"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
+	"github.com/ethereum-optimism/optimism/op-node/sources"
 	cldr "github.com/ethereum-optimism/optimism/op-program/client/driver"
 	"github.com/ethereum-optimism/optimism/op-program/host/config"
 	"github.com/ethereum-optimism/optimism/op-program/host/flags"
+	"github.com/ethereum-optimism/optimism/op-program/host/kvstore"
 	"github.com/ethereum-optimism/optimism/op-program/host/l1"
 	"github.com/ethereum-optimism/optimism/op-program/host/l2"
+	"github.com/ethereum-optimism/optimism/op-program/host/prefetcher"
 	"github.com/ethereum-optimism/optimism/op-program/host/version"
+	"github.com/ethereum-optimism/optimism/op-program/preimage"
 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/urfave/cli"
@@ -96,6 +99,11 @@ func setupLogging(ctx *cli.Context) (log.Logger, error) {
 	return logger, nil
 }
 
+type L2Source struct {
+	*sources.L2Client
+	*sources.DebugClient
+}
+
 // FaultProofProgram is the programmatic entry-point for the fault proof program
 func FaultProofProgram(logger log.Logger, cfg *config.Config) error {
 	cfg.Rollup.LogDescription(logger, chaincfg.L2ChainIDToNetworkName)
@@ -104,27 +112,49 @@ func FaultProofProgram(logger log.Logger, cfg *config.Config) error {
 	}
 
 	ctx := context.Background()
+	kv := kvstore.NewMemKV()
+
 	logger.Info("Connecting to L1 node", "l1", cfg.L1URL)
-	l1Source, err := l1.NewFetchingL1(ctx, logger, cfg)
+	l1RPC, err := client.NewRPC(ctx, logger, cfg.L1URL)
 	if err != nil {
-		return fmt.Errorf("connect l1 oracle: %w", err)
+		return fmt.Errorf("failed to setup L1 RPC: %w", err)
 	}
 
 	logger.Info("Connecting to L2 node", "l2", cfg.L2URL)
-	l2Source, err := l2.NewFetchingEngine(ctx, logger, cfg)
+	l2RPC, err := client.NewRPC(ctx, logger, cfg.L2URL)
+	if err != nil {
+		return fmt.Errorf("failed to setup L2 RPC: %w", err)
+	}
+
+	l1ClCfg := sources.L1ClientDefaultConfig(cfg.Rollup, cfg.L1TrustRPC, cfg.L1RPCKind)
+	l2ClCfg := sources.L2ClientDefaultConfig(cfg.Rollup, true)
+	l1Cl, err := sources.NewL1Client(l1RPC, logger, nil, l1ClCfg)
+	if err != nil {
+		return fmt.Errorf("failed to create L1 client: %w", err)
+	}
+	l2Cl, err := sources.NewL2Client(l2RPC, logger, nil, l2ClCfg)
+	if err != nil {
+		return fmt.Errorf("failed to create L2 client: %w", err)
+	}
+	l2DebugCl := &L2Source{L2Client: l2Cl, DebugClient: sources.NewDebugClient(l2RPC.CallContext)}
+
+	logger.Info("Setting up pre-fetcher")
+	prefetch := prefetcher.NewPrefetcher(l1Cl, l2DebugCl, kv)
+	preimageOracle := asOracleFn(ctx, prefetch)
+	hinter := asHinter(prefetch)
+	l1Source := l1.NewSource(logger, preimageOracle, hinter, cfg.L1Head)
+
+	logger.Info("Connecting to L2 node", "l2", cfg.L2URL)
+	l2Source, err := l2.NewEngine(logger, preimageOracle, hinter, cfg)
 	if err != nil {
 		return fmt.Errorf("connect l2 oracle: %w", err)
 	}
 
+	logger.Info("Starting derivation")
 	d := cldr.NewDriver(logger, cfg.Rollup, l1Source, l2Source)
 	for {
 		if err = d.Step(ctx); errors.Is(err, io.EOF) {
 			break
-		} else if cfg.FetchingEnabled() && errors.Is(err, derive.ErrTemporary) {
-			// When in fetching mode, recover from temporary errors to allow us to keep fetching data
-			// TODO(CLI-3780) Ideally the retry would happen in the fetcher so this is not needed
-			logger.Warn("Temporary error in pipeline", "err", err)
-			time.Sleep(5 * time.Second)
 		} else if err != nil {
 			return err
 		}
@@ -135,3 +165,22 @@ func FaultProofProgram(logger log.Logger, cfg *config.Config) error {
 	}
 	return nil
 }
+
+func asOracleFn(ctx context.Context, prefetcher *prefetcher.Prefetcher) preimage.OracleFn {
+	return func(key preimage.Key) []byte {
+		pre, err := prefetcher.GetPreimage(ctx, key.PreimageKey())
+		if err != nil {
+			panic(fmt.Errorf("preimage unavailable for key %v: %w", key, err))
+		}
+		return pre
+	}
+}
+
+func asHinter(prefetcher *prefetcher.Prefetcher) preimage.HinterFn {
+	return func(v preimage.Hint) {
+		err := prefetcher.Hint(v.Hint())
+		if err != nil {
+			panic(fmt.Errorf("hint rejected %v: %w", v, err))
+		}
+	}
+}

op-program/host/l1/l1.go

@@ -8,10 +8,12 @@ import (
 	"github.com/ethereum-optimism/optimism/op-node/sources"
 	cll1 "github.com/ethereum-optimism/optimism/op-program/client/l1"
 	"github.com/ethereum-optimism/optimism/op-program/host/config"
+	"github.com/ethereum-optimism/optimism/op-program/preimage"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 )
 
-func NewFetchingL1(ctx context.Context, logger log.Logger, cfg *config.Config) (derive.L1Fetcher, error) {
+func NewFetchingOracle(ctx context.Context, logger log.Logger, cfg *config.Config) (cll1.Oracle, error) {
 	rpc, err := client.NewRPC(ctx, logger, cfg.L1URL)
 	if err != nil {
 		return nil, err
@@ -21,6 +23,10 @@ func NewFetchingL1(ctx context.Context, logger log.Logger, cfg *config.Config) (
 	if err != nil {
 		return nil, err
 	}
-	oracle := cll1.NewCachingOracle(NewFetchingL1Oracle(ctx, logger, source))
-	return cll1.NewOracleL1Client(logger, oracle, cfg.L1Head), err
+	return NewFetchingL1Oracle(ctx, logger, source), nil
+}
+
+func NewSource(logger log.Logger, oracle preimage.Oracle, hint preimage.Hinter, l1Head common.Hash) derive.L1Fetcher {
+	l1Oracle := cll1.NewCachingOracle(cll1.NewPreimageOracle(oracle, hint))
+	return cll1.NewOracleL1Client(logger, l1Oracle, l1Head)
 }

op-program/host/l2/l2.go

@@ -8,22 +8,18 @@ import (
 
 	cll2 "github.com/ethereum-optimism/optimism/op-program/client/l2"
 	"github.com/ethereum-optimism/optimism/op-program/host/config"
+	"github.com/ethereum-optimism/optimism/op-program/preimage"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 )
 
-func NewFetchingEngine(ctx context.Context, logger log.Logger, cfg *config.Config) (*cll2.OracleEngine, error) {
+func NewEngine(logger log.Logger, pre preimage.Oracle, hint preimage.Hinter, cfg *config.Config) (*cll2.OracleEngine, error) {
+	oracle := cll2.NewCachingOracle(cll2.NewPreimageOracle(pre, hint))
 	genesis, err := loadL2Genesis(cfg)
 	if err != nil {
 		return nil, err
 	}
-	fetcher, err := NewFetchingL2Oracle(ctx, logger, cfg.L2URL, cfg.L2Head)
-	if err != nil {
-		return nil, fmt.Errorf("connect l2 oracle: %w", err)
-	}
-	oracle := cll2.NewCachingOracle(fetcher)
-
 	engineBackend, err := cll2.NewOracleBackedL2Chain(logger, oracle, genesis, cfg.L2Head)
 	if err != nil {
 		return nil, fmt.Errorf("create l2 chain: %w", err)
@@ -31,6 +27,14 @@ func NewFetchingEngine(ctx context.Context, logger log.Logger, cfg *config.Confi
 	return cll2.NewOracleEngine(cfg.Rollup, logger, engineBackend), nil
 }
 
+func NewFetchingOracle(ctx context.Context, logger log.Logger, cfg *config.Config) (cll2.Oracle, error) {
+	oracle, err := NewFetchingL2Oracle(ctx, logger, cfg.L2URL, cfg.L2Head)
+	if err != nil {
+		return nil, fmt.Errorf("connect l2 oracle: %w", err)
+	}
+	return oracle, nil
+}
+
 func loadL2Genesis(cfg *config.Config) (*params.ChainConfig, error) {
 	data, err := os.ReadFile(cfg.L2GenesisPath)
 	if err != nil {

op-program/host/prefetcher/prefetcher.go

+package prefetcher
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+
+	"github.com/ethereum-optimism/optimism/op-node/eth"
+	"github.com/ethereum-optimism/optimism/op-program/client/l1"
+	"github.com/ethereum-optimism/optimism/op-program/client/l2"
+	"github.com/ethereum-optimism/optimism/op-program/client/mpt"
+	"github.com/ethereum-optimism/optimism/op-program/host/kvstore"
+	"github.com/ethereum-optimism/optimism/op-program/preimage"
+)
+
+type L1Source interface {
+	InfoByHash(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, error)
+	InfoAndTxsByHash(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, types.Transactions, error)
+	FetchReceipts(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, types.Receipts, error)
+}
+
+type L2Source interface {
+	InfoAndTxsByHash(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, types.Transactions, error)
+	NodeByHash(ctx context.Context, hash common.Hash) ([]byte, error)
+	CodeByHash(ctx context.Context, hash common.Hash) ([]byte, error)
+}
+
+type Prefetcher struct {
+	l1Fetcher L1Source
+	l2Fetcher L2Source
+	lastHint  string
+	kvStore   kvstore.KV
+}
+
+func NewPrefetcher(l1Fetcher L1Source, l2Fetcher L2Source, kvStore kvstore.KV) *Prefetcher {
+	return &Prefetcher{
+		l1Fetcher: l1Fetcher,
+		l2Fetcher: l2Fetcher,
+		kvStore:   kvStore,
+	}
+}
+
+func (p *Prefetcher) Hint(hint string) error {
+	p.lastHint = hint
+	return nil
+}
+
+func (p *Prefetcher) GetPreimage(ctx context.Context, key common.Hash) ([]byte, error) {
+	pre, err := p.kvStore.Get(key)
+	if errors.Is(err, kvstore.ErrNotFound) && p.lastHint != "" {
+		hint := p.lastHint
+		p.lastHint = ""
+		if err := p.prefetch(ctx, hint); err != nil {
+			return nil, fmt.Errorf("prefetch failed: %w", err)
+		}
+		// Should now be available
+		return p.kvStore.Get(key)
+	}
+	return pre, err
+}
+
+func (p *Prefetcher) prefetch(ctx context.Context, hint string) error {
+	hintType, hash, err := parseHint(hint)
+	if err != nil {
+		return err
+	}
+	switch hintType {
+	case l1.HintL1BlockHeader:
+		header, err := p.l1Fetcher.InfoByHash(ctx, hash)
+		if err != nil {
+			return fmt.Errorf("failed to fetch L1 block %s header: %w", hash, err)
+		}
+		data, err := header.HeaderRLP()
+		if err != nil {
+			return fmt.Errorf("marshall header: %w", err)
+		}
+		return p.kvStore.Put(preimage.Keccak256Key(hash).PreimageKey(), data)
+	case l1.HintL1Transactions:
+		_, txs, err := p.l1Fetcher.InfoAndTxsByHash(ctx, hash)
+		if err != nil {
+			return fmt.Errorf("failed to fetch L1 block %s txs: %w", hash, err)
+		}
+		return p.storeTransactions(txs)
+	case l1.HintL1Receipts:
+		_, receipts, err := p.l1Fetcher.FetchReceipts(ctx, hash)
+		if err != nil {
+			return fmt.Errorf("failed to fetch L1 block %s receipts: %w", hash, err)
+		}
+		return p.storeReceipts(receipts)
+	case l2.HintL2BlockHeader:
+		header, txs, err := p.l2Fetcher.InfoAndTxsByHash(ctx, hash)
+		if err != nil {
+			return fmt.Errorf("failed to fetch L2 block %s: %w", hash, err)
+		}
+		data, err := header.HeaderRLP()
+		if err != nil {
+			return fmt.Errorf("failed to encode header to RLP: %w", err)
+		}
+		err = p.kvStore.Put(preimage.Keccak256Key(hash).PreimageKey(), data)
+		if err != nil {
+			return err
+		}
+		return p.storeTransactions(txs)
+	case l2.HintL2StateNode:
+		node, err := p.l2Fetcher.NodeByHash(ctx, hash)
+		if err != nil {
+			return fmt.Errorf("failed to fetch L2 state node %s: %w", hash, err)
+		}
+		return p.kvStore.Put(preimage.Keccak256Key(hash).PreimageKey(), node)
+	case l2.HintL2Code:
+		code, err := p.l2Fetcher.CodeByHash(ctx, hash)
+		if err != nil {
+			return fmt.Errorf("failed to fetch L2 contract code %s: %w", hash, err)
+		}
+		return p.kvStore.Put(preimage.Keccak256Key(hash).PreimageKey(), code)
+	}
+	return fmt.Errorf("unknown hint type: %v", hintType)
+}
+
+func (p *Prefetcher) storeReceipts(receipts types.Receipts) error {
+	opaqueReceipts, err := eth.EncodeReceipts(receipts)
+	if err != nil {
+		return err
+	}
+	return p.storeTrieNodes(opaqueReceipts)
+}
+
+func (p *Prefetcher) storeTransactions(txs types.Transactions) error {
+	opaqueTxs, err := eth.EncodeTransactions(txs)
+	if err != nil {
+		return err
+	}
+	return p.storeTrieNodes(opaqueTxs)
+}
+
+func (p *Prefetcher) storeTrieNodes(values []hexutil.Bytes) error {
+	_, nodes := mpt.WriteTrie(values)
+	for _, node := range nodes {
+		err := p.kvStore.Put(preimage.Keccak256Key(crypto.Keccak256Hash(node)).PreimageKey(), node)
+		if err != nil {
+			return fmt.Errorf("failed to store node: %w", err)
+		}
+	}
+	return nil
+}
+
+// parseHint parses a hint string in wire protocol. Returns the hint type, requested hash and error (if any).
+func parseHint(hint string) (string, common.Hash, error) {
+	hintType, hashStr, found := strings.Cut(hint, " ")
+	if !found {
+		return "", common.Hash{}, fmt.Errorf("unsupported hint: %s", hint)
+	}
+	hash := common.HexToHash(hashStr)
+	if hash == (common.Hash{}) {
+		return "", common.Hash{}, fmt.Errorf("invalid hash: %s", hashStr)
+	}
+	return hintType, hash, nil
+}

op-program/host/prefetcher/prefetcher_test.go

+package prefetcher
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum-optimism/optimism/op-node/eth"
+	"github.com/ethereum-optimism/optimism/op-node/testutils"
+	"github.com/ethereum-optimism/optimism/op-program/client/l1"
+	"github.com/ethereum-optimism/optimism/op-program/client/l2"
+	"github.com/ethereum-optimism/optimism/op-program/client/mpt"
+	"github.com/ethereum-optimism/optimism/op-program/host/kvstore"
+	"github.com/ethereum-optimism/optimism/op-program/preimage"
+)
+
+func TestNoHint(t *testing.T) {
+	t.Run("NotFound", func(t *testing.T) {
+		prefetcher, _, _, _ := createPrefetcher(t)
+		res, err := prefetcher.GetPreimage(context.Background(), common.Hash{0xab})
+		require.ErrorIs(t, err, kvstore.ErrNotFound)
+		require.Nil(t, res)
+	})
+
+	t.Run("Exists", func(t *testing.T) {
+		prefetcher, _, _, kv := createPrefetcher(t)
+		data := []byte{1, 2, 3}
+		hash := crypto.Keccak256Hash(data)
+		require.NoError(t, kv.Put(hash, data))
+
+		res, err := prefetcher.GetPreimage(context.Background(), hash)
+		require.NoError(t, err)
+		require.Equal(t, res, data)
+	})
+}
+
+func TestFetchL1BlockHeader(t *testing.T) {
+	rng := rand.New(rand.NewSource(123))
+	block, rcpts := testutils.RandomBlock(rng, 2)
+	hash := block.Hash()
+	key := preimage.Keccak256Key(hash).PreimageKey()
+	pre, err := rlp.EncodeToBytes(block.Header())
+	require.NoError(t, err)
+
+	t.Run("AlreadyKnown", func(t *testing.T) {
+		prefetcher, _, _, kv := createPrefetcher(t)
+		storeBlock(t, kv, block, rcpts)
+
+		oracle := l1.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+		result := oracle.HeaderByBlockHash(hash)
+		require.Equal(t, eth.HeaderBlockInfo(block.Header()), result)
+	})
+
+	t.Run("Unknown", func(t *testing.T) {
+		prefetcher, l1Cl, _, _ := createPrefetcher(t)
+		l1Cl.ExpectInfoByHash(hash, eth.HeaderBlockInfo(block.Header()), nil)
+		defer l1Cl.AssertExpectations(t)
+
+		require.NoError(t, prefetcher.Hint(l1.BlockHeaderHint(hash).Hint()))
+		result, err := prefetcher.GetPreimage(context.Background(), key)
+		require.NoError(t, err)
+		require.Equal(t, pre, result)
+	})
+}
+
+func TestFetchL1Transactions(t *testing.T) {
+	rng := rand.New(rand.NewSource(123))
+	block, rcpts := testutils.RandomBlock(rng, 10)
+	hash := block.Hash()
+
+	t.Run("AlreadyKnown", func(t *testing.T) {
+		prefetcher, _, _, kv := createPrefetcher(t)
+
+		storeBlock(t, kv, block, rcpts)
+
+		// Check the data is available (note the oracle does not know about the block, only the kvstore does)
+		oracle := l1.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+		header, txs := oracle.TransactionsByBlockHash(hash)
+		require.EqualValues(t, hash, header.Hash())
+		assertTransactionsEqual(t, block.Transactions(), txs)
+	})
+
+	t.Run("Unknown", func(t *testing.T) {
+		prefetcher, l1Cl, _, _ := createPrefetcher(t)
+		l1Cl.ExpectInfoByHash(hash, eth.BlockToInfo(block), nil)
+		l1Cl.ExpectInfoAndTxsByHash(hash, eth.BlockToInfo(block), block.Transactions(), nil)
+		defer l1Cl.AssertExpectations(t)
+
+		oracle := l1.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+		header, txs := oracle.TransactionsByBlockHash(hash)
+		require.EqualValues(t, hash, header.Hash())
+		assertTransactionsEqual(t, block.Transactions(), txs)
+	})
+}
+
+func TestFetchL1Receipts(t *testing.T) {
+	rng := rand.New(rand.NewSource(123))
+	block, receipts := testutils.RandomBlock(rng, 10)
+	hash := block.Hash()
+
+	t.Run("AlreadyKnown", func(t *testing.T) {
+		prefetcher, _, _, kv := createPrefetcher(t)
+		storeBlock(t, kv, block, receipts)
+
+		// Check the data is available (note the oracle does not know about the block, only the kvstore does)
+		oracle := l1.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+		header, actualReceipts := oracle.ReceiptsByBlockHash(hash)
+		require.EqualValues(t, hash, header.Hash())
+		assertReceiptsEqual(t, receipts, actualReceipts)
+	})
+
+	t.Run("Unknown", func(t *testing.T) {
+		prefetcher, l1Cl, _, _ := createPrefetcher(t)
+		l1Cl.ExpectInfoByHash(hash, eth.BlockToInfo(block), nil)
+		l1Cl.ExpectInfoAndTxsByHash(hash, eth.BlockToInfo(block), block.Transactions(), nil)
+		l1Cl.ExpectFetchReceipts(hash, eth.BlockToInfo(block), receipts, nil)
+		defer l1Cl.AssertExpectations(t)
+
+		oracle := l1.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+		header, actualReceipts := oracle.ReceiptsByBlockHash(hash)
+		require.EqualValues(t, hash, header.Hash())
+		assertReceiptsEqual(t, receipts, actualReceipts)
+	})
+}
+
+func TestFetchL2Block(t *testing.T) {
+	rng := rand.New(rand.NewSource(123))
+	block, rcpts := testutils.RandomBlock(rng, 10)
+	hash := block.Hash()
+
+	t.Run("AlreadyKnown", func(t *testing.T) {
+		prefetcher, _, _, kv := createPrefetcher(t)
+		storeBlock(t, kv, block, rcpts)
+
+		oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+		result := oracle.BlockByHash(hash)
+		require.EqualValues(t, block.Header(), result.Header())
+		assertTransactionsEqual(t, block.Transactions(), result.Transactions())
+	})
+
+	t.Run("Unknown", func(t *testing.T) {
+		prefetcher, _, l2Cl, _ := createPrefetcher(t)
+		l2Cl.ExpectInfoAndTxsByHash(hash, eth.BlockToInfo(block), block.Transactions(), nil)
+		defer l2Cl.MockL2Client.AssertExpectations(t)
+
+		oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+		result := oracle.BlockByHash(hash)
+		require.EqualValues(t, block.Header(), result.Header())
+		assertTransactionsEqual(t, block.Transactions(), result.Transactions())
+	})
+}
+
+func TestFetchL2Node(t *testing.T) {
+	rng := rand.New(rand.NewSource(123))
+	node := testutils.RandomData(rng, 30)
+	hash := crypto.Keccak256Hash(node)
+	key := preimage.Keccak256Key(hash).PreimageKey()
+
+	t.Run("AlreadyKnown", func(t *testing.T) {
+		prefetcher, _, _, kv := createPrefetcher(t)
+		require.NoError(t, kv.Put(key, node))
+
+		oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+		result := oracle.NodeByHash(hash)
+		require.EqualValues(t, node, result)
+	})
+
+	t.Run("Unknown", func(t *testing.T) {
+		prefetcher, _, l2Cl, _ := createPrefetcher(t)
+		l2Cl.ExpectNodeByHash(hash, node, nil)
+		defer l2Cl.MockDebugClient.AssertExpectations(t)
+
+		oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+		result := oracle.NodeByHash(hash)
+		require.EqualValues(t, node, result)
+	})
+}
+
+func TestFetchL2Code(t *testing.T) {
+	rng := rand.New(rand.NewSource(123))
+	code := testutils.RandomData(rng, 30)
+	hash := crypto.Keccak256Hash(code)
+	key := preimage.Keccak256Key(hash).PreimageKey()
+
+	t.Run("AlreadyKnown", func(t *testing.T) {
+		prefetcher, _, _, kv := createPrefetcher(t)
+		require.NoError(t, kv.Put(key, code))
+
+		oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+		result := oracle.CodeByHash(hash)
+		require.EqualValues(t, code, result)
+	})
+
+	t.Run("Unknown", func(t *testing.T) {
+		prefetcher, _, l2Cl, _ := createPrefetcher(t)
+		l2Cl.ExpectCodeByHash(hash, code, nil)
+		defer l2Cl.MockDebugClient.AssertExpectations(t)
+
+		oracle := l2.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+		result := oracle.CodeByHash(hash)
+		require.EqualValues(t, code, result)
+	})
+}
+
+func TestBadHints(t *testing.T) {
+	prefetcher, _, _, kv := createPrefetcher(t)
+	hash := common.Hash{0xad}
+
+	t.Run("NoSpace", func(t *testing.T) {
+		// Accept the hint
+		require.NoError(t, prefetcher.Hint(l1.HintL1BlockHeader))
+
+		// But it will fail to prefetch when the pre-image isn't available
+		pre, err := prefetcher.GetPreimage(context.Background(), hash)
+		require.ErrorContains(t, err, "unsupported hint")
+		require.Nil(t, pre)
+	})
+
+	t.Run("InvalidHash", func(t *testing.T) {
+		// Accept the hint
+		require.NoError(t, prefetcher.Hint(l1.HintL1BlockHeader+" asdfsadf"))
+
+		// But it will fail to prefetch when the pre-image isn't available
+		pre, err := prefetcher.GetPreimage(context.Background(), hash)
+		require.ErrorContains(t, err, "invalid hash")
+		require.Nil(t, pre)
+	})
+
+	t.Run("UnknownType", func(t *testing.T) {
+		// Accept the hint
+		require.NoError(t, prefetcher.Hint("unknown "+hash.Hex()))
+
+		// But it will fail to prefetch when the pre-image isn't available
+		pre, err := prefetcher.GetPreimage(context.Background(), hash)
+		require.ErrorContains(t, err, "unknown hint type")
+		require.Nil(t, pre)
+	})
+
+	// Should not return hint errors if the preimage is already available
+	t.Run("KeyExists", func(t *testing.T) {
+		// Prepopulate the requested preimage
+		value := []byte{1, 2, 3, 4}
+		require.NoError(t, kv.Put(hash, value))
+
+		// Hint is invalid
+		require.NoError(t, prefetcher.Hint("asdfsadf"))
+		// But fetching the key fails because prefetching isn't required
+		pre, err := prefetcher.GetPreimage(context.Background(), hash)
+		require.NoError(t, err)
+		require.Equal(t, value, pre)
+	})
+}
+
+type l2Client struct {
+	*testutils.MockL2Client
+	*testutils.MockDebugClient
+}
+
+func createPrefetcher(t *testing.T) (*Prefetcher, *testutils.MockL1Source, *l2Client, kvstore.KV) {
+	kv := kvstore.NewMemKV()
+
+	l1Source := new(testutils.MockL1Source)
+	l2Source := &l2Client{
+		MockL2Client:    new(testutils.MockL2Client),
+		MockDebugClient: new(testutils.MockDebugClient),
+	}
+
+	prefetcher := NewPrefetcher(l1Source, l2Source, kv)
+	return prefetcher, l1Source, l2Source, kv
+}
+
+func storeBlock(t *testing.T, kv kvstore.KV, block *types.Block, receipts types.Receipts) {
+	// Pre-store receipts
+	opaqueRcpts, err := eth.EncodeReceipts(receipts)
+	require.NoError(t, err)
+	_, nodes := mpt.WriteTrie(opaqueRcpts)
+	for _, p := range nodes {
+		require.NoError(t, kv.Put(preimage.Keccak256Key(crypto.Keccak256Hash(p)).PreimageKey(), p))
+	}
+
+	// Pre-store transactions
+	opaqueTxs, err := eth.EncodeTransactions(block.Transactions())
+	require.NoError(t, err)
+	_, txsNodes := mpt.WriteTrie(opaqueTxs)
+	for _, p := range txsNodes {
+		require.NoError(t, kv.Put(preimage.Keccak256Key(crypto.Keccak256Hash(p)).PreimageKey(), p))
+	}
+
+	// Pre-store block
+	headerRlp, err := rlp.EncodeToBytes(block.Header())
+	require.NoError(t, err)
+	require.NoError(t, kv.Put(preimage.Keccak256Key(block.Hash()).PreimageKey(), headerRlp))
+}
+
+func asOracleFn(t *testing.T, prefetcher *Prefetcher) preimage.OracleFn {
+	return func(key preimage.Key) []byte {
+		pre, err := prefetcher.GetPreimage(context.Background(), key.PreimageKey())
+		require.NoError(t, err)
+		return pre
+	}
+}
+
+func asHinter(t *testing.T, prefetcher *Prefetcher) preimage.HinterFn {
+	return func(v preimage.Hint) {
+		err := prefetcher.Hint(v.Hint())
+		require.NoError(t, err)
+	}
+}
+
+func assertTransactionsEqual(t *testing.T, blockTx types.Transactions, txs types.Transactions) {
+	require.Equal(t, len(blockTx), len(txs))
+	for i, tx := range txs {
+		require.Equal(t, blockTx[i].Hash(), tx.Hash())
+	}
+}
+
+func assertReceiptsEqual(t *testing.T, expectedRcpt types.Receipts, actualRcpt types.Receipts) {
+	require.Equal(t, len(expectedRcpt), len(actualRcpt))
+	for i, rcpt := range actualRcpt {
+		// Make a copy of each to zero out fields we expect to be different
+		expected := *expectedRcpt[i]
+		actual := *rcpt
+		expected.ContractAddress = common.Address{}
+		actual.ContractAddress = common.Address{}
+		require.Equal(t, expected, actual)
+	}
+}

op-program/preimage/hints.go

@@ -55,6 +55,8 @@ func (hr *HintReader) NextHint(router func(hint string) error) error {
 		}
 	}
 	if err := router(string(payload)); err != nil {
+		// stream recovery
+		_, _ = hr.r.Read([]byte{0})
 		return fmt.Errorf("failed to handle hint: %w", err)
 	}
 	if _, err := hr.r.Read([]byte{0}); err != nil {

op-program/preimage/hints_test.go

@@ -3,6 +3,7 @@ package preimage
 import (
 	"bytes"
 	"crypto/rand"
+	"errors"
 	"io"
 	"testing"
 
@@ -71,4 +72,21 @@ func TestHints(t *testing.T) {
 		err := hr.NextHint(func(hint string) error { return nil })
 		require.ErrorIs(t, err, io.ErrUnexpectedEOF)
 	})
+	t.Run("cb error", func(t *testing.T) {
+		var buf bytes.Buffer
+		hw := NewHintWriter(&buf)
+		hw.Hint(rawHint("one"))
+		hw.Hint(rawHint("two"))
+		hr := NewHintReader(&buf)
+		cbErr := errors.New("fail")
+		err := hr.NextHint(func(hint string) error { return cbErr })
+		require.ErrorIs(t, err, cbErr)
+		var readHint string
+		err = hr.NextHint(func(hint string) error {
+			readHint = hint
+			return nil
+		})
+		require.NoError(t, err)
+		require.Equal(t, readHint, "two")
+	})
 }

ops/docker/ci-builder/package.json

-{
-  "name": "@eth-optimism/ci-builder",
-  "version": "0.5.0",
-  "scripts": {},
-  "license": "MIT",
-  "dependencies": {}
-}

package.json

@@ -16,7 +16,6 @@
       "ops/docker/hardhat",
       "ops/docker/go-builder",
       "ops/docker/js-builder",
-      "ops/docker/ci-builder",
       "ops/docker/foundry",
       "endpoint-monitor"
     ],

packages/sdk/src/utils/message-utils.ts

-import { hashWithdrawal, calldataCost } from '@eth-optimism/core-utils'
-import { BigNumber } from 'ethers'
+import { hashWithdrawal } from '@eth-optimism/core-utils'
+import { BigNumber, utils } from 'ethers'
 
 import { LowLevelMessage } from '../interfaces'
 
+const { hexDataLength } = utils
+
 /**
  * Utility for hashing a LowLevelMessage object.
  *
@@ -25,7 +27,7 @@ export const hashLowLevelMessage = (message: LowLevelMessage): string => {
  */
 export const migratedWithdrawalGasLimit = (data: string): BigNumber => {
   // Compute the gas limit and cap at 25 million
-  const dataCost = calldataCost(data)
+  const dataCost = BigNumber.from(hexDataLength(data)).mul(16)
   let minGasLimit = dataCost.add(200_000)
   if (minGasLimit.gt(25_000_000)) {
     minGasLimit = BigNumber.from(25_000_000)

packages/sdk/test/utils/message-utils.spec.ts

@@ -15,9 +15,9 @@ describe('Message Utils', () => {
       const tests = [
         { input: '0x', result: BigNumber.from(200_000) },
         { input: '0xff', result: BigNumber.from(200_000 + 16) },
-        { input: '0xff00', result: BigNumber.from(200_000 + 16 + 4) },
-        { input: '0x00', result: BigNumber.from(200_000 + 4) },
-        { input: '0x000000', result: BigNumber.from(200_000 + 4 + 4 + 4) },
+        { input: '0xff00', result: BigNumber.from(200_000 + 16 + 16) },
+        { input: '0x00', result: BigNumber.from(200_000 + 16) },
+        { input: '0x000000', result: BigNumber.from(200_000 + 16 + 16 + 16) },
       ]
 
       for (const test of tests) {

specs/withdrawals.md

@@ -18,12 +18,17 @@ an L2 account to an L1 account.
 more specific terms to differentiate:
 
 - A _withdrawal initiating transaction_ refers specifically to a transaction on L2 sent to the Withdrawals predeploy.
+- A _withdrawal proving transaction_ refers specifically to an L1 transaction
+  which proves the withdrawal is correct (that it has been included in a merkle
+  tree whose root is available on L1).
 - A _withdrawal finalizing transaction_ refers specifically to an L1 transaction which finalizes and relays the
   withdrawal.
 
 Withdrawals are initiated on L2 via a call to the Message Passer predeploy contract, which records the important
-properties of the message in its storage. Withdrawals are finalized on L1 via a call to the `OptimismPortal`
-contract, which proves the inclusion of this withdrawal message.
+properties of the message in its storage.
+Withdrawals are proven on L1 via a call to the `OptimismPortal`, which proves the inclusion of this withdrawal message.
+Withdrawals are finalized on L1 via a call to the `OptimismPortal` contract,
+which verifies that the fault challenge period has passed since the withdrawal message has been proved.
 
 In this way, withdrawals are different from [deposits][g-deposits] which make use of a special transaction type in the
 [execution engine][g-execution-engine] client. Rather, withdrawals transaction must use smart contracts on L1 for
@@ -59,18 +64,20 @@ This is a very simple contract that stores the hash of the withdrawal data.
 
 ### On L1
 
-1. A [relayer][g-relayer] submits the required inputs to the `OptimismPortal` contract. The relayer need
-   not be the same entity which initiated the withdrawal on L2.
+1. A [relayer][g-relayer] submits a withdrawal proving transaction with the required inputs
+   to the `OptimismPortal` contract.
+   The relayer is not necessarily the same entity which initiated the withdrawal on L2.
    These inputs include the withdrawal transaction data, inclusion proofs, and a block number. The block number
    must be one for which an L2 output root exists, which commits to the withdrawal as registered on L2.
 1. The `OptimismPortal` contract retrieves the output root for the given block number from the `L2OutputOracle`'s
-   `getL2OutputAfter()` function, and performs the remainder of the verification process internally.
+   `getL2Output()` function, and performs the remainder of the verification process internally.
 1. If proof verification fails, the call reverts. Otherwise the hash is recorded to prevent it from being re-proven.
    Note that the withdrawal can be proven more than once if the corresponding output root changes.
 1. After the withdrawal is proven, it enters a 7 day challenge period, allowing time for other network participants
    to challenge the integrity of the corresponding output root.
-1. Once the challenge period has passed, a relayer submits the withdrawal transaction once again to the
-   `OptimismPortal` contract. Again, the relayer need not be the same entity which initiated the withdrawal on L2.
+1. Once the challenge period has passed, a relayer submits a withdrawal finalizing transaction to the
+   `OptimismPortal` contract.
+   The relayer doesn't need to be the same entity that initiated the withdrawal on L2.
 1. The `OptimismPortal` contract receives the withdrawal transaction data and verifies that the withdrawal has
    both been proven and passed the challenge period.
 1. If the requirements are not met, the call reverts. Otherwise the call is forwarded, and the hash is recorded to
@@ -102,7 +109,7 @@ interface L2ToL1MessagePasser {
 
     function initiateWithdrawal(address _target, uint256 _gasLimit, bytes memory _data) payable external;
 
-    function nonce() view external returns (uint256);
+    function messageNonce() public view returns (uint256);
 
     function sentMessages(bytes32) view external returns (bool);
 }
@@ -139,13 +146,14 @@ withdrawals:
 ```js
 interface OptimismPortal {
 
-    event WithdrawalFinalized(bytes32 indexed);
+    event WithdrawalFinalized(bytes32 indexed withdrawalHash, bool success);
+
 
     function l2Sender() returns(address) external;
 
     function proveWithdrawalTransaction(
         Types.WithdrawalTransaction memory _tx,
-        uint256 _l2BlockNumber,
+        uint256 _l2OutputIndex,
         Types.OutputRootProof calldata _outputRootProof,
         bytes[] calldata _withdrawalProof
     ) external;
@@ -168,14 +176,14 @@ The following inputs are required to prove and finalize a withdrawal:
   - `data`: Data to send to the target.
   - `gasLimit`: Gas to be forwarded to the target.
 - Proof and verification data:
-  - `l2BlockNumber`: The L2 block number that corresponds to the output root.
+  - `l2OutputIndex`: The index in the L2 outputs where the applicable output root may be found.
   - `outputRootProof`: Four `bytes32` values which are used to derive the output root.
   - `withdrawalProof`: An inclusion proof for the given withdrawal in the L2ToL1MessagePasser contract.
 
 These inputs must satisfy the following conditions:
 
-1. The `l2BlockNumber` must be the block number that corresponds to the `OutputProposal` being proven.
-1. `L2OutputOracle.getL2OutputAfter(l2BlockNumber)` returns a non-zero `OutputProposal`.
+1. The `l2OutputIndex` must be the index in the L2 outputs that contains the applicable output root.
+1. `L2OutputOracle.getL2Output(l2OutputIndex)` returns a non-zero `OutputProposal`.
 1. The keccak256 hash of the `outputRootProof` values is equal to the `outputRoot`.
 1. The `withdrawalProof` is a valid inclusion proof demonstrating that a hash of the Withdrawal transaction data
    is contained in the storage of the L2ToL1MessagePasser contract on L2.
@@ -190,13 +198,13 @@ These inputs must satisfy the following conditions:
 
    [polygon-dbl-spend]: https://gerhard-wagner.medium.com/double-spending-bug-in-polygons-plasma-bridge-2e0954ccadf1
 
-1. For each withdrawal initiated on L2 (ie. with a unique `nonce`), the following properties must hold:
+1. For each withdrawal initiated on L2 (i.e. with a unique `messageNonce()`), the following properties must hold:
    1. It should only be possible to prove the withdrawal once, unless the outputRoot for the withdrawal
       has changed.
    1. It should only be possible to finalize the withdrawal once.
    1. It should not be possible to relay the message with any of its fields modified, ie.
       1. Modifying the `sender` field would enable a 'spoofing' attack.
-      1. Modifying the `target`, `message`, or `value` fields would enable an attacker to dangerously change the
+      1. Modifying the `target`, `data`, or `value` fields would enable an attacker to dangerously change the
          intended outcome of the withdrawal.
       1. Modifying the `gasLimit` could make the cost of relaying too high, or allow the relayer to cause execution
          to fail (out of gas) in the `target`.