Merge branch 'develop' into inphi/fpp-comms

.circleci/config.yml

@@ -660,6 +660,45 @@ jobs:
           command: npx depcheck
           working_directory: integration-tests
 
+  atst-tests:
+    docker:
+      - image: ethereumoptimism/ci-builder:latest
+    resource_class: large
+    steps:
+      - checkout
+      - attach_workspace: { at: '.' }
+      - check-changed:
+          patterns: atst,contracts-periphery
+      - restore_cache:
+          name: Restore Yarn Package Cache
+          keys:
+            - yarn-packages-v2-{{ checksum "yarn.lock" }}
+      - run:
+          name: anvil
+          background: true
+          command: anvil --fork-url $ANVIL_L2_FORK_URL_MAINNET --fork-block-number 92093723
+      - run:
+          name: build
+          command: yarn build
+          working_directory: packages/atst
+      - run:
+          name: typecheck
+          command: yarn typecheck
+          working_directory: packages/atst
+      - run:
+          name: lint
+          command: yarn lint:check
+          working_directory: packages/atst
+      - run:
+          name: make sure anvil is up
+          command: npx wait-on tcp:8545 && cast block-number --rpc-url http://localhost:8545
+      - run:
+          name: test
+          command: yarn test
+          no_output_timeout: 5m
+          working_directory: packages/atst
+
+
   go-lint:
     parameters:
       module:
@@ -1094,6 +1133,9 @@ workflows:
       - op-bindings-build:
           requires:
             - yarn-monorepo
+      - atst-tests:
+          requires:
+            - yarn-monorepo
       - js-lint-test:
           name: actor-tests-tests
           coverage_flag: actor-tests-tests
@@ -1464,4 +1506,4 @@ workflows:
           docker_tags: <<pipeline.git.revision>>,latest
           docker_context: ./ops/docker/ci-builder
           context:
-            - oplabs-gcr
\ No newline at end of file
+            - oplabs-gcr

docs/op-stack/src/docs/build/explorer.md

@@ -11,7 +11,8 @@ One easy way to do this is to use [Blockscout](https://www.blockscout.com/).
 ### Archive mode
 
 Blockscout expects to interact with an Ethereum execution client in [archive mode](https://www.alchemy.com/overviews/archive-nodes#archive-nodes).
-To create such a node, follow the [directions to add a node](./getting-started.md#adding-nodes), but in the command you use to start `op-geth` replace:
+If your `op-geth` is running in full mode, you can create a separate archive node.
+To do so, follow the [directions to add a node](./getting-started.md#adding-nodes), but in the command you use to start `op-geth` replace:
 
 ```sh
 	--gcmode=full \

op-e2e/system_fpp_test.go

@@ -12,6 +12,7 @@ import (
 	oppcl "github.com/ethereum-optimism/optimism/op-program/client"
 	opp "github.com/ethereum-optimism/optimism/op-program/host"
 	oppconf "github.com/ethereum-optimism/optimism/op-program/host/config"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -39,26 +40,51 @@ func TestVerifyL2OutputRoot(t *testing.T) {
 	require.Nil(t, err)
 	rollupClient := sources.NewRollupClient(client.NewBaseRPCClient(rollupRPCClient))
 
-	// TODO (CLI-3855): Actually perform some tx to set up a more complex chain.
-
-	// Wait for the safe head to reach block 10
-	require.NoError(t, waitForSafeHead(ctx, 10, rollupClient))
-
-	// Use block 5 as the agreed starting block on L2
-	l2AgreedBlock, err := l2Seq.BlockByNumber(ctx, big.NewInt(5))
-	require.NoError(t, err, "could not retrieve l2 genesis")
-	l2Head := l2AgreedBlock.Hash() // Agreed starting L2 block
-
-	// Get the expected output at block 10
-	l2ClaimBlockNumber := uint64(10)
+	t.Log("Sending transactions to setup existing state, prior to challenged period")
+	aliceKey := cfg.Secrets.Alice
+	opts, err := bind.NewKeyedTransactorWithChainID(aliceKey, cfg.L1ChainIDBig())
+	require.Nil(t, err)
+	SendDepositTx(t, cfg, l1Client, l2Seq, opts, func(l2Opts *DepositTxOpts) {
+		l2Opts.Value = big.NewInt(100_000_000)
+	})
+	SendL2Tx(t, cfg, l2Seq, aliceKey, func(opts *TxOpts) {
+		opts.ToAddr = &cfg.Secrets.Addresses().Bob
+		opts.Value = big.NewInt(1_000)
+		opts.Nonce = 1
+	})
+	SendWithdrawal(t, cfg, l2Seq, aliceKey, func(opts *WithdrawalTxOpts) {
+		opts.Value = big.NewInt(500)
+		opts.Nonce = 2
+	})
+
+	t.Log("Capture current L2 head as agreed starting point")
+	l2AgreedBlock, err := l2Seq.BlockByNumber(ctx, nil)
+	require.NoError(t, err, "could not retrieve l2 agreed block")
+	l2Head := l2AgreedBlock.Hash()
+
+	t.Log("Sending transactions to modify existing state, within challenged period")
+	SendDepositTx(t, cfg, l1Client, l2Seq, opts, func(l2Opts *DepositTxOpts) {
+		l2Opts.Value = big.NewInt(5_000)
+	})
+	SendL2Tx(t, cfg, l2Seq, cfg.Secrets.Bob, func(opts *TxOpts) {
+		opts.ToAddr = &cfg.Secrets.Addresses().Alice
+		opts.Value = big.NewInt(100)
+	})
+	SendWithdrawal(t, cfg, l2Seq, aliceKey, func(opts *WithdrawalTxOpts) {
+		opts.Value = big.NewInt(100)
+		opts.Nonce = 4
+	})
+
+	t.Log("Determine L2 claim")
+	l2ClaimBlockNumber, err := l2Seq.BlockNumber(ctx)
+	require.NoError(t, err, "get L2 claim block number")
 	l2Output, err := rollupClient.OutputAtBlock(ctx, l2ClaimBlockNumber)
 	require.NoError(t, err, "could not get expected output")
 	l2Claim := l2Output.OutputRoot
 
-	// Find the current L1 head
-	l1BlockNumber, err := l1Client.BlockNumber(ctx)
-	require.NoError(t, err, "get l1 head block number")
-	l1HeadBlock, err := l1Client.BlockByNumber(ctx, new(big.Int).SetUint64(l1BlockNumber))
+	t.Log("Determine L1 head that includes all batches required for L2 claim block")
+	require.NoError(t, waitForSafeHead(ctx, l2ClaimBlockNumber, rollupClient))
+	l1HeadBlock, err := l1Client.BlockByNumber(ctx, nil)
 	require.NoError(t, err, "get l1 head block")
 	l1Head := l1HeadBlock.Hash()
 
@@ -73,7 +99,11 @@ func TestVerifyL2OutputRoot(t *testing.T) {
 	err = opp.FaultProofProgram(log, fppConfig)
 	require.NoError(t, err)
 
+	t.Log("Shutting down network")
 	// Shutdown the nodes from the actual chain. Should now be able to run using only the pre-fetched data.
+	sys.BatchSubmitter.StopIfRunning(context.Background())
+	sys.L2OutputSubmitter.Stop()
+	sys.L2OutputSubmitter = nil
 	for _, node := range sys.Nodes {
 		require.NoError(t, node.Close())
 	}

op-program/host/prefetcher/prefetcher.go

@@ -6,18 +6,17 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/log"
-
 	"github.com/ethereum-optimism/optimism/op-node/eth"
 	"github.com/ethereum-optimism/optimism/op-program/client/l1"
 	"github.com/ethereum-optimism/optimism/op-program/client/l2"
 	"github.com/ethereum-optimism/optimism/op-program/client/mpt"
 	"github.com/ethereum-optimism/optimism/op-program/host/kvstore"
 	"github.com/ethereum-optimism/optimism/op-program/preimage"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
 )
 
 type L1Source interface {
@@ -33,6 +32,7 @@ type L2Source interface {
 }
 
 type Prefetcher struct {
+	logger    log.Logger
 	l1Fetcher L1Source
 	l2Fetcher L2Source
 	lastHint  string
@@ -41,6 +41,7 @@ type Prefetcher struct {
 
 func NewPrefetcher(logger log.Logger, l1Fetcher L1Source, l2Fetcher L2Source, kvStore kvstore.KV) *Prefetcher {
 	return &Prefetcher{
+		logger:    logger,
 		l1Fetcher: NewRetryingL1Source(logger, l1Fetcher),
 		l2Fetcher: NewRetryingL2Source(logger, l2Fetcher),
 		kvStore:   kvStore,
@@ -71,6 +72,7 @@ func (p *Prefetcher) prefetch(ctx context.Context, hint string) error {
 	if err != nil {
 		return err
 	}
+	p.logger.Debug("Prefetching", "type", hintType, "hash", hash)
 	switch hintType {
 	case l1.HintL1BlockHeader:
 		header, err := p.l1Fetcher.InfoByHash(ctx, hash)
@@ -143,8 +145,11 @@ func (p *Prefetcher) storeTransactions(txs types.Transactions) error {
 func (p *Prefetcher) storeTrieNodes(values []hexutil.Bytes) error {
 	_, nodes := mpt.WriteTrie(values)
 	for _, node := range nodes {
-		err := p.kvStore.Put(preimage.Keccak256Key(crypto.Keccak256Hash(node)).PreimageKey(), node)
-		if err != nil {
+		key := preimage.Keccak256Key(crypto.Keccak256Hash(node)).PreimageKey()
+		if err := p.kvStore.Put(key, node); errors.Is(err, kvstore.ErrAlreadyExists) {
+			// It's not uncommon for different tries to contain common nodes (esp for receipts)
+			continue
+		} else if err != nil {
 			return fmt.Errorf("failed to store node: %w", err)
 		}
 	}

op-program/host/prefetcher/prefetcher_test.go

@@ -129,6 +129,28 @@ func TestFetchL1Receipts(t *testing.T) {
 		require.EqualValues(t, hash, header.Hash())
 		assertReceiptsEqual(t, receipts, actualReceipts)
 	})
+
+	// Blocks may have identical RLP receipts for different transactions.
+	// Check that the node already existing is handled
+	t.Run("CommonTrieNodes", func(t *testing.T) {
+		prefetcher, l1Cl, _, kv := createPrefetcher(t)
+		l1Cl.ExpectInfoByHash(hash, eth.BlockToInfo(block), nil)
+		l1Cl.ExpectInfoAndTxsByHash(hash, eth.BlockToInfo(block), block.Transactions(), nil)
+		l1Cl.ExpectFetchReceipts(hash, eth.BlockToInfo(block), receipts, nil)
+		defer l1Cl.AssertExpectations(t)
+
+		// Pre-store one receipt node (but not the whole trie leading to it)
+		// This would happen if an identical receipt was in an earlier block
+		opaqueRcpts, err := eth.EncodeReceipts(receipts)
+		require.NoError(t, err)
+		_, nodes := mpt.WriteTrie(opaqueRcpts)
+		require.NoError(t, kv.Put(preimage.Keccak256Key(crypto.Keccak256Hash(nodes[0])).PreimageKey(), nodes[0]))
+
+		oracle := l1.NewPreimageOracle(asOracleFn(t, prefetcher), asHinter(t, prefetcher))
+		header, actualReceipts := oracle.ReceiptsByBlockHash(hash)
+		require.EqualValues(t, hash, header.Hash())
+		assertReceiptsEqual(t, receipts, actualReceipts)
+	})
 }
 
 func TestFetchL2Block(t *testing.T) {

op-program/preimage/iface.go

@@ -55,6 +55,14 @@ func (k Keccak256Key) PreimageKey() (out common.Hash) {
 	return
 }
 
+func (k Keccak256Key) String() string {
+	return common.Hash(k).String()
+}
+
+func (k Keccak256Key) TerminalString() string {
+	return common.Hash(k).String()
+}
+
 // Hint is an interface to enable any program type to function as a hint,
 // when passed to the Hinter interface, returning a string representation
 // of what data the host should prepare pre-images for.

packages/atst/src/README.md

@@ -18,7 +18,7 @@ Vitest snapshots for the vitest tests
 
 CLI implementations of atst read and write
 
-## contants
+## constants
 
 Internal and external constants
 
@@ -32,4 +32,4 @@ Test helpers
 
 ## types
 
-Zod and typscript types
\ No newline at end of file
+Zod and typscript types