Merge branch 'develop' into aj/faster-dockerfile

.circleci/config.yml

@@ -1142,7 +1142,7 @@ workflows:
           working_directory: proxyd
       - indexer-tests
       - go-lint-test-build:
-          name: op-heartbeat tests
+          name: op-heartbeat-tests
           binary_name: op-heartbeat
           working_directory: op-heartbeat
       - semgrep-scan
@@ -1224,7 +1224,11 @@ workflows:
           target: test-external-geth
       - bedrock-go-tests:
           requires:
+            - go-mod-tidy
+            - cannon-build-test-vectors
             - cannon-go-lint-and-test
+            - check-generated-mocks-op-node
+            - check-generated-mocks-op-service
             - op-batcher-lint
             - op-bootnode-lint
             - op-bindings-lint
@@ -1238,6 +1242,7 @@ workflows:
             - op-batcher-tests
             - op-bindings-tests
             - op-chain-ops-tests
+            - op-heartbeat-tests
             - op-node-tests
             - op-proposer-tests
             - op-challenger-tests
@@ -1245,6 +1250,7 @@ workflows:
             - op-service-tests
             - op-e2e-WS-tests
             - op-e2e-HTTP-tests
+            - op-e2e-ext-geth-tests
       - docker-build:
           name: op-node-docker-build
           docker_file: op-node/Dockerfile

codecov.yml

-comment:
-  layout: "reach, diff, flags, files"
-  behavior: default
-  require_changes: true    # only post the comment if coverage changes
+comment: false
 ignore:
   - "op-e2e"
   - "**/*.t.sol"

op-e2e/deposit_test.go

+package op_e2e
+
+import (
+	"context"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMintOnRevertedDeposit(t *testing.T) {
+	InitParallel(t)
+	cfg := DefaultSystemConfig(t)
+
+	sys, err := cfg.Start(t)
+	require.Nil(t, err, "Error starting up system")
+	defer sys.Close()
+
+	l1Client := sys.Clients["l1"]
+	l2Verif := sys.Clients["verifier"]
+
+	// create signer
+	aliceKey := cfg.Secrets.Alice
+	opts, err := bind.NewKeyedTransactorWithChainID(aliceKey, cfg.L1ChainIDBig())
+	require.Nil(t, err)
+	fromAddr := opts.From
+
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+	startBalance, err := l2Verif.BalanceAt(ctx, fromAddr, nil)
+	cancel()
+	require.Nil(t, err)
+
+	ctx, cancel = context.WithTimeout(context.Background(), 1*time.Second)
+	startNonce, err := l2Verif.NonceAt(ctx, fromAddr, nil)
+	require.NoError(t, err)
+	cancel()
+
+	toAddr := common.Address{0xff, 0xff}
+	mintAmount := big.NewInt(9_000_000)
+	opts.Value = mintAmount
+	SendDepositTx(t, cfg, l1Client, l2Verif, opts, func(l2Opts *DepositTxOpts) {
+		l2Opts.ToAddr = toAddr
+		// trigger a revert by transferring more than we have available
+		l2Opts.Value = new(big.Int).Mul(common.Big2, startBalance)
+		l2Opts.ExpectedStatus = types.ReceiptStatusFailed
+	})
+
+	// Confirm balance
+	ctx, cancel = context.WithTimeout(context.Background(), 1*time.Second)
+	endBalance, err := l2Verif.BalanceAt(ctx, fromAddr, nil)
+	cancel()
+	require.Nil(t, err)
+	ctx, cancel = context.WithTimeout(context.Background(), 1*time.Second)
+	toAddrBalance, err := l2Verif.BalanceAt(ctx, toAddr, nil)
+	require.NoError(t, err)
+	cancel()
+
+	diff := new(big.Int)
+	diff = diff.Sub(endBalance, startBalance)
+	require.Equal(t, mintAmount, diff, "Did not get expected balance change")
+	require.Equal(t, common.Big0.Int64(), toAddrBalance.Int64(), "The recipient account balance should be zero")
+
+	ctx, cancel = context.WithTimeout(context.Background(), 1*time.Second)
+	endNonce, err := l2Verif.NonceAt(ctx, fromAddr, nil)
+	require.NoError(t, err)
+	cancel()
+	require.Equal(t, startNonce+1, endNonce, "Nonce of deposit sender should increment on L2, even if the deposit fails")
+}
+
+func TestDepositTxCreateContract(t *testing.T) {
+	InitParallel(t)
+	cfg := DefaultSystemConfig(t)
+	delete(cfg.Nodes, "verifier")
+
+	sys, err := cfg.Start(t)
+	require.Nil(t, err, "Error starting up system")
+	defer sys.Close()
+
+	l1Client := sys.Clients["l1"]
+	l2Client := sys.Clients["sequencer"]
+
+	opts, err := bind.NewKeyedTransactorWithChainID(cfg.Secrets.Alice, cfg.L1ChainIDBig())
+	require.Nil(t, err)
+
+	// Simple constructor that is prefixed to the actual contract code
+	// Results in the contract code being returned as the code for the new contract
+	deployPrefixSize := byte(16)
+	deployPrefix := []byte{
+		// Copy input data after this prefix into memory starting at address 0x00
+		// CODECOPY arg size
+		byte(vm.PUSH1), deployPrefixSize,
+		byte(vm.CODESIZE),
+		byte(vm.SUB),
+		// CODECOPY arg offset
+		byte(vm.PUSH1), deployPrefixSize,
+		// CODECOPY arg destOffset
+		byte(vm.PUSH1), 0x00,
+		byte(vm.CODECOPY),
+
+		// Return code from memory
+		// RETURN arg size
+		byte(vm.PUSH1), deployPrefixSize,
+		byte(vm.CODESIZE),
+		byte(vm.SUB),
+		// RETURN arg offset
+		byte(vm.PUSH1), 0x00,
+		byte(vm.RETURN),
+	}
+	// Stores the first word from call data code to storage slot 0
+	sstoreContract := []byte{
+		// Load first word from call data
+		byte(vm.PUSH1), 0x00,
+		byte(vm.CALLDATALOAD),
+
+		// Store it to slot 0
+		byte(vm.PUSH1), 0x00,
+		byte(vm.SSTORE),
+	}
+
+	deployData := append(deployPrefix, sstoreContract...)
+
+	l2Receipt := SendDepositTx(t, cfg, l1Client, l2Client, opts, func(l2Opts *DepositTxOpts) {
+		l2Opts.Data = deployData
+		l2Opts.Value = common.Big0
+		l2Opts.IsCreation = true
+		l2Opts.ToAddr = common.Address{}
+		l2Opts.GasLimit = 1_000_000
+	})
+	require.NotEqual(t, common.Address{}, l2Receipt.ContractAddress, "should not have zero address")
+	code, err := l2Client.CodeAt(context.Background(), l2Receipt.ContractAddress, nil)
+	require.NoError(t, err, "get deployed contract code")
+	require.Equal(t, sstoreContract, code, "should have deployed correct contract code")
+}

op-e2e/e2eutils/geth/peers.go

+package geth
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/wait"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/stretchr/testify/require"
+)
+
+// ConnectP2P creates a p2p peer connection between node1 and node2.
+func ConnectP2P(t *testing.T, node1 *ethclient.Client, node2 *ethclient.Client) {
+	var targetInfo p2p.NodeInfo
+	require.NoError(t, node2.Client().Call(&targetInfo, "admin_nodeInfo"), "get node info")
+
+	var peerAdded bool
+	require.NoError(t, node1.Client().Call(&peerAdded, "admin_addPeer", targetInfo.Enode), "add peer")
+	require.True(t, peerAdded, "should have added peer successfully")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	err := wait.For(ctx, time.Second, func() (bool, error) {
+		var peerCount hexutil.Uint64
+		if err := node1.Client().Call(&peerCount, "net_peerCount"); err != nil {
+			return false, err
+		}
+		t.Logf("Peer count %v", uint64(peerCount))
+		return peerCount >= hexutil.Uint64(1), nil
+	})
+	require.NoError(t, err, "wait for a peer to be connected")
+}
+
+func WithP2P() func(ethCfg *ethconfig.Config, nodeCfg *node.Config) error {
+	return func(ethCfg *ethconfig.Config, nodeCfg *node.Config) error {
+		ethCfg.RollupDisableTxPoolGossip = false
+		nodeCfg.P2P = p2p.Config{
+			NoDiscovery: true,
+			ListenAddr:  "127.0.0.1:0",
+			MaxPeers:    10,
+		}
+		return nil
+	}
+}

op-e2e/l2_gossip_test.go

+package op_e2e
+
+import (
+	"testing"
+
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils/geth"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTxGossip(t *testing.T) {
+	InitParallel(t)
+	cfg := DefaultSystemConfig(t)
+	gethOpts := []GethOption{
+		geth.WithP2P(),
+	}
+	cfg.GethOptions["sequencer"] = gethOpts
+	cfg.GethOptions["verifier"] = gethOpts
+	sys, err := cfg.Start(t)
+	require.NoError(t, err, "Start system")
+
+	seqClient := sys.Clients["sequencer"]
+	verifClient := sys.Clients["verifier"]
+	geth.ConnectP2P(t, seqClient, verifClient)
+
+	// Send a transaction to the verifier and it should be gossiped to the sequencer and included in a block.
+	SendL2Tx(t, cfg, verifClient, cfg.Secrets.Alice, func(opts *TxOpts) {
+		opts.ToAddr = &common.Address{0xaa}
+		opts.Value = common.Big1
+		opts.VerifyOnClients(seqClient, verifClient)
+	})
+}

op-e2e/setup.go

@@ -84,7 +84,7 @@ func DefaultSystemConfig(t *testing.T) SystemConfig {
 	require.NoError(t, err)
 	deployConfig := config.DeployConfig.Copy()
 	deployConfig.L1GenesisBlockTimestamp = hexutil.Uint64(time.Now().Unix())
-	require.NoError(t, deployConfig.Check())
+	require.NoError(t, deployConfig.Check(), "Deploy config is invalid, do you need to run make devnet-allocs?")
 	l1Deployments := config.L1Deployments.Copy()
 	require.NoError(t, l1Deployments.Check())
 
@@ -457,7 +457,7 @@ func (cfg SystemConfig) Start(t *testing.T, _opts ...SystemConfigOption) (*Syste
 			ethClient = gethInst
 		} else {
 			if len(cfg.GethOptions[name]) > 0 {
-				t.Errorf("External L2 nodes do not support configuration through GethOptions")
+				t.Skip("External L2 nodes do not support configuration through GethOptions")
 			}
 			ethClient = (&ExternalRunner{
 				Name:    name,

op-e2e/system_test.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
-	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -337,67 +336,6 @@ func TestFinalize(t *testing.T) {
 	require.NotZerof(t, l2Finalized.NumberU64(), "must have finalized L2 block")
 }
 
-func TestMintOnRevertedDeposit(t *testing.T) {
-	InitParallel(t)
-	cfg := DefaultSystemConfig(t)
-
-	sys, err := cfg.Start(t)
-	require.Nil(t, err, "Error starting up system")
-	defer sys.Close()
-
-	l1Client := sys.Clients["l1"]
-	l2Verif := sys.Clients["verifier"]
-
-	l1Node := sys.EthInstances["l1"].(*GethInstance).Node
-
-	// create signer
-	ks := l1Node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
-	opts, err := bind.NewKeyStoreTransactorWithChainID(ks, ks.Accounts()[0], cfg.L1ChainIDBig())
-	require.Nil(t, err)
-	fromAddr := opts.From
-
-	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
-	startBalance, err := l2Verif.BalanceAt(ctx, fromAddr, nil)
-	cancel()
-	require.Nil(t, err)
-
-	ctx, cancel = context.WithTimeout(context.Background(), 1*time.Second)
-	startNonce, err := l2Verif.NonceAt(ctx, fromAddr, nil)
-	require.NoError(t, err)
-	cancel()
-
-	toAddr := common.Address{0xff, 0xff}
-	mintAmount := big.NewInt(9_000_000)
-	opts.Value = mintAmount
-	SendDepositTx(t, cfg, l1Client, l2Verif, opts, func(l2Opts *DepositTxOpts) {
-		l2Opts.ToAddr = toAddr
-		// trigger a revert by transferring more than we have available
-		l2Opts.Value = new(big.Int).Mul(common.Big2, startBalance)
-		l2Opts.ExpectedStatus = types.ReceiptStatusFailed
-	})
-
-	// Confirm balance
-	ctx, cancel = context.WithTimeout(context.Background(), 1*time.Second)
-	endBalance, err := l2Verif.BalanceAt(ctx, fromAddr, nil)
-	cancel()
-	require.Nil(t, err)
-	ctx, cancel = context.WithTimeout(context.Background(), 1*time.Second)
-	toAddrBalance, err := l2Verif.BalanceAt(ctx, toAddr, nil)
-	require.NoError(t, err)
-	cancel()
-
-	diff := new(big.Int)
-	diff = diff.Sub(endBalance, startBalance)
-	require.Equal(t, mintAmount, diff, "Did not get expected balance change")
-	require.Equal(t, common.Big0.Int64(), toAddrBalance.Int64(), "The recipient account balance should be zero")
-
-	ctx, cancel = context.WithTimeout(context.Background(), 1*time.Second)
-	endNonce, err := l2Verif.NonceAt(ctx, fromAddr, nil)
-	require.NoError(t, err)
-	cancel()
-	require.Equal(t, startNonce+1, endNonce, "Nonce of deposit sender should increment on L2, even if the deposit fails")
-}
-
 func TestMissingBatchE2E(t *testing.T) {
 	InitParallel(t)
 	// Note this test zeroes the balance of the batch-submitter to make the batches unable to go into L1.

op-e2e/tx_helper.go

@@ -21,7 +21,8 @@ import (
 // The L1 transaction, including sender, is configured by the l1Opts param.
 // The L2 transaction options can be configured by modifying the DepositTxOps value supplied to applyL2Opts
 // Will verify that the transaction is included with the expected status on L1 and L2
-func SendDepositTx(t *testing.T, cfg SystemConfig, l1Client *ethclient.Client, l2Client *ethclient.Client, l1Opts *bind.TransactOpts, applyL2Opts DepositTxOptsFn) {
+// Returns the receipt of the L2 transaction
+func SendDepositTx(t *testing.T, cfg SystemConfig, l1Client *ethclient.Client, l2Client *ethclient.Client, l1Opts *bind.TransactOpts, applyL2Opts DepositTxOptsFn) *types.Receipt {
 	l2Opts := defaultDepositTxOpts(l1Opts)
 	applyL2Opts(l2Opts)
 
@@ -38,16 +39,17 @@ func SendDepositTx(t *testing.T, cfg SystemConfig, l1Client *ethclient.Client, l
 	require.Nil(t, err, "with deposit tx")
 
 	// Wait for transaction on L1
-	receipt, err := waitForTransaction(tx.Hash(), l1Client, 10*time.Duration(cfg.DeployConfig.L1BlockTime)*time.Second)
+	l1Receipt, err := waitForTransaction(tx.Hash(), l1Client, 10*time.Duration(cfg.DeployConfig.L1BlockTime)*time.Second)
 	require.Nil(t, err, "Waiting for deposit tx on L1")
 
 	// Wait for transaction to be included on L2
-	reconstructedDep, err := derive.UnmarshalDepositLogEvent(receipt.Logs[0])
+	reconstructedDep, err := derive.UnmarshalDepositLogEvent(l1Receipt.Logs[0])
 	require.NoError(t, err, "Could not reconstruct L2 Deposit")
 	tx = types.NewTx(reconstructedDep)
-	receipt, err = waitForTransaction(tx.Hash(), l2Client, 10*time.Duration(cfg.DeployConfig.L2BlockTime)*time.Second)
+	l2Receipt, err := waitForTransaction(tx.Hash(), l2Client, 10*time.Duration(cfg.DeployConfig.L2BlockTime)*time.Second)
 	require.NoError(t, err)
-	require.Equal(t, l2Opts.ExpectedStatus, receipt.Status, "l2 transaction status")
+	require.Equal(t, l2Opts.ExpectedStatus, l2Receipt.Status, "l2 transaction status")
+	return l2Receipt
 }
 
 type DepositTxOptsFn func(l2Opts *DepositTxOpts)
@@ -91,16 +93,16 @@ func SendL2Tx(t *testing.T, cfg SystemConfig, l2Client *ethclient.Client, privKe
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	err := l2Client.SendTransaction(ctx, tx)
-	require.Nil(t, err, "Sending L2 tx")
+	require.NoError(t, err, "Sending L2 tx")
 
 	receipt, err := waitForTransaction(tx.Hash(), l2Client, 10*time.Duration(cfg.DeployConfig.L2BlockTime)*time.Second)
-	require.Nil(t, err, "Waiting for L2 tx")
+	require.NoError(t, err, "Waiting for L2 tx")
 	require.Equal(t, opts.ExpectedStatus, receipt.Status, "TX should have expected status")
 
 	for i, client := range opts.VerifyClients {
 		t.Logf("Waiting for tx %v on verification client %d", tx.Hash(), i)
 		receiptVerif, err := waitForTransaction(tx.Hash(), client, 10*time.Duration(cfg.DeployConfig.L2BlockTime)*time.Second)
-		require.Nilf(t, err, "Waiting for L2 tx on verification client %d", i)
+		require.NoErrorf(t, err, "Waiting for L2 tx on verification client %d", i)
 		require.Equalf(t, receipt, receiptVerif, "Receipts should be the same on sequencer and verification client %d", i)
 	}
 	return receipt

op-program/host/kvstore/disk.go

@@ -37,7 +37,7 @@ func (d *DiskKV) pathKey(k common.Hash) string {
 func (d *DiskKV) Put(k common.Hash, v []byte) error {
 	d.Lock()
 	defer d.Unlock()
-	f, err := os.CreateTemp(d.path, k.String()+".txt.*")
+	f, err := openTempFile(d.path, k.String()+".txt.*")
 	if err != nil {
 		return fmt.Errorf("failed to open temp file for pre-image %s: %w", k, err)
 	}
@@ -57,6 +57,21 @@ func (d *DiskKV) Put(k common.Hash, v []byte) error {
 	return nil
 }
 
+func openTempFile(dir string, nameTemplate string) (*os.File, error) {
+	f, err := os.CreateTemp(dir, nameTemplate)
+	// Directory has been deleted out from underneath us. Recreate it.
+	if errors.Is(err, os.ErrNotExist) {
+		if mkdirErr := os.MkdirAll(dir, 0777); mkdirErr != nil {
+			return nil, errors.Join(fmt.Errorf("failed to create directory %v: %w", dir, mkdirErr), err)
+		}
+		f, err = os.CreateTemp(dir, nameTemplate)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return f, nil
+}
+
 func (d *DiskKV) Get(k common.Hash) ([]byte, error) {
 	d.RLock()
 	defer d.RUnlock()

op-program/host/kvstore/disk_test.go

 package kvstore
 
-import "testing"
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/stretchr/testify/require"
+)
 
 func TestDiskKV(t *testing.T) {
 	tmp := t.TempDir() // automatically removed by testing cleanup
 	kv := NewDiskKV(tmp)
 	kvTest(t, kv)
 }
+
+func TestCreateMissingDirectory(t *testing.T) {
+	tmp := t.TempDir()
+	dir := filepath.Join(tmp, "data")
+	kv := NewDiskKV(dir)
+	val := []byte{1, 2, 3, 4}
+	key := crypto.Keccak256Hash(val)
+	require.NoError(t, kv.Put(key, val))
+}