Merge pull request #5151 from ethereum-optimism/feat/parallel-migration

op-chain-ops: Add parallel migration

op-chain-ops/cmd/check-migration/main.go

+package main
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"os"
+	"strings"
+
+	"github.com/ethereum-optimism/optimism/op-chain-ops/crossdomain"
+
+	"github.com/ethereum-optimism/optimism/op-chain-ops/db"
+	"github.com/mattn/go-isatty"
+
+	"github.com/ethereum-optimism/optimism/op-node/eth"
+	"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/ethereum/go-ethereum/core/types"
+
+	"github.com/ethereum/go-ethereum/log"
+
+	"github.com/ethereum-optimism/optimism/op-bindings/hardhat"
+
+	"github.com/ethereum-optimism/optimism/op-chain-ops/genesis"
+	"github.com/ethereum/go-ethereum/ethclient"
+
+	"github.com/urfave/cli"
+)
+
+func main() {
+	log.Root().SetHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd()))))
+
+	app := &cli.App{
+		Name:  "check-migration",
+		Usage: "Run sanity checks on a migrated database",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "l1-rpc-url",
+				Value:    "http://127.0.0.1:8545",
+				Usage:    "RPC URL for an L1 Node",
+				Required: true,
+			},
+			&cli.StringFlag{
+				Name:     "ovm-addresses",
+				Usage:    "Path to ovm-addresses.json",
+				Required: true,
+			},
+			&cli.StringFlag{
+				Name:     "ovm-allowances",
+				Usage:    "Path to ovm-allowances.json",
+				Required: true,
+			},
+			&cli.StringFlag{
+				Name:     "ovm-messages",
+				Usage:    "Path to ovm-messages.json",
+				Required: true,
+			},
+			&cli.StringFlag{
+				Name:     "witness-file",
+				Usage:    "Path to witness file",
+				Required: true,
+			},
+			&cli.StringFlag{
+				Name:     "db-path",
+				Usage:    "Path to database",
+				Required: true,
+			},
+			cli.StringFlag{
+				Name:     "deploy-config",
+				Usage:    "Path to hardhat deploy config file",
+				Required: true,
+			},
+			cli.StringFlag{
+				Name:     "network",
+				Usage:    "Name of hardhat deploy network",
+				Required: true,
+			},
+			cli.StringFlag{
+				Name:     "hardhat-deployments",
+				Usage:    "Comma separated list of hardhat deployment directories",
+				Required: true,
+			},
+			cli.IntFlag{
+				Name:  "db-cache",
+				Usage: "LevelDB cache size in mb",
+				Value: 1024,
+			},
+			cli.IntFlag{
+				Name:  "db-handles",
+				Usage: "LevelDB number of handles",
+				Value: 60,
+			},
+		},
+		Action: func(ctx *cli.Context) error {
+			deployConfig := ctx.String("deploy-config")
+			config, err := genesis.NewDeployConfig(deployConfig)
+			if err != nil {
+				return err
+			}
+
+			ovmAddresses, err := crossdomain.NewAddresses(ctx.String("ovm-addresses"))
+			if err != nil {
+				return err
+			}
+			ovmAllowances, err := crossdomain.NewAllowances(ctx.String("ovm-allowances"))
+			if err != nil {
+				return err
+			}
+			ovmMessages, err := crossdomain.NewSentMessageFromJSON(ctx.String("ovm-messages"))
+			if err != nil {
+				return err
+			}
+			evmMessages, evmAddresses, err := crossdomain.ReadWitnessData(ctx.String("witness-file"))
+			if err != nil {
+				return err
+			}
+
+			log.Info(
+				"Loaded witness data",
+				"ovmAddresses", len(ovmAddresses),
+				"evmAddresses", len(evmAddresses),
+				"ovmAllowances", len(ovmAllowances),
+				"ovmMessages", len(ovmMessages),
+				"evmMessages", len(evmMessages),
+			)
+
+			migrationData := crossdomain.MigrationData{
+				OvmAddresses:  ovmAddresses,
+				EvmAddresses:  evmAddresses,
+				OvmAllowances: ovmAllowances,
+				OvmMessages:   ovmMessages,
+				EvmMessages:   evmMessages,
+			}
+
+			network := ctx.String("network")
+			deployments := strings.Split(ctx.String("hardhat-deployments"), ",")
+			hh, err := hardhat.New(network, []string{}, deployments)
+			if err != nil {
+				return err
+			}
+
+			l1RpcURL := ctx.String("l1-rpc-url")
+			l1Client, err := ethclient.Dial(l1RpcURL)
+			if err != nil {
+				return err
+			}
+
+			var block *types.Block
+			tag := config.L1StartingBlockTag
+			if tag.BlockNumber != nil {
+				block, err = l1Client.BlockByNumber(context.Background(), big.NewInt(tag.BlockNumber.Int64()))
+			} else if tag.BlockHash != nil {
+				block, err = l1Client.BlockByHash(context.Background(), *tag.BlockHash)
+			} else {
+				return fmt.Errorf("invalid l1StartingBlockTag in deploy config: %v", tag)
+			}
+			if err != nil {
+				return err
+			}
+
+			dbCache := ctx.Int("db-cache")
+			dbHandles := ctx.Int("db-handles")
+
+			// Read the required deployment addresses from disk if required
+			if err := config.GetDeployedAddresses(hh); err != nil {
+				return err
+			}
+
+			if err := config.Check(); err != nil {
+				return err
+			}
+
+			postLDB, err := db.Open(ctx.String("db-path"), dbCache, dbHandles)
+			if err != nil {
+				return err
+			}
+
+			if err := genesis.PostCheckMigratedDB(
+				postLDB,
+				migrationData,
+				&config.L1CrossDomainMessengerProxy,
+				config.L1ChainID,
+				config.FinalSystemOwner,
+				config.ProxyAdminOwner,
+				&derive.L1BlockInfo{
+					Number:        block.NumberU64(),
+					Time:          block.Time(),
+					BaseFee:       block.BaseFee(),
+					BlockHash:     block.Hash(),
+					BatcherAddr:   config.BatchSenderAddress,
+					L1FeeOverhead: eth.Bytes32(common.BigToHash(new(big.Int).SetUint64(config.GasPriceOracleOverhead))),
+					L1FeeScalar:   eth.Bytes32(common.BigToHash(new(big.Int).SetUint64(config.GasPriceOracleScalar))),
+				},
+			); err != nil {
+				return err
+			}
+
+			if err := postLDB.Close(); err != nil {
+				return err
+			}
+
+			return nil
+		},
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Crit("error in migration", "err", err)
+	}
+}

op-chain-ops/cmd/op-migrate/main.go

@@ -106,6 +106,11 @@ func main() {
 				Value:    "rollup.json",
 				Required: true,
 			},
+			cli.BoolFlag{
+				Name:     "post-check-only",
+				Usage:    "Only perform sanity checks",
+				Required: false,
+			},
 		},
 		Action: func(ctx *cli.Context) error {
 			deployConfig := ctx.String("deploy-config")

op-chain-ops/ether/cli.go

@@ -25,8 +25,8 @@ func GetOVMETHTotalSupplySlot() common.Hash {
 	return getOVMETHTotalSupplySlot()
 }
 
-// getOVMETHBalance gets a user's OVM ETH balance from state by querying the
+// GetOVMETHBalance gets a user's OVM ETH balance from state by querying the
 // appropriate storage slot directly.
-func getOVMETHBalance(db *state.StateDB, addr common.Address) *big.Int {
+func GetOVMETHBalance(db *state.StateDB, addr common.Address) *big.Int {
 	return db.GetState(OVMETHAddress, CalcOVMETHStorageKey(addr)).Big()
 }

op-chain-ops/ether/migrate.go

@@ -29,7 +29,9 @@ var (
 	}
 )
 
-func MigrateLegacyETH(db *state.StateDB, addresses []common.Address, chainID int, noCheck bool) error {
+type FilteredOVMETHAddresses []common.Address
+
+func MigrateLegacyETH(db *state.StateDB, addresses FilteredOVMETHAddresses, chainID int, noCheck bool) error {
 	// Chain params to use for integrity checking.
 	params := crossdomain.ParamsByChainID[chainID]
 	if params == nil {
@@ -39,28 +41,15 @@ func MigrateLegacyETH(db *state.StateDB, addresses []common.Address, chainID int
 	// Log the chain params for debugging purposes.
 	log.Info("Chain params", "chain-id", chainID, "supply-delta", params.ExpectedSupplyDelta)
 
-	// Deduplicate the list of addresses by converting to a map.
-	deduped := make(map[common.Address]bool)
-	for _, addr := range addresses {
-		deduped[addr] = true
-	}
-
 	// Migrate the legacy ETH to ETH.
 	log.Info("Migrating legacy ETH to ETH", "num-accounts", len(addresses))
 	totalMigrated := new(big.Int)
 	logAccountProgress := util.ProgressLogger(1000, "imported accounts")
-	for addr := range deduped {
-		// No accounts should have a balance in state. If they do, bail.
-		if db.GetBalance(addr).Sign() > 0 {
-			if noCheck {
-				log.Error("account has non-zero balance in state - should never happen", "addr", addr)
-			} else {
-				log.Crit("account has non-zero balance in state - should never happen", "addr", addr)
-			}
-		}
+	for _, addr := range addresses {
+		// Balances are pre-checked not have any balances in state.
 
 		// Pull out the OVM ETH balance.
-		ovmBalance := getOVMETHBalance(db, addr)
+		ovmBalance := GetOVMETHBalance(db, addr)
 
 		// Actually perform the migration by setting the appropriate values in state.
 		db.SetBalance(addr, ovmBalance)

op-chain-ops/ether/precheck_test.go

+package ether
+
+import (
+	"bytes"
+	"math/big"
+	"math/rand"
+	"os"
+	"sort"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/log"
+
+	"github.com/ethereum-optimism/optimism/op-chain-ops/crossdomain"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPreCheckBalances(t *testing.T) {
+	log.Root().SetHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(true)))
+
+	tests := []struct {
+		name            string
+		totalSupply     *big.Int
+		expDiff         *big.Int
+		stateBalances   map[common.Address]*big.Int
+		stateAllowances map[common.Address]common.Address
+		inputAddresses  []common.Address
+		inputAllowances []*crossdomain.Allowance
+		check           func(t *testing.T, addrs FilteredOVMETHAddresses, err error)
+	}{
+		{
+			name:        "everything matches",
+			totalSupply: big.NewInt(3),
+			expDiff:     big.NewInt(0),
+			stateBalances: map[common.Address]*big.Int{
+				common.HexToAddress("0x123"): big.NewInt(1),
+				common.HexToAddress("0x456"): big.NewInt(2),
+			},
+			stateAllowances: map[common.Address]common.Address{
+				common.HexToAddress("0x123"): common.HexToAddress("0x456"),
+			},
+			inputAddresses: []common.Address{
+				common.HexToAddress("0x123"),
+				common.HexToAddress("0x456"),
+			},
+			inputAllowances: []*crossdomain.Allowance{
+				{
+					From: common.HexToAddress("0x123"),
+					To:   common.HexToAddress("0x456"),
+				},
+			},
+			check: func(t *testing.T, addrs FilteredOVMETHAddresses, err error) {
+				require.NoError(t, err)
+				require.EqualValues(t, FilteredOVMETHAddresses{
+					common.HexToAddress("0x123"),
+					common.HexToAddress("0x456"),
+				}, addrs)
+			},
+		},
+		{
+			name:        "extra input addresses",
+			totalSupply: big.NewInt(1),
+			expDiff:     big.NewInt(0),
+			stateBalances: map[common.Address]*big.Int{
+				common.HexToAddress("0x123"): big.NewInt(1),
+			},
+			inputAddresses: []common.Address{
+				common.HexToAddress("0x123"),
+				common.HexToAddress("0x456"),
+			},
+			check: func(t *testing.T, addrs FilteredOVMETHAddresses, err error) {
+				require.NoError(t, err)
+				require.EqualValues(t, FilteredOVMETHAddresses{
+					common.HexToAddress("0x123"),
+				}, addrs)
+			},
+		},
+		{
+			name:        "extra input allowances",
+			totalSupply: big.NewInt(1),
+			expDiff:     big.NewInt(0),
+			stateBalances: map[common.Address]*big.Int{
+				common.HexToAddress("0x123"): big.NewInt(1),
+			},
+			stateAllowances: map[common.Address]common.Address{
+				common.HexToAddress("0x123"): common.HexToAddress("0x456"),
+			},
+			inputAddresses: []common.Address{
+				common.HexToAddress("0x123"),
+				common.HexToAddress("0x456"),
+			},
+			inputAllowances: []*crossdomain.Allowance{
+				{
+					From: common.HexToAddress("0x123"),
+					To:   common.HexToAddress("0x456"),
+				},
+				{
+					From: common.HexToAddress("0x123"),
+					To:   common.HexToAddress("0x789"),
+				},
+			},
+			check: func(t *testing.T, addrs FilteredOVMETHAddresses, err error) {
+				require.NoError(t, err)
+				require.EqualValues(t, FilteredOVMETHAddresses{
+					common.HexToAddress("0x123"),
+				}, addrs)
+			},
+		},
+		{
+			name:        "missing input addresses",
+			totalSupply: big.NewInt(2),
+			expDiff:     big.NewInt(0),
+			stateBalances: map[common.Address]*big.Int{
+				common.HexToAddress("0x123"): big.NewInt(1),
+				common.HexToAddress("0x456"): big.NewInt(1),
+			},
+			inputAddresses: []common.Address{
+				common.HexToAddress("0x123"),
+			},
+			check: func(t *testing.T, addrs FilteredOVMETHAddresses, err error) {
+				require.Error(t, err)
+				require.ErrorContains(t, err, "unknown storage slot")
+			},
+		},
+		{
+			name:        "missing input allowances",
+			totalSupply: big.NewInt(2),
+			expDiff:     big.NewInt(0),
+			stateBalances: map[common.Address]*big.Int{
+				common.HexToAddress("0x123"): big.NewInt(1),
+			},
+			stateAllowances: map[common.Address]common.Address{
+				common.HexToAddress("0x123"): common.HexToAddress("0x456"),
+				common.HexToAddress("0x123"): common.HexToAddress("0x789"),
+			},
+			inputAddresses: []common.Address{
+				common.HexToAddress("0x123"),
+			},
+			inputAllowances: []*crossdomain.Allowance{
+				{
+					From: common.HexToAddress("0x123"),
+					To:   common.HexToAddress("0x456"),
+				},
+			},
+			check: func(t *testing.T, addrs FilteredOVMETHAddresses, err error) {
+				require.Error(t, err)
+				require.ErrorContains(t, err, "unknown storage slot")
+			},
+		},
+		{
+			name:        "bad supply diff",
+			totalSupply: big.NewInt(4),
+			expDiff:     big.NewInt(0),
+			stateBalances: map[common.Address]*big.Int{
+				common.HexToAddress("0x123"): big.NewInt(1),
+				common.HexToAddress("0x456"): big.NewInt(2),
+			},
+			inputAddresses: []common.Address{
+				common.HexToAddress("0x123"),
+				common.HexToAddress("0x456"),
+			},
+			check: func(t *testing.T, addrs FilteredOVMETHAddresses, err error) {
+				require.Error(t, err)
+				require.ErrorContains(t, err, "supply mismatch")
+			},
+		},
+		{
+			name:        "good supply diff",
+			totalSupply: big.NewInt(4),
+			expDiff:     big.NewInt(1),
+			stateBalances: map[common.Address]*big.Int{
+				common.HexToAddress("0x123"): big.NewInt(1),
+				common.HexToAddress("0x456"): big.NewInt(2),
+			},
+			inputAddresses: []common.Address{
+				common.HexToAddress("0x123"),
+				common.HexToAddress("0x456"),
+			},
+			check: func(t *testing.T, addrs FilteredOVMETHAddresses, err error) {
+				require.NoError(t, err)
+				require.EqualValues(t, FilteredOVMETHAddresses{
+					common.HexToAddress("0x123"),
+					common.HexToAddress("0x456"),
+				}, addrs)
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			db := makeLegacyETH(t, tt.totalSupply, tt.stateBalances, tt.stateAllowances)
+			factory := func() (*state.StateDB, error) {
+				return db, nil
+			}
+			addrs, err := doMigration(factory, tt.inputAddresses, tt.inputAllowances, tt.expDiff, false)
+
+			// Sort the addresses since they come in in a random order.
+			sort.Slice(addrs, func(i, j int) bool {
+				return bytes.Compare(addrs[i][:], addrs[j][:]) < 0
+			})
+
+			tt.check(t, addrs, err)
+		})
+	}
+}
+
+func makeLegacyETH(t *testing.T, totalSupply *big.Int, balances map[common.Address]*big.Int, allowances map[common.Address]common.Address) *state.StateDB {
+	db, err := state.New(common.Hash{}, state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{
+		Preimages: true,
+		Cache:     1024,
+	}), nil)
+	require.NoError(t, err)
+
+	db.CreateAccount(OVMETHAddress)
+	db.SetState(OVMETHAddress, getOVMETHTotalSupplySlot(), common.BigToHash(totalSupply))
+
+	for slot := range ignoredSlots {
+		if slot == getOVMETHTotalSupplySlot() {
+			continue
+		}
+		db.SetState(OVMETHAddress, slot, common.Hash{31: 0xff})
+	}
+	for addr, balance := range balances {
+		db.SetState(OVMETHAddress, CalcOVMETHStorageKey(addr), common.BigToHash(balance))
+	}
+	for from, to := range allowances {
+		db.SetState(OVMETHAddress, CalcAllowanceStorageKey(from, to), common.BigToHash(big.NewInt(1)))
+	}
+
+	root, err := db.Commit(false)
+	require.NoError(t, err)
+
+	err = db.Database().TrieDB().Commit(root, true)
+	require.NoError(t, err)
+
+	return db
+}
+
+// TestPreCheckBalancesRandom tests that the pre-check balances function works
+// with random addresses. This test makes sure that the partition logic doesn't
+// miss anything.
+func TestPreCheckBalancesRandom(t *testing.T) {
+	addresses := make([]common.Address, 0)
+	stateBalances := make(map[common.Address]*big.Int)
+
+	allowances := make([]*crossdomain.Allowance, 0)
+	stateAllowances := make(map[common.Address]common.Address)
+
+	totalSupply := big.NewInt(0)
+
+	for i := 0; i < 100; i++ {
+		for i := 0; i < rand.Intn(1000); i++ {
+			addr := randAddr(t)
+			addresses = append(addresses, addr)
+			stateBalances[addr] = big.NewInt(int64(rand.Intn(1_000_000)))
+			totalSupply = new(big.Int).Add(totalSupply, stateBalances[addr])
+		}
+
+		sort.Slice(addresses, func(i, j int) bool {
+			return bytes.Compare(addresses[i][:], addresses[j][:]) < 0
+		})
+
+		for i := 0; i < rand.Intn(1000); i++ {
+			addr := randAddr(t)
+			to := randAddr(t)
+			allowances = append(allowances, &crossdomain.Allowance{
+				From: addr,
+				To:   to,
+			})
+			stateAllowances[addr] = to
+		}
+
+		db := makeLegacyETH(t, totalSupply, stateBalances, stateAllowances)
+		factory := func() (*state.StateDB, error) {
+			return db, nil
+		}
+
+		outAddrs, err := doMigration(factory, addresses, allowances, big.NewInt(0), false)
+		require.NoError(t, err)
+
+		sort.Slice(outAddrs, func(i, j int) bool {
+			return bytes.Compare(outAddrs[i][:], outAddrs[j][:]) < 0
+		})
+		require.EqualValues(t, addresses, outAddrs)
+	}
+}
+
+func randAddr(t *testing.T) common.Address {
+	var addr common.Address
+	_, err := rand.Read(addr[:])
+	require.NoError(t, err)
+	return addr
+}

op-chain-ops/genesis/check.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"math/big"
 
+	"github.com/ethereum-optimism/optimism/op-chain-ops/ether"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
@@ -251,7 +253,7 @@ func PostCheckPredeploys(prevDB, currDB *state.StateDB) error {
 
 		// Balances and nonces should match legacy
 		oldNonce := prevDB.GetNonce(addr)
-		oldBalance := prevDB.GetBalance(addr)
+		oldBalance := ether.GetOVMETHBalance(prevDB, addr)
 		newNonce := currDB.GetNonce(addr)
 		newBalance := currDB.GetBalance(addr)
 		if oldNonce != newNonce {
@@ -543,7 +545,7 @@ func CheckWithdrawalsAfter(db vm.StateDB, data crossdomain.MigrationData, l1Cros
 
 		// If the sender is _not_ the L2XDM, the value should not be migrated.
 		wd := wdsByOldSlot[key]
-		if wd.XDomainSender == predeploys.L2CrossDomainMessengerAddr {
+		if wd.MessageSender == predeploys.L2CrossDomainMessengerAddr {
 			// Make sure the value is abiTrue if this withdrawal should be migrated.
 			if migratedValue != abiTrue {
 				innerErr = fmt.Errorf("expected migrated value to be true, but got %s", migratedValue)
@@ -552,7 +554,7 @@ func CheckWithdrawalsAfter(db vm.StateDB, data crossdomain.MigrationData, l1Cros
 		} else {
 			// Otherwise, ensure that withdrawals from senders other than the L2XDM are _not_ migrated.
 			if migratedValue != abiFalse {
-				innerErr = fmt.Errorf("a migration from a sender other than the L2XDM was migrated")
+				innerErr = fmt.Errorf("a migration from a sender other than the L2XDM was migrated. sender: %s, migrated value: %s", wd.MessageSender, migratedValue)
 				return false
 			}
 		}

op-chain-ops/genesis/db_migration.go

@@ -82,6 +82,7 @@ func MigrateDB(ldb ethdb.Database, config *DeployConfig, l1Block *types.Block, m
 		)
 	}
 
+	dbFactory := func() (*state.StateDB, error) {
 		// Set up the backing store.
 		underlyingDB := state.NewDatabaseWithConfig(ldb, &trie.Config{
 			Preimages: true,
@@ -94,6 +95,14 @@ func MigrateDB(ldb ethdb.Database, config *DeployConfig, l1Block *types.Block, m
 			return nil, fmt.Errorf("cannot open StateDB: %w", err)
 		}
 
+		return db, nil
+	}
+
+	db, err := dbFactory()
+	if err != nil {
+		return nil, fmt.Errorf("cannot create StateDB: %w", err)
+	}
+
 	// Before we do anything else, we need to ensure that all of the input configuration is correct
 	// and nothing is missing. We'll first verify the contract configuration, then we'll verify the
 	// witness data for the migration. We operate under the assumption that the witness data is
@@ -139,7 +148,7 @@ func MigrateDB(ldb ethdb.Database, config *DeployConfig, l1Block *types.Block, m
 	// Unlike with withdrawals, we do not need to filter out extra addresses because their balances
 	// would necessarily be zero and therefore not affect the migration.
 	log.Info("Checking addresses...", "no-check", noCheck)
-	addrs, err := ether.PreCheckBalances(ldb, db, migrationData.Addresses(), migrationData.OvmAllowances, int(config.L1ChainID), noCheck)
+	addrs, err := ether.PreCheckBalances(dbFactory, migrationData.Addresses(), migrationData.OvmAllowances, int(config.L1ChainID), noCheck)
 	if err != nil {
 		return nil, fmt.Errorf("addresses mismatch: %w", err)
 	}