state-surgery: Add state sturgery (#2655)

See the README for an overview.
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

go.work

@@ -15,6 +15,7 @@ use (
 	./op-proposer
 	./proxyd
 	./teleportr
+	./state-surgery
 )
 
 replace github.com/ethereum/go-ethereum v1.10.17 => github.com/ethereum-optimism/reference-optimistic-geth v0.0.0-20220602230953-dd2e24b3359f

state-surgery/.gitignore

+surgery

state-surgery/Makefile

+surgery:
+	go build -o ./surgery ./cmd/main.go
+.PHONY: surgery

state-surgery/README.md

+# state-surgery
+
+This package performs state surgery. It takes the following input:
+
+1. A v0 database
+2. A partial `genesis.json`
+3. A list of addresses that transacted on the network prior to this past regenesis.
+4. A list of addresses that performed approvals on prior versions of the OVM ETH contract.
+
+It creates an initialized Bedrock Geth database as output. It does this by performing the following steps:
+
+1. Iterates over the old state.
+2. For each account in the old state, add that account and its storage to the new state after copying its balance from the OVM_ETH contract.
+3. Iterates over the pre-allocated accounts in the genesis file and adds them to the new state.
+4. Imports any accounts that have OVM ETH balances but aren't in state.
+5. Configures a genesis block in the new state using `genesis.json`.
+
+It performs the following integrity checks:
+
+1. OVM ETH storage slots must be completely accounted for.
+2. The total supply of OVM ETH migrated must match the total supply of the OVM ETH contract.
+
+This process takes about two hours on mainnet.
+
+Unlike previous iterations of our state surgery scripts, this one does not write results to a `genesis.json` file. This is for the following reasons:
+
+1. **Performance**. It's much faster to write binary to LevelDB than it is to write strings to a JSON file.
+2. **State Size**. There are nearly 1MM accounts on mainnet, which would create a genesis file several gigabytes in size. This is impossible for Geth to import without a large amount of memory, since the entire JSON gets buffered into memory. Importing the entire state database will be much faster, and can be done with fewer resources.
+
+## Data Files
+
+The following data files are used for mainnet:
+
+1. `mainnet-ovm-4-addresses.csv`: Contains all addresses that used OVM ETH during regenesis 4. Calculated by parsing Mint, Burn, and Transfer events from that network's OVM ETH contract.
+2. `mainnet-ovm-4-allowances.csv`: Contains all addresses that performed an approval on OVM ETH during regenesis 4 and who they approved. Calculated by parsing Approve events on that network's OVM ETH contract.
+
+These files are used to build the list of OVM ETH storage slots.
+
+## Compilation
+
+Run `make surgery`.
+
+## Usage
+
+```
+NAME:
+   surgery - migrates data from v0 to Bedrock
+
+USAGE:
+   surgery [global options] command [command options] [arguments...]
+
+COMMANDS:
+   dump-addresses  dumps addresses from OVM ETH
+   migrate         migrates state in OVM ETH
+   help, h         Shows a list of commands or help for one command
+
+GLOBAL OPTIONS:
+   --data-dir value, -d value  data directory to read
+   --help, -h                  show help (default: false)
+```

state-surgery/addresses.go

+package state_surgery
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+
+	l2grawdb "github.com/ethereum-optimism/optimism/l2geth/core/rawdb"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+var (
+	// AddressPreimagePrefix is the byte prefix of address preimages
+	// in Geth's database.
+	AddressPreimagePrefix = []byte("addr-preimage-")
+
+	// ErrStopIteration will stop iterators early when returned from the
+	// iterator's callback.
+	ErrStopIteration = errors.New("iteration stopped")
+
+	// MintTopic is the topic for mint events on OVM ETH.
+	MintTopic = common.HexToHash("0x0f6798a560793a54c3bcfe86a93cde1e73087d944c0ea20544137d4121396885")
+)
+
+type AddressCB func(address common.Address) error
+type AllowanceCB func(owner, spender common.Address) error
+
+// IterateDBAddresses iterates over each address in Geth's address
+// preimage database, calling the callback with the address.
+func IterateDBAddresses(inDB ethdb.Database, cb AddressCB) error {
+	iter := inDB.NewIterator(AddressPreimagePrefix, nil)
+	for iter.Next() {
+		if iter.Error() != nil {
+			return iter.Error()
+		}
+
+		addr := common.BytesToAddress(bytes.TrimPrefix(iter.Key(), AddressPreimagePrefix))
+		cbErr := cb(addr)
+		if cbErr == ErrStopIteration {
+			return nil
+		}
+		if cbErr != nil {
+			return cbErr
+		}
+	}
+	return iter.Error()
+}
+
+// IterateAddrList iterates over each address in an address list,
+// calling the callback with the address.
+func IterateAddrList(r io.Reader, cb AddressCB) error {
+	scan := bufio.NewScanner(r)
+	for scan.Scan() {
+		addrStr := scan.Text()
+		if !common.IsHexAddress(addrStr) {
+			return fmt.Errorf("invalid address %s", addrStr)
+		}
+		err := cb(common.HexToAddress(addrStr))
+		if err == ErrStopIteration {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// IterateAllowanceList iterates over each address in an allowance list,
+// calling the callback with the owner and the spender.
+func IterateAllowanceList(r io.Reader, cb AllowanceCB) error {
+	scan := bufio.NewScanner(r)
+	for scan.Scan() {
+		line := scan.Text()
+		splits := strings.Split(line, ",")
+		if len(splits) != 2 {
+			return fmt.Errorf("invalid allowance %s", line)
+		}
+		owner := splits[0]
+		spender := splits[1]
+		if !common.IsHexAddress(owner) {
+			return fmt.Errorf("invalid address %s", owner)
+		}
+		if !common.IsHexAddress(spender) {
+			return fmt.Errorf("invalid address %s", spender)
+		}
+		err := cb(common.HexToAddress(owner), common.HexToAddress(spender))
+		if err == ErrStopIteration {
+			return nil
+		}
+	}
+	return nil
+}
+
+// IterateMintEvents iterates over each mint event in the database starting
+// from head and stopping at genesis.
+func IterateMintEvents(inDB ethdb.Database, headNum uint64, cb AddressCB) error {
+	for headNum > 0 {
+		hash := l2grawdb.ReadCanonicalHash(inDB, headNum)
+		receipts := l2grawdb.ReadRawReceipts(inDB, hash, headNum)
+		for _, receipt := range receipts {
+			for _, l := range receipt.Logs {
+				if common.BytesToHash(l.Topics[0].Bytes()) != MintTopic {
+					continue
+				}
+
+				err := cb(common.BytesToAddress(l.Topics[1][12:]))
+				if errors.Is(err, ErrStopIteration) {
+					return nil
+				}
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		headNum--
+	}
+	return nil
+}

state-surgery/cmd/main.go

+package main
+
+import (
+	"os"
+	"strings"
+
+	surgery "github.com/ethereum-optimism/optimism/state-surgery"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/mattn/go-isatty"
+	"github.com/urfave/cli/v2"
+)
+
+func main() {
+	log.Root().SetHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd()))))
+
+	app := &cli.App{
+		Name:  "surgery",
+		Usage: "migrates data from v0 to Bedrock",
+		Commands: []*cli.Command{
+			{
+				Name:  "dump-addresses",
+				Usage: "dumps addresses from OVM ETH",
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "out-file",
+						Aliases:  []string{"o"},
+						Usage:    "file to write addresses to",
+						Required: true,
+					},
+				},
+				Action: dumpAddressesAction,
+			},
+			{
+				Name:  "migrate",
+				Usage: "migrates state in OVM ETH",
+				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:     "genesis-file",
+						Aliases:  []string{"g"},
+						Usage:    "path to a genesis file",
+						Required: true,
+					},
+					&cli.StringFlag{
+						Name:     "out-dir",
+						Aliases:  []string{"o"},
+						Usage:    "path to output directory",
+						Required: true,
+					},
+					&cli.StringFlag{
+						Name:     "address-lists",
+						Aliases:  []string{"a"},
+						Usage:    "comma-separated list of address files to read",
+						Required: true,
+					},
+					&cli.StringFlag{
+						Name:     "allowance-lists",
+						Aliases:  []string{"l"},
+						Usage:    "comma-separated list of allowance lists to read",
+						Required: true,
+					},
+					&cli.IntFlag{
+						Name:     "chain-id",
+						Usage:    "chain ID",
+						Value:    1,
+						Required: false,
+					},
+					&cli.IntFlag{
+						Name:     "leveldb-cache-size-mb",
+						Usage:    "leveldb cache size in MB",
+						Value:    16,
+						Required: false,
+					},
+					&cli.IntFlag{
+						Name:     "leveldb-file-handles",
+						Usage:    "leveldb file handles",
+						Value:    16,
+						Required: false,
+					},
+				},
+				Action: migrateAction,
+			},
+		},
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "data-dir",
+				Aliases:  []string{"d"},
+				Usage:    "data directory to read",
+				Required: true,
+			},
+		},
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Crit("error in migration", "err", err)
+	}
+}
+
+func dumpAddressesAction(cliCtx *cli.Context) error {
+	dataDir := cliCtx.String("data-dir")
+	outFile := cliCtx.String("out-file")
+	return surgery.DumpAddresses(dataDir, outFile)
+}
+
+func migrateAction(cliCtx *cli.Context) error {
+	dataDir := cliCtx.String("data-dir")
+	outDir := cliCtx.String("out-dir")
+	genesisPath := cliCtx.String("genesis-file")
+	addressLists := strings.Split(cliCtx.String("address-lists"), ",")
+	allowanceLists := strings.Split(cliCtx.String("allowance-lists"), ",")
+	chainID := cliCtx.Int("chain-id")
+	levelDBCacheSize := cliCtx.Int("leveldb-cache-size-mb")
+	levelDBHandles := cliCtx.Int("leveldb-file-handles")
+
+	genesis, err := surgery.ReadGenesisFromFile(genesisPath)
+	if err != nil {
+		return err
+	}
+
+	return surgery.Migrate(dataDir, outDir, genesis, addressLists, allowanceLists, chainID, levelDBCacheSize, levelDBHandles)
+}

state-surgery/db.go

+package state_surgery
+
+import (
+	"path/filepath"
+
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// MustOpenDB opens a Geth database, or panics. Note that
+// the database must be opened with a freezer in order to
+// properly read historical data.
+func MustOpenDB(dataDir string) ethdb.Database {
+	return MustOpenDBWithCacheOpts(dataDir, 0, 0)
+}
+
+// MustOpenDBWithCacheOpts opens a Geth database or panics. Allows
+// the caller to pass in LevelDB cache parameters.
+func MustOpenDBWithCacheOpts(dataDir string, cacheSize, handles int) ethdb.Database {
+	dir := filepath.Join(dataDir, "geth", "chaindata")
+	db, err := rawdb.NewLevelDBDatabaseWithFreezer(
+		dir,
+		cacheSize,
+		handles,
+		filepath.Join(dir, "ancient"),
+		"",
+		true,
+	)
+	if err != nil {
+		log.Crit("error opening raw DB", "err", err)
+	}
+	return db
+}

state-surgery/genesis.go

+package state_surgery
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+
+	"github.com/ethereum/go-ethereum/core"
+)
+
+// ReadGenesisFromFile reads a genesis object from a file.
+func ReadGenesisFromFile(path string) (*core.Genesis, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return ReadGenesis(f)
+}
+
+// ReadGenesis reads a genesis object from an io.Reader.
+func ReadGenesis(r io.Reader) (*core.Genesis, error) {
+	genesis := new(core.Genesis)
+	if err := json.NewDecoder(r).Decode(genesis); err != nil {
+		return nil, err
+	}
+	return genesis, nil
+}

state-surgery/go.mod

+module github.com/ethereum-optimism/optimism/state-surgery
+
+go 1.18
+
+require (
+	github.com/ethereum/go-ethereum v1.10.17
+	github.com/mattn/go-isatty v0.0.12
+	github.com/urfave/cli/v2 v2.3.0
+)
+
+require (
+	github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/russross/blackfriday/v2 v2.0.1 // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
+)

state-surgery/params.go

+package state_surgery
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// Params contains the configuration parameters used for verifying
+// the integrity of the migration.
+type Params struct {
+	// KnownMissingKeys is a set of known OVM ETH storage keys that are unaccounted for.
+	KnownMissingKeys map[common.Hash]bool
+
+	// ExpectedSupplyDelta is the expected delta between the total supply of OVM ETH,
+	// and ETH we were able to migrate. This is used to account for supply bugs in
+	//previous regenesis events.
+	ExpectedSupplyDelta *big.Int
+}
+
+var ParamsByChainID = map[int]*Params{
+	1: {
+		// These storage keys were unaccounted for in the genesis state of regenesis 5.
+		map[common.Hash]bool{
+			common.HexToHash("0x8632b3478ce27e6c2251f16f71bf134373ff9d23cff5b8d5f95475fa6e52fe22"): true,
+			common.HexToHash("0x47c25b07402d92e0d7f0cd9e347329fa0d86d16717cf933f836732313929fc1f"): true,
+			common.HexToHash("0x2acc0ec5cc86ffda9ceba005a317bcf0e86863e11be3981e923d5b103990055d"): true,
+		},
+		// Regenesis 4 contained a supply bug.
+		new(big.Int).SetUint64(1637102600003999992),
+	},
+}

state-surgery/storage.go

+package state_surgery
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"golang.org/x/crypto/sha3"
+)
+
+// BytesBacked is a re-export of the same interface in Geth,
+// which is unfortunately private.
+type BytesBacked interface {
+	Bytes() []byte
+}
+
+// CalcAllowanceStorageKey calculates the storage key of an allowance in OVM ETH.
+func CalcAllowanceStorageKey(owner common.Address, spender common.Address) common.Hash {
+	inner := CalcStorageKey(owner, common.Big1)
+	return CalcStorageKey(spender, inner)
+}
+
+// CalcOVMETHStorageKey calculates the storage key of an OVM ETH balance.
+func CalcOVMETHStorageKey(addr common.Address) common.Hash {
+	return CalcStorageKey(addr, common.Big0)
+}
+
+// CalcStorageKey is a helper method to calculate storage keys.
+func CalcStorageKey(a, b BytesBacked) common.Hash {
+	hasher := sha3.NewLegacyKeccak256()
+	hasher.Write(common.LeftPadBytes(a.Bytes(), 32))
+	hasher.Write(common.LeftPadBytes(b.Bytes(), 32))
+	digest := hasher.Sum(nil)
+	return common.BytesToHash(digest)
+}

state-surgery/util.go

+package state_surgery
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+func wrapErr(err error, msg string, ctx ...any) error {
+	return fmt.Errorf("%s: %w", fmt.Sprintf(msg, ctx...), err)
+}
+
+func ProgressLogger(n int, msg string) func() {
+	var i int
+
+	return func() {
+		i++
+		if i%n != 0 {
+			return
+		}
+		log.Info(msg, "count", i)
+	}
+}