Commit 21161265 authored by Adrian Sutton's avatar Adrian Sutton Committed by GitHub

cannon: Autodetect VM type from state (#11803)

* cannon: Autodetect VM type from state in run command

* cannon: Autodetect VM type from state in witness command

* cannon: Remove vm type flag from run and witness

* cannon: Only peek the version byte

* cannon: Move all version handling to VersionedState, simplify a lot and forbid serializing multithreaded states to JSON

* cannon: Rename method

* op-challenger: Update cannon state parsing to use version detecting methods

* cannon: Move CreateVM to FPVMState for simplicity

Test read/write/create for VersionedState

* cannon: Readd detect_test

* cannon: Remove json names from multithreaded.State.

Multithreaded states always use binary serialization.

* cannon: Move vmtype to load_elf since it is no longer shared.

* cannon: Ensure metadata is available and sleepCheck used even if debug is disabled.

* op-challenger: Update canon state loading test to cover multiple state versions.
parent af78edde
......@@ -147,7 +147,7 @@ cannon-prestate: op-program cannon ## Generates prestate using cannon and op-pro
cannon-prestate-mt: op-program cannon ## Generates prestate using cannon and op-program in the multithreaded cannon format
./cannon/bin/cannon load-elf --type cannon-mt --path op-program/bin/op-program-client.elf --out op-program/bin/prestate-mt.bin.gz --meta op-program/bin/meta-mt.json
./cannon/bin/cannon run --type cannon-mt --proof-at '=0' --stop-at '=1' --input op-program/bin/prestate-mt.bin.gz --meta op-program/bin/meta-mt.json --proof-fmt 'op-program/bin/%d-mt.json' --output ""
./cannon/bin/cannon run --proof-at '=0' --stop-at '=1' --input op-program/bin/prestate-mt.bin.gz --meta op-program/bin/meta-mt.json --proof-fmt 'op-program/bin/%d-mt.json' --output ""
mv op-program/bin/0-mt.json op-program/bin/prestate-proof-mt.json
.PHONY: cannon-prestate
......
......@@ -6,6 +6,7 @@ import (
"github.com/ethereum-optimism/optimism/cannon/mipsevm"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/multithreaded"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/versions"
"github.com/ethereum-optimism/optimism/cannon/serialize"
"github.com/ethereum-optimism/optimism/op-service/ioutil"
"github.com/urfave/cli/v2"
......@@ -16,6 +17,12 @@ import (
)
var (
LoadELFVMTypeFlag = &cli.StringFlag{
Name: "type",
Usage: "VM type to create state for. Options are 'cannon' (default), 'cannon-mt'",
Value: "cannon",
Required: false,
}
LoadELFPathFlag = &cli.PathFlag{
Name: "path",
Usage: "Path to 32-bit big-endian MIPS ELF file",
......@@ -42,9 +49,25 @@ var (
}
)
type VMType string
var (
cannonVMType VMType = "cannon"
mtVMType VMType = "cannon-mt"
)
func vmTypeFromString(ctx *cli.Context) (VMType, error) {
if vmTypeStr := ctx.String(LoadELFVMTypeFlag.Name); vmTypeStr == string(cannonVMType) {
return cannonVMType, nil
} else if vmTypeStr == string(mtVMType) {
return mtVMType, nil
} else {
return "", fmt.Errorf("unknown VM type %q", vmTypeStr)
}
}
func LoadELF(ctx *cli.Context) error {
var createInitialState func(f *elf.File) (mipsevm.FPVMState, error)
var writeState func(path string, state mipsevm.FPVMState) error
if vmType, err := vmTypeFromString(ctx); err != nil {
return err
......@@ -52,16 +75,10 @@ func LoadELF(ctx *cli.Context) error {
createInitialState = func(f *elf.File) (mipsevm.FPVMState, error) {
return program.LoadELF(f, singlethreaded.CreateInitialState)
}
writeState = func(path string, state mipsevm.FPVMState) error {
return serialize.Write[*singlethreaded.State](path, state.(*singlethreaded.State), OutFilePerm)
}
} else if vmType == mtVMType {
createInitialState = func(f *elf.File) (mipsevm.FPVMState, error) {
return program.LoadELF(f, multithreaded.CreateInitialState)
}
writeState = func(path string, state mipsevm.FPVMState) error {
return serialize.Write[*multithreaded.State](path, state.(*multithreaded.State), OutFilePerm)
}
} else {
return fmt.Errorf("invalid VM type: %q", vmType)
}
......@@ -97,7 +114,13 @@ func LoadELF(ctx *cli.Context) error {
if err := jsonutil.WriteJSON[*program.Metadata](meta, ioutil.ToStdOutOrFileOrNoop(ctx.Path(LoadELFMetaFlag.Name), OutFilePerm)); err != nil {
return fmt.Errorf("failed to output metadata: %w", err)
}
return writeState(ctx.Path(LoadELFOutFlag.Name), state)
// Ensure the state is written with appropriate version information
versionedState, err := versions.NewFromState(state)
if err != nil {
return fmt.Errorf("failed to create versioned state: %w", err)
}
return serialize.Write(ctx.Path(LoadELFOutFlag.Name), versionedState, OutFilePerm)
}
var LoadELFCommand = &cli.Command{
......@@ -106,7 +129,7 @@ var LoadELFCommand = &cli.Command{
Description: "Load ELF file into Cannon JSON state, optionally patch out functions",
Action: LoadELF,
Flags: []cli.Flag{
VMTypeFlag,
LoadELFVMTypeFlag,
LoadELFPathFlag,
LoadELFPatchFlag,
LoadELFOutFlag,
......
......@@ -10,7 +10,7 @@ import (
"strings"
"time"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/multithreaded"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/versions"
"github.com/ethereum-optimism/optimism/cannon/serialize"
"github.com/ethereum-optimism/optimism/op-service/ioutil"
"github.com/ethereum/go-ethereum/common"
......@@ -21,7 +21,6 @@ import (
"github.com/ethereum-optimism/optimism/cannon/mipsevm"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/program"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/singlethreaded"
preimage "github.com/ethereum-optimism/optimism/op-preimage"
"github.com/ethereum-optimism/optimism/op-service/jsonutil"
)
......@@ -279,11 +278,6 @@ func Run(ctx *cli.Context) error {
defer profile.Start(profile.NoShutdownHook, profile.ProfilePath("."), profile.CPUProfile).Stop()
}
vmType, err := vmTypeFromString(ctx)
if err != nil {
return err
}
guestLogger := Logger(os.Stderr, log.LevelInfo)
outLog := &mipsevm.LoggingWriter{Log: guestLogger.With("module", "guest", "stream", "stdout")}
errLog := &mipsevm.LoggingWriter{Log: guestLogger.With("module", "guest", "stream", "stderr")}
......@@ -373,42 +367,19 @@ func Run(ctx *cli.Context) error {
}
}
var vm mipsevm.FPVM
var debugProgram bool
if vmType == cannonVMType {
l.Info("Using cannon VM")
cannon, err := singlethreaded.NewInstrumentedStateFromFile(ctx.Path(RunInputFlag.Name), po, outLog, errLog, meta)
if err != nil {
return err
}
debugProgram = ctx.Bool(RunDebugFlag.Name)
if debugProgram {
if metaPath := ctx.Path(RunMetaFlag.Name); metaPath == "" {
return fmt.Errorf("cannot enable debug mode without a metadata file")
}
if err := cannon.InitDebug(); err != nil {
return fmt.Errorf("failed to initialize debug mode: %w", err)
}
}
vm = cannon
} else if vmType == mtVMType {
l.Info("Using cannon multithreaded VM")
cannon, err := multithreaded.NewInstrumentedStateFromFile(ctx.Path(RunInputFlag.Name), po, outLog, errLog, l)
if err != nil {
return err
state, err := versions.LoadStateFromFile(ctx.Path(RunInputFlag.Name))
if err != nil {
return fmt.Errorf("failed to load state: %w", err)
}
vm := state.CreateVM(l, po, outLog, errLog, meta)
debugProgram := ctx.Bool(RunDebugFlag.Name)
if debugProgram {
if metaPath := ctx.Path(RunMetaFlag.Name); metaPath == "" {
return fmt.Errorf("cannot enable debug mode without a metadata file")
}
debugProgram = ctx.Bool(RunDebugFlag.Name)
if debugProgram {
if metaPath := ctx.Path(RunMetaFlag.Name); metaPath == "" {
return fmt.Errorf("cannot enable debug mode without a metadata file")
}
if err := cannon.InitDebug(meta); err != nil {
return fmt.Errorf("failed to initialize debug mode: %w", err)
}
if err := vm.InitDebug(); err != nil {
return fmt.Errorf("failed to initialize debug mode: %w", err)
}
vm = cannon
} else {
return fmt.Errorf("unknown VM type %q", vmType)
}
proofFmt := ctx.String(RunProofFmtFlag.Name)
......@@ -421,7 +392,6 @@ func Run(ctx *cli.Context) error {
start := time.Now()
state := vm.GetState()
startStep := state.GetStep()
for !state.GetExited() {
......@@ -530,7 +500,6 @@ var RunCommand = &cli.Command{
Description: "Run VM step(s) and generate proof data to replicate onchain. See flags to match when to output a proof, a snapshot, or to stop early.",
Action: Run,
Flags: []cli.Flag{
VMTypeFlag,
RunInputFlag,
RunOutputFlag,
RunProofAtFlag,
......
package cmd
import (
"fmt"
"github.com/urfave/cli/v2"
)
type VMType string
var cannonVMType VMType = "cannon"
var mtVMType VMType = "cannon-mt"
var VMTypeFlag = &cli.StringFlag{
Name: "type",
Usage: "VM type to create state for. Options are 'cannon' (default), 'cannon-mt'",
Value: "cannon",
Required: false,
}
func vmTypeFromString(ctx *cli.Context) (VMType, error) {
if vmTypeStr := ctx.String(VMTypeFlag.Name); vmTypeStr == string(cannonVMType) {
return cannonVMType, nil
} else if vmTypeStr == string(mtVMType) {
return mtVMType, nil
} else {
return "", fmt.Errorf("unknown VM type %q", vmTypeStr)
}
}
......@@ -4,12 +4,8 @@ import (
"fmt"
"os"
"github.com/ethereum-optimism/optimism/cannon/mipsevm"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/multithreaded"
"github.com/ethereum-optimism/optimism/cannon/serialize"
factory "github.com/ethereum-optimism/optimism/cannon/mipsevm/versions"
"github.com/urfave/cli/v2"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/singlethreaded"
)
var (
......@@ -29,23 +25,10 @@ var (
func Witness(ctx *cli.Context) error {
input := ctx.Path(WitnessInputFlag.Name)
output := ctx.Path(WitnessOutputFlag.Name)
var state mipsevm.FPVMState
if vmType, err := vmTypeFromString(ctx); err != nil {
return err
} else if vmType == cannonVMType {
state, err = serialize.Load[singlethreaded.State](input)
if err != nil {
return fmt.Errorf("invalid input state (%v): %w", input, err)
}
} else if vmType == mtVMType {
state, err = serialize.Load[multithreaded.State](input)
if err != nil {
return fmt.Errorf("invalid input state (%v): %w", input, err)
}
} else {
return fmt.Errorf("invalid VM type: %q", vmType)
state, err := factory.LoadStateFromFile(input)
if err != nil {
return fmt.Errorf("invalid input state (%v): %w", input, err)
}
witness, h := state.EncodeWitness()
if output != "" {
if err := os.WriteFile(output, witness, 0755); err != nil {
......@@ -62,7 +45,6 @@ var WitnessCommand = &cli.Command{
Description: "Convert a Cannon JSON state into a binary witness. The hash of the witness is written to stdout",
Action: Witness,
Flags: []cli.Flag{
VMTypeFlag,
WitnessInputFlag,
WitnessOutputFlag,
},
......
......@@ -5,7 +5,6 @@ import (
"fmt"
"github.com/ethereum-optimism/optimism/cannon/mipsevm"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/program"
)
type StackTracker interface {
......@@ -31,10 +30,10 @@ type StackTrackerImpl struct {
stack []uint32
caller []uint32
meta *program.Metadata
meta mipsevm.Metadata
}
func NewStackTracker(state mipsevm.FPVMState, meta *program.Metadata) (*StackTrackerImpl, error) {
func NewStackTracker(state mipsevm.FPVMState, meta mipsevm.Metadata) (*StackTrackerImpl, error) {
if meta == nil {
return nil, errors.New("metadata is nil")
}
......@@ -42,7 +41,7 @@ func NewStackTracker(state mipsevm.FPVMState, meta *program.Metadata) (*StackTra
}
// NewStackTrackerUnsafe creates a new TraceableStackTracker without verifying meta is not nil
func NewStackTrackerUnsafe(state mipsevm.FPVMState, meta *program.Metadata) *StackTrackerImpl {
func NewStackTrackerUnsafe(state mipsevm.FPVMState, meta mipsevm.Metadata) *StackTrackerImpl {
return &StackTrackerImpl{state: state, meta: meta}
}
......
package mipsevm
import (
"io"
"github.com/ethereum-optimism/optimism/cannon/serialize"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/memory"
)
......@@ -52,6 +55,16 @@ type FPVMState interface {
// EncodeWitness returns the witness for the current state and the state hash
EncodeWitness() (witness []byte, hash common.Hash)
// CreateVM creates a FPVM that can operate on this state.
CreateVM(logger log.Logger, po PreimageOracle, stdOut, stdErr io.Writer, meta Metadata) FPVM
}
type SymbolMatcher func(addr uint32) bool
type Metadata interface {
LookupSymbol(addr uint32) string
CreateSymbolMatcher(name string) SymbolMatcher
}
type FPVM interface {
......@@ -73,6 +86,9 @@ type FPVM interface {
// GetDebugInfo returns debug information about the VM
GetDebugInfo() *DebugInfo
// InitDebug initializes the debug mode of the VM
InitDebug() error
// LookupSymbol returns the symbol located at the specified address.
// May return an empty string if there's no symbol table available.
LookupSymbol(addr uint32) string
......
......@@ -3,13 +3,11 @@ package multithreaded
import (
"io"
"github.com/ethereum-optimism/optimism/cannon/serialize"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum-optimism/optimism/cannon/mipsevm"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/exec"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/program"
)
type InstrumentedState struct {
......@@ -23,12 +21,12 @@ type InstrumentedState struct {
stackTracker ThreadedStackTracker
preimageOracle *exec.TrackingPreimageOracleReader
meta *program.Metadata
meta mipsevm.Metadata
}
var _ mipsevm.FPVM = (*InstrumentedState)(nil)
func NewInstrumentedState(state *State, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, log log.Logger) *InstrumentedState {
func NewInstrumentedState(state *State, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, log log.Logger, meta mipsevm.Metadata) *InstrumentedState {
return &InstrumentedState{
state: state,
log: log,
......@@ -37,24 +35,16 @@ func NewInstrumentedState(state *State, po mipsevm.PreimageOracle, stdOut, stdEr
memoryTracker: exec.NewMemoryTracker(state.Memory),
stackTracker: &NoopThreadedStackTracker{},
preimageOracle: exec.NewTrackingPreimageOracleReader(po),
meta: meta,
}
}
func NewInstrumentedStateFromFile(stateFile string, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, log log.Logger) (*InstrumentedState, error) {
state, err := serialize.Load[State](stateFile)
if err != nil {
return nil, err
}
return NewInstrumentedState(state, po, stdOut, stdErr, log), nil
}
func (m *InstrumentedState) InitDebug(meta *program.Metadata) error {
stackTracker, err := NewThreadedStackTracker(m.state, meta)
func (m *InstrumentedState) InitDebug() error {
stackTracker, err := NewThreadedStackTracker(m.state, m.meta)
if err != nil {
return err
}
m.stackTracker = stackTracker
m.meta = meta
return nil
}
......
......@@ -15,7 +15,7 @@ import (
)
func vmFactory(state *State, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, log log.Logger) mipsevm.FPVM {
return NewInstrumentedState(state, po, stdOut, stdErr, log)
return NewInstrumentedState(state, po, stdOut, stdErr, log, nil)
}
func TestInstrumentedState_OpenMips(t *testing.T) {
......@@ -36,7 +36,7 @@ func TestInstrumentedState_MultithreadedProgram(t *testing.T) {
oracle := testutil.StaticOracle(t, []byte{})
var stdOutBuf, stdErrBuf bytes.Buffer
us := NewInstrumentedState(state, oracle, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr), testutil.CreateLogger())
us := NewInstrumentedState(state, oracle, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr), testutil.CreateLogger(), nil)
for i := 0; i < 1_000_000; i++ {
if us.GetState().GetExited() {
break
......@@ -61,7 +61,7 @@ func TestInstrumentedState_Alloc(t *testing.T) {
oracle := testutil.AllocOracle(t, numAllocs)
// completes in ~870 M steps
us := NewInstrumentedState(state, oracle, os.Stdout, os.Stderr, testutil.CreateLogger())
us := NewInstrumentedState(state, oracle, os.Stdout, os.Stderr, testutil.CreateLogger(), nil)
for i := 0; i < 20_000_000_000; i++ {
if us.GetState().GetExited() {
break
......
......@@ -3,8 +3,8 @@ package multithreaded
import (
"errors"
"github.com/ethereum-optimism/optimism/cannon/mipsevm"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/exec"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/program"
)
type ThreadedStackTracker interface {
......@@ -21,14 +21,14 @@ var _ ThreadedStackTracker = (*ThreadedStackTrackerImpl)(nil)
func (n *NoopThreadedStackTracker) DropThread(threadId uint32) {}
type ThreadedStackTrackerImpl struct {
meta *program.Metadata
meta mipsevm.Metadata
state *State
trackersByThreadId map[uint32]exec.TraceableStackTracker
}
var _ ThreadedStackTracker = (*ThreadedStackTrackerImpl)(nil)
func NewThreadedStackTracker(state *State, meta *program.Metadata) (*ThreadedStackTrackerImpl, error) {
func NewThreadedStackTracker(state *State, meta mipsevm.Metadata) (*ThreadedStackTrackerImpl, error) {
if meta == nil {
return nil, errors.New("metadata is nil")
}
......
......@@ -5,11 +5,11 @@ import (
"fmt"
"io"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/versions"
"github.com/ethereum-optimism/optimism/cannon/serialize"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum-optimism/optimism/cannon/mipsevm"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/exec"
......@@ -35,27 +35,27 @@ const (
)
type State struct {
Memory *memory.Memory `json:"memory"`
Memory *memory.Memory
PreimageKey common.Hash `json:"preimageKey"`
PreimageOffset uint32 `json:"preimageOffset"` // note that the offset includes the 8-byte length prefix
PreimageKey common.Hash
PreimageOffset uint32 // note that the offset includes the 8-byte length prefix
Heap uint32 `json:"heap"` // to handle mmap growth
Heap uint32 // to handle mmap growth
ExitCode uint8 `json:"exit"`
Exited bool `json:"exited"`
ExitCode uint8
Exited bool
Step uint64 `json:"step"`
StepsSinceLastContextSwitch uint64 `json:"stepsSinceLastContextSwitch"`
Wakeup uint32 `json:"wakeup"`
Step uint64
StepsSinceLastContextSwitch uint64
Wakeup uint32
TraverseRight bool `json:"traverseRight"`
LeftThreadStack []*ThreadState `json:"leftThreadStack"`
RightThreadStack []*ThreadState `json:"rightThreadStack"`
NextThreadId uint32 `json:"nextThreadId"`
TraverseRight bool
LeftThreadStack []*ThreadState
RightThreadStack []*ThreadState
NextThreadId uint32
// LastHint is optional metadata, and not part of the VM state itself.
LastHint hexutil.Bytes `json:"lastHint,omitempty"`
LastHint hexutil.Bytes
}
var _ mipsevm.FPVMState = (*State)(nil)
......@@ -87,6 +87,11 @@ func CreateInitialState(pc, heapStart uint32) *State {
return state
}
func (s *State) CreateVM(logger log.Logger, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, meta mipsevm.Metadata) mipsevm.FPVM {
logger.Info("Using cannon multithreaded VM")
return NewInstrumentedState(s, po, stdOut, stdErr, logger, meta)
}
func (s *State) GetCurrentThread() *ThreadState {
activeStack := s.getActiveThreadStack()
......@@ -246,9 +251,6 @@ func (s *State) ThreadCount() int {
// LastHint []byte
func (s *State) Serialize(out io.Writer) error {
bout := serialize.NewBinaryWriter(out)
if err := bout.WriteUInt(versions.VersionMultiThreaded); err != nil {
return err
}
if err := s.Memory.Serialize(out); err != nil {
return err
......@@ -309,13 +311,6 @@ func (s *State) Serialize(out io.Writer) error {
func (s *State) Deserialize(in io.Reader) error {
bin := serialize.NewBinaryReader(in)
var version versions.StateVersion
if err := bin.ReadUInt(&version); err != nil {
return err
}
if version != versions.VersionMultiThreaded {
return fmt.Errorf("invalid state encoding version %d", version)
}
s.Memory = memory.NewMemory()
if err := s.Memory.Deserialize(in); err != nil {
return err
......
......@@ -4,6 +4,8 @@ import (
"debug/elf"
"fmt"
"sort"
"github.com/ethereum-optimism/optimism/cannon/mipsevm"
)
type Symbol struct {
......@@ -16,6 +18,8 @@ type Metadata struct {
Symbols []Symbol `json:"symbols"`
}
var _ mipsevm.Metadata = (*Metadata)(nil)
func MakeMetadata(elfProgram *elf.File) (*Metadata, error) {
syms, err := elfProgram.Symbols()
if err != nil {
......@@ -50,9 +54,7 @@ func (m *Metadata) LookupSymbol(addr uint32) string {
return out.Name
}
type SymbolMatcher func(addr uint32) bool
func (m *Metadata) CreateSymbolMatcher(name string) SymbolMatcher {
func (m *Metadata) CreateSymbolMatcher(name string) mipsevm.SymbolMatcher {
for _, s := range m.Symbols {
if s.Name == name {
start := s.Start
......
......@@ -5,14 +5,12 @@ import (
"github.com/ethereum-optimism/optimism/cannon/mipsevm"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/exec"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/program"
"github.com/ethereum-optimism/optimism/cannon/serialize"
"github.com/ethereum/go-ethereum/common/hexutil"
)
type InstrumentedState struct {
meta *program.Metadata
sleepCheck program.SymbolMatcher
meta mipsevm.Metadata
sleepCheck mipsevm.SymbolMatcher
state *State
......@@ -27,16 +25,14 @@ type InstrumentedState struct {
var _ mipsevm.FPVM = (*InstrumentedState)(nil)
func NewInstrumentedState(state *State, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, meta *program.Metadata) *InstrumentedState {
var sleepCheck program.SymbolMatcher
func NewInstrumentedState(state *State, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, meta mipsevm.Metadata) *InstrumentedState {
var sleepCheck mipsevm.SymbolMatcher
if meta == nil {
sleepCheck = func(addr uint32) bool { return false }
} else {
sleepCheck = meta.CreateSymbolMatcher("runtime.notesleep")
}
return &InstrumentedState{
meta: meta,
sleepCheck: sleepCheck,
state: state,
stdOut: stdOut,
......@@ -44,17 +40,10 @@ func NewInstrumentedState(state *State, po mipsevm.PreimageOracle, stdOut, stdEr
memoryTracker: exec.NewMemoryTracker(state.Memory),
stackTracker: &exec.NoopStackTracker{},
preimageOracle: exec.NewTrackingPreimageOracleReader(po),
meta: meta,
}
}
func NewInstrumentedStateFromFile(stateFile string, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, meta *program.Metadata) (*InstrumentedState, error) {
state, err := serialize.Load[State](stateFile)
if err != nil {
return nil, err
}
return NewInstrumentedState(state, po, stdOut, stdErr, meta), nil
}
func (m *InstrumentedState) InitDebug() error {
stackTracker, err := exec.NewStackTracker(m.state, m.meta)
if err != nil {
......
......@@ -6,11 +6,11 @@ import (
"fmt"
"io"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/versions"
"github.com/ethereum-optimism/optimism/cannon/serialize"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum-optimism/optimism/cannon/mipsevm"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/memory"
......@@ -68,6 +68,11 @@ func CreateInitialState(pc, heapStart uint32) *State {
return state
}
func (s *State) CreateVM(logger log.Logger, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, meta mipsevm.Metadata) mipsevm.FPVM {
logger.Info("Using cannon VM")
return NewInstrumentedState(s, po, stdOut, stdErr, meta)
}
type stateMarshaling struct {
Memory *memory.Memory `json:"memory"`
PreimageKey common.Hash `json:"preimageKey"`
......@@ -201,9 +206,6 @@ func (s *State) EncodeWitness() ([]byte, common.Hash) {
// LastHint []byte
func (s *State) Serialize(out io.Writer) error {
bout := serialize.NewBinaryWriter(out)
if err := bout.WriteUInt(versions.VersionSingleThreaded); err != nil {
return err
}
if err := s.Memory.Serialize(out); err != nil {
return err
......@@ -251,13 +253,6 @@ func (s *State) Serialize(out io.Writer) error {
func (s *State) Deserialize(in io.Reader) error {
bin := serialize.NewBinaryReader(in)
var version versions.StateVersion
if err := bin.ReadUInt(&version); err != nil {
return err
}
if version != versions.VersionSingleThreaded {
return fmt.Errorf("invalid state encoding version %d", version)
}
s.Memory = memory.NewMemory()
if err := s.Memory.Deserialize(in); err != nil {
return err
......
......@@ -48,7 +48,7 @@ func TestEVM_SysClone_FlagHandling(t *testing.T) {
var err error
var stepWitness *mipsevm.StepWitness
us := multithreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr, nil)
us := multithreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr, nil, nil)
if !c.valid {
// The VM should exit
stepWitness, err = us.Step(true)
......
......@@ -31,7 +31,7 @@ func multiThreadedVmFactory(po mipsevm.PreimageOracle, stdOut, stdErr io.Writer,
for _, opt := range opts {
opt(mutator)
}
return multithreaded.NewInstrumentedState(state, po, stdOut, stdErr, log)
return multithreaded.NewInstrumentedState(state, po, stdOut, stdErr, log, nil)
}
type ElfVMFactory func(t require.TestingT, elfFile string, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, log log.Logger) mipsevm.FPVM
......@@ -45,8 +45,8 @@ func singleThreadElfVmFactory(t require.TestingT, elfFile string, po mipsevm.Pre
func multiThreadElfVmFactory(t require.TestingT, elfFile string, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, log log.Logger) mipsevm.FPVM {
state, meta := testutil.LoadELFProgram(t, elfFile, multithreaded.CreateInitialState, false)
fpvm := multithreaded.NewInstrumentedState(state, po, stdOut, stdErr, log)
require.NoError(t, fpvm.InitDebug(meta))
fpvm := multithreaded.NewInstrumentedState(state, po, stdOut, stdErr, log, meta)
require.NoError(t, fpvm.InitDebug())
return fpvm
}
......
package versions
import (
"encoding/json"
"errors"
"fmt"
"io"
"github.com/ethereum-optimism/optimism/cannon/mipsevm"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/multithreaded"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/singlethreaded"
"github.com/ethereum-optimism/optimism/cannon/serialize"
"github.com/ethereum-optimism/optimism/op-service/jsonutil"
)
type StateVersion uint8
const (
VersionSingleThreaded StateVersion = iota
VersionMultiThreaded
)
var (
ErrUnknownVersion = errors.New("unknown version")
ErrJsonNotSupported = errors.New("json not supported")
)
func LoadStateFromFile(path string) (*VersionedState, error) {
if !serialize.IsBinaryFile(path) {
// Always use singlethreaded for JSON states
state, err := jsonutil.LoadJSON[singlethreaded.State](path)
if err != nil {
return nil, err
}
return NewFromState(state)
}
return serialize.LoadSerializedBinary[VersionedState](path)
}
func NewFromState(state mipsevm.FPVMState) (*VersionedState, error) {
switch state := state.(type) {
case *singlethreaded.State:
return &VersionedState{
Version: VersionSingleThreaded,
FPVMState: state,
}, nil
case *multithreaded.State:
return &VersionedState{
Version: VersionMultiThreaded,
FPVMState: state,
}, nil
default:
return nil, fmt.Errorf("%w: %T", ErrUnknownVersion, state)
}
}
// VersionedState deserializes a FPVMState and implements VersionedState based on the version of that state.
// It does this based on the version byte read in Deserialize
type VersionedState struct {
Version StateVersion
mipsevm.FPVMState
}
func (s *VersionedState) Serialize(w io.Writer) error {
bout := serialize.NewBinaryWriter(w)
if err := bout.WriteUInt(s.Version); err != nil {
return err
}
return s.FPVMState.Serialize(w)
}
func (s *VersionedState) Deserialize(in io.Reader) error {
bin := serialize.NewBinaryReader(in)
if err := bin.ReadUInt(&s.Version); err != nil {
return err
}
switch s.Version {
case VersionSingleThreaded:
state := &singlethreaded.State{}
if err := state.Deserialize(in); err != nil {
return err
}
s.FPVMState = state
return nil
case VersionMultiThreaded:
state := &multithreaded.State{}
if err := state.Deserialize(in); err != nil {
return err
}
s.FPVMState = state
return nil
default:
return fmt.Errorf("%w: %d", ErrUnknownVersion, s.Version)
}
}
// MarshalJSON marshals the underlying state without adding version prefix.
// JSON states are always assumed to be single threaded
func (s *VersionedState) MarshalJSON() ([]byte, error) {
if s.Version != VersionSingleThreaded {
return nil, fmt.Errorf("%w for type %T", ErrJsonNotSupported, s.FPVMState)
}
return json.Marshal(s.FPVMState)
}
package versions
import (
"path/filepath"
"testing"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/multithreaded"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/singlethreaded"
"github.com/ethereum-optimism/optimism/cannon/serialize"
"github.com/stretchr/testify/require"
)
func TestNewFromState(t *testing.T) {
t.Run("singlethreaded", func(t *testing.T) {
actual, err := NewFromState(singlethreaded.CreateEmptyState())
require.NoError(t, err)
require.IsType(t, &singlethreaded.State{}, actual.FPVMState)
require.Equal(t, VersionSingleThreaded, actual.Version)
})
t.Run("multithreaded", func(t *testing.T) {
actual, err := NewFromState(multithreaded.CreateEmptyState())
require.NoError(t, err)
require.IsType(t, &multithreaded.State{}, actual.FPVMState)
require.Equal(t, VersionMultiThreaded, actual.Version)
})
}
func TestLoadStateFromFile(t *testing.T) {
t.Run("SinglethreadedFromJSON", func(t *testing.T) {
expected, err := NewFromState(singlethreaded.CreateEmptyState())
require.NoError(t, err)
path := writeToFile(t, "state.json", expected)
actual, err := LoadStateFromFile(path)
require.NoError(t, err)
require.Equal(t, expected, actual)
})
t.Run("SinglethreadedFromBinary", func(t *testing.T) {
expected, err := NewFromState(singlethreaded.CreateEmptyState())
require.NoError(t, err)
path := writeToFile(t, "state.bin.gz", expected)
actual, err := LoadStateFromFile(path)
require.NoError(t, err)
require.Equal(t, expected, actual)
})
t.Run("MultithreadedFromBinary", func(t *testing.T) {
expected, err := NewFromState(multithreaded.CreateEmptyState())
require.NoError(t, err)
path := writeToFile(t, "state.bin.gz", expected)
actual, err := LoadStateFromFile(path)
require.NoError(t, err)
require.Equal(t, expected, actual)
})
}
func TestMultithreadedDoesNotSupportJSON(t *testing.T) {
state, err := NewFromState(multithreaded.CreateEmptyState())
require.NoError(t, err)
dir := t.TempDir()
path := filepath.Join(dir, "test.json")
err = serialize.Write(path, state, 0o644)
require.ErrorIs(t, err, ErrJsonNotSupported)
}
func writeToFile(t *testing.T, filename string, data serialize.Serializable) string {
dir := t.TempDir()
path := filepath.Join(dir, filename)
require.NoError(t, serialize.Write(path, data, 0o644))
return path
}
......@@ -9,13 +9,16 @@ import (
"github.com/ethereum-optimism/optimism/op-service/ioutil"
)
// Deserializable defines functionality for a type that may be deserialized from raw bytes.
type Deserializable interface {
// Deserialize decodes raw bytes into the type.
Deserialize(in io.Reader) error
}
// Serializable defines functionality for a type that may be serialized to raw bytes.
type Serializable interface {
// Serialize encodes the type as raw bytes.
Serialize(out io.Writer) error
// Deserialize decodes raw bytes into the type.
Deserialize(in io.Reader) error
}
func LoadSerializedBinary[X any](inputPath string) (*X, error) {
......@@ -30,7 +33,7 @@ func LoadSerializedBinary[X any](inputPath string) (*X, error) {
defer f.Close()
var x X
serializable, ok := reflect.ValueOf(&x).Interface().(Serializable)
serializable, ok := reflect.ValueOf(&x).Interface().(Deserializable)
if !ok {
return nil, fmt.Errorf("%T is not a Serializable", x)
}
......
......@@ -8,20 +8,13 @@ import (
"github.com/ethereum-optimism/optimism/op-service/jsonutil"
)
func Load[X any](inputPath string) (*X, error) {
if isBinary(inputPath) {
return LoadSerializedBinary[X](inputPath)
}
return jsonutil.LoadJSON[X](inputPath)
}
func Write[X Serializable](outputPath string, x X, perm os.FileMode) error {
if isBinary(outputPath) {
if IsBinaryFile(outputPath) {
return WriteSerializedBinary(x, ioutil.ToStdOutOrFileOrNoop(outputPath, perm))
}
return jsonutil.WriteJSON[X](x, ioutil.ToStdOutOrFileOrNoop(outputPath, perm))
}
func isBinary(path string) bool {
func IsBinaryFile(path string) bool {
return strings.HasSuffix(path, ".bin") || strings.HasSuffix(path, ".bin.gz")
}
......@@ -6,6 +6,7 @@ import (
"testing"
"github.com/ethereum-optimism/optimism/op-service/ioutil"
"github.com/ethereum-optimism/optimism/op-service/jsonutil"
"github.com/stretchr/testify/require"
)
......@@ -42,13 +43,16 @@ func TestRoundtrip(t *testing.T) {
start := make([]byte, 1)
_, err = io.ReadFull(decompressed, start)
require.NoError(t, err)
var load func(path string) (*serializableTestData, error)
if test.expectJSON {
load = jsonutil.LoadJSON[serializableTestData]
require.Equal(t, "{", string(start))
} else {
load = LoadSerializedBinary[serializableTestData]
require.NotEqual(t, "{", string(start))
}
result, err := Load[serializableTestData](path)
result, err := load(path)
require.NoError(t, err)
require.EqualValues(t, data, result)
})
......
......@@ -3,8 +3,8 @@ package cannon
import (
"fmt"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/singlethreaded"
"github.com/ethereum-optimism/optimism/cannon/serialize"
"github.com/ethereum-optimism/optimism/cannon/mipsevm"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/versions"
"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/utils"
)
......@@ -30,9 +30,9 @@ func (c *StateConverter) ConvertStateToProof(statePath string) (*utils.ProofData
OracleKey: nil,
OracleValue: nil,
OracleOffset: 0,
}, state.Step, state.Exited, nil
}, state.GetStep(), state.GetExited(), nil
}
func parseState(path string) (*singlethreaded.State, error) {
return serialize.Load[singlethreaded.State](path)
func parseState(path string) (mipsevm.FPVMState, error) {
return versions.LoadStateFromFile(path)
}
package cannon
import (
"compress/gzip"
_ "embed"
"encoding/json"
"os"
"path/filepath"
"testing"
"github.com/ethereum-optimism/optimism/cannon/mipsevm"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/multithreaded"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/versions"
"github.com/ethereum-optimism/optimism/cannon/serialize"
"github.com/stretchr/testify/require"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/singlethreaded"
)
//go:embed test_data/state.json
var testState []byte
func TestLoadState(t *testing.T) {
t.Run("Uncompressed", func(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "state.json")
require.NoError(t, os.WriteFile(path, testState, 0644))
state, err := parseState(path)
require.NoError(t, err)
var expected singlethreaded.State
require.NoError(t, json.Unmarshal(testState, &expected))
require.Equal(t, &expected, state)
})
t.Run("Gzipped", func(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "state.json.gz")
f, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
require.NoError(t, err)
defer f.Close()
writer := gzip.NewWriter(f)
_, err = writer.Write(testState)
require.NoError(t, err)
require.NoError(t, writer.Close())
state, err := parseState(path)
require.NoError(t, err)
var expected singlethreaded.State
require.NoError(t, json.Unmarshal(testState, &expected))
require.Equal(t, &expected, state)
})
t.Run("Binary", func(t *testing.T) {
var expected singlethreaded.State
require.NoError(t, json.Unmarshal(testState, &expected))
dir := t.TempDir()
path := filepath.Join(dir, "state.bin")
require.NoError(t, serialize.Write[*singlethreaded.State](path, &expected, 0644))
state, err := parseState(path)
require.NoError(t, err)
require.Equal(t, &expected, state)
})
t.Run("BinaryGzip", func(t *testing.T) {
var expected singlethreaded.State
require.NoError(t, json.Unmarshal(testState, &expected))
dir := t.TempDir()
path := filepath.Join(dir, "state.bin.gz")
require.NoError(t, serialize.Write[*singlethreaded.State](path, &expected, 0644))
state, err := parseState(path)
require.NoError(t, err)
require.Equal(t, &expected, state)
})
tests := []struct {
name string
creator func() mipsevm.FPVMState
supportsJSON bool
}{
{
name: "singlethreaded",
creator: func() mipsevm.FPVMState { return singlethreaded.CreateInitialState(234, 82) },
supportsJSON: true,
},
{
name: "multithreaded",
creator: func() mipsevm.FPVMState { return multithreaded.CreateInitialState(982, 492) },
supportsJSON: false,
},
}
for _, test := range tests {
test := test
loadExpectedState := func(t *testing.T) *versions.VersionedState {
state, err := versions.NewFromState(test.creator())
require.NoError(t, err)
return state
}
t.Run(test.name, func(t *testing.T) {
t.Run("Uncompressed", func(t *testing.T) {
if !test.supportsJSON {
t.Skip("JSON not supported by state version")
}
expected := loadExpectedState(t)
path := writeState(t, "state.json", expected)
state, err := parseState(path)
require.NoError(t, err)
require.Equal(t, expected, state)
})
t.Run("Gzipped", func(t *testing.T) {
if !test.supportsJSON {
t.Skip("JSON not supported by state version")
}
expected := loadExpectedState(t)
path := writeState(t, "state.json.gz", expected)
state, err := parseState(path)
require.NoError(t, err)
require.Equal(t, expected, state)
})
t.Run("Binary", func(t *testing.T) {
expected := loadExpectedState(t)
path := writeState(t, "state.bin", expected)
state, err := parseState(path)
require.NoError(t, err)
require.Equal(t, expected, state)
})
t.Run("BinaryGzip", func(t *testing.T) {
expected := loadExpectedState(t)
path := writeState(t, "state.bin.gz", expected)
state, err := parseState(path)
require.NoError(t, err)
require.Equal(t, expected, state)
})
})
}
}
func writeState(t *testing.T, filename string, state *versions.VersionedState) string {
dir := t.TempDir()
path := filepath.Join(dir, filename)
require.NoError(t, serialize.Write(path, state, 0644))
return path
}
{
"memory": [{"index":10,"data":"eJzswAENAAAAwiD7p7bHBwMAAADyHgAA//8QAAAB"}],
"preimageKey": "0xcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
"preimageOffset": 42,
"pc": 94,
"nextPC": 1,
"lo": 3,
"hi": 5,
"heap": 4,
"exit": 1,
"exited": true,
"step": 8849,
"registers": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2147471360,0,0]
}
......@@ -7,7 +7,7 @@ import (
"path/filepath"
"testing"
"github.com/ethereum-optimism/optimism/cannon/serialize"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/versions"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
......@@ -15,7 +15,6 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/stretchr/testify/require"
"github.com/ethereum-optimism/optimism/cannon/mipsevm/singlethreaded"
"github.com/ethereum-optimism/optimism/op-challenger/config"
"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/utils"
"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/vm"
......@@ -261,9 +260,9 @@ func runCannon(t *testing.T, ctx context.Context, sys *op_e2e.System, inputs uti
err := executor.DoGenerateProof(ctx, proofsDir, math.MaxUint, math.MaxUint, extraVmArgs...)
require.NoError(t, err, "failed to generate proof")
state, err := serialize.Load[singlethreaded.State](vm.FinalStatePath(proofsDir, cfg.Cannon.BinarySnapshots))
state, err := versions.LoadStateFromFile(vm.FinalStatePath(proofsDir, cfg.Cannon.BinarySnapshots))
require.NoError(t, err, "failed to parse state")
require.True(t, state.Exited, "cannon did not exit")
require.Zero(t, state.ExitCode, "cannon failed with exit code %d", state.ExitCode)
t.Logf("Completed in %d steps", state.Step)
require.True(t, state.GetExited(), "cannon did not exit")
require.Zero(t, state.GetExitCode(), "cannon failed with exit code %d", state.GetExitCode())
t.Logf("Completed in %d steps", state.GetStep())
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment