mt-cannon: Implement mips logic (#11188)

* cannon: Copy over singlethreaded impls as a starting point

* cannon: Update mips property access to work with MTState

* cannon: Add new syscall constants

* mt-cannon: Implement clone syscall

* mt-cannon: Implement remaining new syscalls

* mt-cannon: Implement thread traversal changes to mipsStep()

* mt-cannon: Add logger, log when max steps reached

* mt-cannon: Implement onWaitComplete()

* mt-cannon: Implement thread manipulation methods

Also, use slices of pointers for the thread stacks

* mt-cannon: Move thread traversal fns to mips.go

* mt-cannon: Fix issue where wakeup traversal never stops

* mt-cannon: Fix issue where we can end up popping an empty stack

* mt-cannon: Move thread definitions to new thread.go file

* cannon: Add compile-time type checks for FPVM(State) impls

* mt-cannon: Add new threaded StackTracker

* mt-cannon: Update proof generation to include thread proof

* mt-cannon: Move FPVM compile-time type check

* cannon: Run common vm tests across all FPVM impls

* cannon: Cut OpenMIPS clone test

* cannon: Cleanup - fix some discrepancies, clarify constant

* cannon: Disable mem profiling in op-program instead of patch.go

* cannon: Consolidate calls to program.PatchGo

* cannon: Disable program.PatchGo in MTCannon tests

* mt-cannon: Add multithreaded program test

* cannon: Only run sleep check for single-threaded cannon

* op-program: Update profiling before dependency init fns are called

* mt-cannon: Track stack on thread clone, handled popped threads

* mt-cannon: Panic if unrecognized syscall is executed

* mt-cannon: Panic if unexpected flags are passed to SysClone

* mt-cannon: Add some tests for EncodeThreadProof()

* mt-cannon: Add some more tests around threadProof edge cases

* mt-cannon: Minimize logging

* cannon: Update go version in cannon/example/multithreaded/go.mod
Co-authored-by: Inphi <mlaw2501@gmail.com>

* mt-cannon: Rework clone behavior based on feedback

* mt-cannon: Rework wakeup logic

* mt-cannon: Cleanup - simplify clone, refine logging

* Revert "cannon: Cut OpenMIPS clone test"

This reverts commit d876d6a44ffc01672a019d5b2411e7d3eab08439.

* mt-cannon: Skip open-mips clone test add todos

* mt-cannon: Handle munmap syscall

* mt-cannon: Exit if the last thread exits

* cannon: Clarify skip comment

* cannon: Add some todos

* cannon: Add guard around logging

---------
Co-authored-by: Inphi <mlaw2501@gmail.com>

cannon/cmd/run.go

@@ -366,7 +366,7 @@ func Run(ctx *cli.Context) error {
 	var vm mipsevm.FPVM
 	var debugProgram bool
 	if vmType == cannonVMType {
-		cannon, err := singlethreaded.NewInstrumentedStateFromFile(ctx.Path(RunInputFlag.Name), po, outLog, errLog)
+		cannon, err := singlethreaded.NewInstrumentedStateFromFile(ctx.Path(RunInputFlag.Name), po, outLog, errLog, meta)
 		if err != nil {
 			return err
 		}
@@ -375,7 +375,7 @@ func Run(ctx *cli.Context) error {
 			if metaPath := ctx.Path(RunMetaFlag.Name); metaPath == "" {
 				return fmt.Errorf("cannot enable debug mode without a metadata file")
 			}
-			if err := cannon.InitDebug(meta); err != nil {
+			if err := cannon.InitDebug(); err != nil {
 				return fmt.Errorf("failed to initialize debug mode: %w", err)
 			}
 		}
@@ -397,9 +397,6 @@ func Run(ctx *cli.Context) error {
 	state := vm.GetState()
 	startStep := state.GetStep()
 
-	// avoid symbol lookups every instruction by preparing a matcher func
-	sleepCheck := meta.SymbolMatcher("runtime.notesleep")
-
 	for !state.GetExited() {
 		step := state.GetStep()
 		if step%100 == 0 { // don't do the ctx err check (includes lock) too often
@@ -421,8 +418,9 @@ func Run(ctx *cli.Context) error {
 			)
 		}
 
-		if sleepCheck(state.GetPC()) { // don't loop forever when we get stuck because of an unexpected bad program
-			return fmt.Errorf("got stuck in Go sleep at step %d", step)
+		if vm.CheckInfiniteLoop() {
+			// don't loop forever when we get stuck because of an unexpected bad program
+			return fmt.Errorf("detected an infinite loop at step %d", step)
 		}
 
 		if stopAt(state) {

cannon/example/multithreaded/go.mod

+module multithreaded
+
+go 1.21

cannon/example/multithreaded/main.go

+package main
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"sync"
+	"sync/atomic"
+)
+
+func main() {
+	// try some concurrency!
+	var wg sync.WaitGroup
+	wg.Add(2)
+	var x atomic.Int32
+	go func() {
+		x.Add(2)
+		wg.Done()
+	}()
+	go func() {
+		x.Add(40)
+		wg.Done()
+	}()
+	wg.Wait()
+	fmt.Printf("waitgroup result: %d\n", x.Load())
+
+	// channels
+	a := make(chan int, 1)
+	b := make(chan int)
+	c := make(chan int)
+	go func() {
+		t0 := <-a
+		b <- t0
+	}()
+	go func() {
+		t1 := <-b
+		c <- t1
+	}()
+	a <- 1234
+	out := <-c
+	fmt.Printf("channels result: %d\n", out)
+
+	// try a GC! (the runtime might not have run one yet)
+	runtime.GC()
+	_, _ = os.Stdout.Write([]byte("GC complete!\n"))
+}

cannon/mipsevm/exec/mips_instructions.go

@@ -22,7 +22,7 @@ func ExecMipsCoreStepLogic(cpu *mipsevm.CpuScalars, registers *[32]uint32, memor
 		}
 		// Take top 4 bits of the next PC (its 256 MB region), and concatenate with the 26-bit offset
 		target := (cpu.NextPC & 0xF0000000) | ((insn & 0x03FFFFFF) << 2)
-		stackTracker.PushStack(target)
+		stackTracker.PushStack(cpu.PC, target)
 		return HandleJump(cpu, registers, linkReg, target)
 	}
 

cannon/mipsevm/exec/mips_syscalls.go

@@ -11,16 +11,65 @@ import (
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/memory"
 )
 
+// Syscall codes
 const (
-	SysMmap      = 4090
-	SysBrk       = 4045
-	SysClone     = 4120
-	SysExitGroup = 4246
-	SysRead      = 4003
-	SysWrite     = 4004
-	SysFcntl     = 4055
+	SysMmap       = 4090
+	SysMunmap     = 4091
+	SysBrk        = 4045
+	SysClone      = 4120
+	SysExitGroup  = 4246
+	SysRead       = 4003
+	SysWrite      = 4004
+	SysFcntl      = 4055
+	SysExit       = 4001
+	SysSchedYield = 4162
+	SysGetTID     = 4222
+	SysFutex      = 4238
+	SysOpen       = 4005
+	SysNanosleep  = 4166
 )
 
+// Noop Syscall codes
+const (
+	SysGetAffinity   = 4240
+	SysMadvise       = 4218
+	SysRtSigprocmask = 4195
+	SysSigaltstack   = 4206
+	SysRtSigaction   = 4194
+	SysPrlimit64     = 4338
+	SysClose         = 4006
+	SysPread64       = 4200
+	SysFstat64       = 4215
+	SysOpenAt        = 4288
+	SysReadlink      = 4085
+	SysReadlinkAt    = 4298
+	SysIoctl         = 4054
+	SysEpollCreate1  = 4326
+	SysPipe2         = 4328
+	SysEpollCtl      = 4249
+	SysEpollPwait    = 4313
+	SysGetRandom     = 4353
+	SysUname         = 4122
+	SysStat64        = 4213
+	SysGetuid        = 4024
+	SysGetgid        = 4047
+	SysLlseek        = 4140
+	SysMinCore       = 4217
+	SysTgkill        = 4266
+)
+
+// Profiling-related syscalls
+// Should be able to ignore if we patch out prometheus calls and disable memprofiling
+// TODO(cp-903) - Update patching for mt-cannon so that these can be ignored
+const (
+	SysSetITimer    = 4104
+	SysTimerCreate  = 4257
+	SysTimerSetTime = 4258
+	SysTimerDelete  = 4261
+	SysClockGetTime = 4263
+)
+
+// File descriptors
 const (
 	FdStdin         = 0
 	FdStdout        = 1
@@ -31,19 +80,70 @@ const (
 	FdPreimageWrite = 6
 )
 
+// Errors
+const (
+	SysErrorSignal = ^uint32(0)
+	MipsEBADF      = 0x9
+	MipsEINVAL     = 0x16
+	MipsEAGAIN     = 0xb
+	MipsETIMEDOUT  = 0x91
+)
+
+// SysFutex-related constants
+const (
+	FutexWaitPrivate  = 128
+	FutexWakePrivate  = 129
+	FutexTimeoutSteps = 10_000
+	FutexNoTimeout    = ^uint64(0)
+	FutexEmptyAddr    = ^uint32(0)
+)
+
+// SysClone flags
+// Handling is meant to support go runtime use cases
+// Pulled from: https://github.com/golang/go/blob/d8392e69973a64d96534d544d1f8ac2defc1bc64/src/runtime/os_linux.go#L124-L158
+const (
+	CloneVm            = 0x100
+	CloneFs            = 0x200
+	CloneFiles         = 0x400
+	CloneSighand       = 0x800
+	ClonePtrace        = 0x2000
+	CloneVfork         = 0x4000
+	CloneParent        = 0x8000
+	CloneThread        = 0x10000
+	CloneNewns         = 0x20000
+	CloneSysvsem       = 0x40000
+	CloneSettls        = 0x80000
+	CloneParentSettid  = 0x100000
+	CloneChildCleartid = 0x200000
+	CloneUntraced      = 0x800000
+	CloneChildSettid   = 0x1000000
+	CloneStopped       = 0x2000000
+	CloneNewuts        = 0x4000000
+	CloneNewipc        = 0x8000000
+
+	ValidCloneFlags = CloneVm |
+		CloneFs |
+		CloneFiles |
+		CloneSighand |
+		CloneSysvsem |
+		CloneThread
+)
+
+// Other constants
 const (
-	MipsEBADF  = 0x9
-	MipsEINVAL = 0x16
+	SchedQuantum = 100_000
+	BrkStart     = 0x40000000
 )
 
-func GetSyscallArgs(registers *[32]uint32) (syscallNum, a0, a1, a2 uint32) {
+func GetSyscallArgs(registers *[32]uint32) (syscallNum, a0, a1, a2, a3 uint32) {
 	syscallNum = registers[2] // v0
 
 	a0 = registers[4]
 	a1 = registers[5]
 	a2 = registers[6]
+	a3 = registers[7]
 
-	return syscallNum, a0, a1, a2
+	return syscallNum, a0, a1, a2, a3
 }
 
 func HandleSysMmap(a0, a1, heap uint32) (v0, v1, newHeap uint32) {

cannon/mipsevm/exec/stack.go

@@ -9,7 +9,7 @@ import (
 )
 
 type StackTracker interface {
-	PushStack(target uint32)
+	PushStack(caller uint32, target uint32)
 	PopStack()
 }
 
@@ -20,7 +20,7 @@ type TraceableStackTracker interface {
 
 type NoopStackTracker struct{}
 
-func (n *NoopStackTracker) PushStack(target uint32) {}
+func (n *NoopStackTracker) PushStack(caller uint32, target uint32) {}
 
 func (n *NoopStackTracker) PopStack() {}
 
@@ -38,12 +38,17 @@ func NewStackTracker(state mipsevm.FPVMState, meta *program.Metadata) (*StackTra
 	if meta == nil {
 		return nil, errors.New("metadata is nil")
 	}
-	return &StackTrackerImpl{state: state}, nil
+	return NewStackTrackerUnsafe(state, meta), nil
 }
 
-func (s *StackTrackerImpl) PushStack(target uint32) {
+// NewStackTrackerUnsafe creates a new TraceableStackTracker without verifying meta is not nil
+func NewStackTrackerUnsafe(state mipsevm.FPVMState, meta *program.Metadata) *StackTrackerImpl {
+	return &StackTrackerImpl{state: state, meta: meta}
+}
+
+func (s *StackTrackerImpl) PushStack(caller uint32, target uint32) {
+	s.caller = append(s.caller, caller)
 	s.stack = append(s.stack, target)
-	s.caller = append(s.caller, s.state.GetPC())
 }
 
 func (s *StackTrackerImpl) PopStack() {

cannon/mipsevm/iface.go

@@ -35,6 +35,9 @@ type FPVM interface {
 	// Step executes a single instruction and returns the witness for the step
 	Step(includeProof bool) (*StepWitness, error)
 
+	// CheckInfiniteLoop returns true if the vm is stuck in an infinite loop
+	CheckInfiniteLoop() bool
+
 	// LastPreimage returns the last preimage accessed by the VM
 	LastPreimage() (preimageKey [32]byte, preimage []byte, preimageOffset uint32)
 

cannon/mipsevm/multithreaded/instrumented.go

+package multithreaded
+
+import (
+	"io"
+
+	"github.com/ethereum/go-ethereum/log"
+
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/exec"
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/program"
+	"github.com/ethereum-optimism/optimism/op-service/jsonutil"
+)
+
+type InstrumentedState struct {
+	state *State
+
+	log    log.Logger
+	stdOut io.Writer
+	stdErr io.Writer
+
+	memoryTracker *exec.MemoryTrackerImpl
+	stackTracker  ThreadedStackTracker
+
+	preimageOracle *exec.TrackingPreimageOracleReader
+}
+
+var _ mipsevm.FPVM = (*InstrumentedState)(nil)
+
+func NewInstrumentedState(state *State, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, log log.Logger) *InstrumentedState {
+	return &InstrumentedState{
+		state:          state,
+		log:            log,
+		stdOut:         stdOut,
+		stdErr:         stdErr,
+		memoryTracker:  exec.NewMemoryTracker(state.Memory),
+		stackTracker:   &NoopThreadedStackTracker{},
+		preimageOracle: exec.NewTrackingPreimageOracleReader(po),
+	}
+}
+
+func NewInstrumentedStateFromFile(stateFile string, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, log log.Logger) (*InstrumentedState, error) {
+	state, err := jsonutil.LoadJSON[State](stateFile)
+	if err != nil {
+		return nil, err
+	}
+	return NewInstrumentedState(state, po, stdOut, stdErr, log), nil
+}
+
+func (m *InstrumentedState) InitDebug(meta *program.Metadata) error {
+	stackTracker, err := NewThreadedStackTracker(m.state, meta)
+	if err != nil {
+		return err
+	}
+	m.stackTracker = stackTracker
+	return nil
+}
+
+func (m *InstrumentedState) Step(proof bool) (wit *mipsevm.StepWitness, err error) {
+	m.preimageOracle.Reset()
+	m.memoryTracker.Reset(proof)
+
+	if proof {
+		proofData := make([]byte, 0)
+		threadProof := m.state.EncodeThreadProof()
+		insnProof := m.state.Memory.MerkleProof(m.state.GetPC())
+		proofData = append(proofData, threadProof[:]...)
+		proofData = append(proofData, insnProof[:]...)
+
+		encodedWitness, stateHash := m.state.EncodeWitness()
+		wit = &mipsevm.StepWitness{
+			State:     encodedWitness,
+			StateHash: stateHash,
+			ProofData: proofData,
+		}
+	}
+	err = m.mipsStep()
+	if err != nil {
+		return nil, err
+	}
+
+	if proof {
+		memProof := m.memoryTracker.MemProof()
+		wit.ProofData = append(wit.ProofData, memProof[:]...)
+		lastPreimageKey, lastPreimage, lastPreimageOffset := m.preimageOracle.LastPreimage()
+		if lastPreimageOffset != ^uint32(0) {
+			wit.PreimageOffset = lastPreimageOffset
+			wit.PreimageKey = lastPreimageKey
+			wit.PreimageValue = lastPreimage
+		}
+	}
+	return
+}
+
+func (m *InstrumentedState) CheckInfiniteLoop() bool {
+	return false
+}
+
+func (m *InstrumentedState) LastPreimage() ([32]byte, []byte, uint32) {
+	return m.preimageOracle.LastPreimage()
+}
+
+func (m *InstrumentedState) GetState() mipsevm.FPVMState {
+	return m.state
+}
+
+func (m *InstrumentedState) GetDebugInfo() *mipsevm.DebugInfo {
+	return &mipsevm.DebugInfo{
+		Pages:               m.state.Memory.PageCount(),
+		NumPreimageRequests: m.preimageOracle.NumPreimageRequests(),
+		TotalPreimageSize:   m.preimageOracle.TotalPreimageSize(),
+	}
+}
+
+func (m *InstrumentedState) Traceback() {
+	m.stackTracker.Traceback()
+}

cannon/mipsevm/multithreaded/instrumented_test.go

+package multithreaded
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/memory"
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/testutil"
+)
+
+func vmFactory(state *State, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, log log.Logger) mipsevm.FPVM {
+	return NewInstrumentedState(state, po, stdOut, stdErr, log)
+}
+
+func TestInstrumentedState_OpenMips(t *testing.T) {
+	// TODO(cp-903): Add mt-specific tests here
+	testutil.RunVMTests_OpenMips(t, CreateEmptyState, vmFactory, "clone.bin")
+}
+
+func TestInstrumentedState_Hello(t *testing.T) {
+	testutil.RunVMTest_Hello(t, CreateInitialState, vmFactory, false)
+}
+
+func TestInstrumentedState_Claim(t *testing.T) {
+	testutil.RunVMTest_Claim(t, CreateInitialState, vmFactory, false)
+}
+
+func TestInstrumentedState_MultithreadedProgram(t *testing.T) {
+	state := testutil.LoadELFProgram(t, "../../example/bin/multithreaded.elf", CreateInitialState, false)
+	oracle := testutil.StaticOracle(t, []byte{})
+
+	var stdOutBuf, stdErrBuf bytes.Buffer
+	us := NewInstrumentedState(state, oracle, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr), testutil.CreateLogger())
+	for i := 0; i < 1_000_000; i++ {
+		if us.GetState().GetExited() {
+			break
+		}
+		_, err := us.Step(false)
+		require.NoError(t, err)
+	}
+	t.Logf("Completed in %d steps", state.Step)
+
+	require.True(t, state.Exited, "must complete program")
+	require.Equal(t, uint8(0), state.ExitCode, "exit with 0")
+	require.Contains(t, "waitgroup result: 42", stdErrBuf.String())
+	require.Contains(t, "channels result: 1234", stdErrBuf.String())
+	require.Equal(t, "", stdErrBuf.String(), "should not print any errors")
+}
+
+func TestInstrumentedState_Alloc(t *testing.T) {
+	t.Skip("TODO(client-pod#906): Currently failing - need to debug.")
+
+	state := testutil.LoadELFProgram(t, "../../example/bin/alloc.elf", CreateInitialState, false)
+	const numAllocs = 100 // where each alloc is a 32 MiB chunk
+	oracle := testutil.AllocOracle(t, numAllocs)
+
+	// completes in ~870 M steps
+	us := NewInstrumentedState(state, oracle, os.Stdout, os.Stderr, testutil.CreateLogger())
+	for i := 0; i < 20_000_000_000; i++ {
+		if us.GetState().GetExited() {
+			break
+		}
+		_, err := us.Step(false)
+		require.NoError(t, err)
+		if state.Step%10_000_000 == 0 {
+			t.Logf("Completed %d steps", state.Step)
+		}
+	}
+	t.Logf("Completed in %d steps", state.Step)
+	require.True(t, state.Exited, "must complete program")
+	require.Equal(t, uint8(0), state.ExitCode, "exit with 0")
+	require.Less(t, state.Memory.PageCount()*memory.PageSize, 1*1024*1024*1024, "must not allocate more than 1 GiB")
+}

cannon/mipsevm/multithreaded/mips.go

+package multithreaded
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/log"
+
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/exec"
+)
+
+func (m *InstrumentedState) handleSyscall() error {
+	thread := m.state.getCurrentThread()
+
+	syscallNum, a0, a1, a2, a3 := exec.GetSyscallArgs(m.state.GetRegisters())
+	v0 := uint32(0)
+	v1 := uint32(0)
+
+	//fmt.Printf("syscall: %d\n", syscallNum)
+	switch syscallNum {
+	case exec.SysMmap:
+		var newHeap uint32
+		v0, v1, newHeap = exec.HandleSysMmap(a0, a1, m.state.Heap)
+		m.state.Heap = newHeap
+	case exec.SysBrk:
+		v0 = exec.BrkStart
+	case exec.SysClone: // clone
+		// a0 = flag bitmask, a1 = stack pointer
+		if exec.ValidCloneFlags != a0 {
+			m.state.Exited = true
+			m.state.ExitCode = mipsevm.VMStatusPanic
+			return nil
+		}
+
+		v0 = m.state.NextThreadId
+		v1 = 0
+		newThread := &ThreadState{
+			ThreadId:         m.state.NextThreadId,
+			ExitCode:         0,
+			Exited:           false,
+			FutexAddr:        exec.FutexEmptyAddr,
+			FutexVal:         0,
+			FutexTimeoutStep: 0,
+			Cpu: mipsevm.CpuScalars{
+				PC:     thread.Cpu.NextPC,
+				NextPC: thread.Cpu.NextPC + 4,
+				HI:     thread.Cpu.HI,
+				LO:     thread.Cpu.LO,
+			},
+			Registers: thread.Registers,
+		}
+
+		newThread.Registers[29] = a1
+		// the child will perceive a 0 value as returned value instead, and no error
+		newThread.Registers[2] = 0
+		newThread.Registers[7] = 0
+		m.state.NextThreadId++
+
+		// Preempt this thread for the new one. But not before updating PCs
+		stackCaller := thread.Cpu.PC
+		stackTarget := thread.Cpu.NextPC
+		exec.HandleSyscallUpdates(&thread.Cpu, &thread.Registers, v0, v1)
+		m.pushThread(newThread)
+		// Note: We need to call stackTracker after pushThread
+		// to ensure we are tracking in the context of the new thread
+		m.stackTracker.PushStack(stackCaller, stackTarget)
+		return nil
+	case exec.SysExitGroup:
+		m.state.Exited = true
+		m.state.ExitCode = uint8(a0)
+		return nil
+	case exec.SysRead:
+		var newPreimageOffset uint32
+		v0, v1, newPreimageOffset = exec.HandleSysRead(a0, a1, a2, m.state.PreimageKey, m.state.PreimageOffset, m.preimageOracle, m.state.Memory, m.memoryTracker)
+		m.state.PreimageOffset = newPreimageOffset
+	case exec.SysWrite:
+		var newLastHint hexutil.Bytes
+		var newPreimageKey common.Hash
+		var newPreimageOffset uint32
+		v0, v1, newLastHint, newPreimageKey, newPreimageOffset = exec.HandleSysWrite(a0, a1, a2, m.state.LastHint, m.state.PreimageKey, m.state.PreimageOffset, m.preimageOracle, m.state.Memory, m.memoryTracker, m.stdOut, m.stdErr)
+		m.state.LastHint = newLastHint
+		m.state.PreimageKey = newPreimageKey
+		m.state.PreimageOffset = newPreimageOffset
+	case exec.SysFcntl:
+		v0, v1 = exec.HandleSysFcntl(a0, a1)
+	case exec.SysGetTID:
+		v0 = thread.ThreadId
+		v1 = 0
+	case exec.SysExit:
+		thread.Exited = true
+		thread.ExitCode = uint8(a0)
+		if m.lastThreadRemaining() {
+			m.state.Exited = true
+			m.state.ExitCode = uint8(a0)
+		}
+		return nil
+	case exec.SysFutex:
+		// args: a0 = addr, a1 = op, a2 = val, a3 = timeout
+		switch a1 {
+		case exec.FutexWaitPrivate:
+			thread.FutexAddr = a0
+			m.memoryTracker.TrackMemAccess(a0)
+			mem := m.state.Memory.GetMemory(a0)
+			if mem != a2 {
+				v0 = exec.SysErrorSignal
+				v1 = exec.MipsEAGAIN
+			} else {
+				thread.FutexVal = a2
+				if a3 == 0 {
+					thread.FutexTimeoutStep = exec.FutexNoTimeout
+				} else {
+					thread.FutexTimeoutStep = m.state.Step + exec.FutexTimeoutSteps
+				}
+				// Leave cpu scalars as-is. This instruction will be completed by `onWaitComplete`
+				return nil
+			}
+		case exec.FutexWakePrivate:
+			// Trigger thread traversal starting from the left stack until we find one waiting on the wakeup
+			// address
+			m.state.Wakeup = a0
+			// Don't indicate to the program that we've woken up a waiting thread, as there are no guarantees.
+			// The woken up thread should indicate this in userspace.
+			v0 = 0
+			v1 = 0
+			exec.HandleSyscallUpdates(&thread.Cpu, &thread.Registers, v0, v1)
+			m.preemptThread(thread)
+			m.state.TraverseRight = len(m.state.LeftThreadStack) == 0
+			return nil
+		default:
+			v0 = exec.SysErrorSignal
+			v1 = exec.MipsEINVAL
+		}
+	case exec.SysSchedYield, exec.SysNanosleep:
+		v0 = 0
+		v1 = 0
+		exec.HandleSyscallUpdates(&thread.Cpu, &thread.Registers, v0, v1)
+		m.preemptThread(thread)
+		return nil
+	case exec.SysOpen:
+		v0 = exec.SysErrorSignal
+		v1 = exec.MipsEBADF
+	case exec.SysMunmap:
+	case exec.SysGetAffinity:
+	case exec.SysMadvise:
+	case exec.SysRtSigprocmask:
+	case exec.SysSigaltstack:
+	case exec.SysRtSigaction:
+	case exec.SysPrlimit64:
+	case exec.SysClose:
+	case exec.SysPread64:
+	case exec.SysFstat64:
+	case exec.SysOpenAt:
+	case exec.SysReadlink:
+	case exec.SysReadlinkAt:
+	case exec.SysIoctl:
+	case exec.SysEpollCreate1:
+	case exec.SysPipe2:
+	case exec.SysEpollCtl:
+	case exec.SysEpollPwait:
+	case exec.SysGetRandom:
+	case exec.SysUname:
+	case exec.SysStat64:
+	case exec.SysGetuid:
+	case exec.SysGetgid:
+	case exec.SysLlseek:
+	case exec.SysMinCore:
+	case exec.SysTgkill:
+	case exec.SysSetITimer:
+	case exec.SysTimerCreate:
+	case exec.SysTimerSetTime:
+	case exec.SysTimerDelete:
+	case exec.SysClockGetTime:
+	default:
+		m.Traceback()
+		panic(fmt.Sprintf("unrecognized syscall: %d", syscallNum))
+	}
+
+	exec.HandleSyscallUpdates(&thread.Cpu, &thread.Registers, v0, v1)
+	return nil
+}
+
+func (m *InstrumentedState) mipsStep() error {
+	if m.state.Exited {
+		return nil
+	}
+	m.state.Step += 1
+	thread := m.state.getCurrentThread()
+
+	// During wakeup traversal, search for the first thread blocked on the wakeup address.
+	// Don't allow regular execution until we have found such a thread or else we have visited all threads.
+	if m.state.Wakeup != exec.FutexEmptyAddr {
+		// We are currently performing a wakeup traversal
+		if m.state.Wakeup == thread.FutexAddr {
+			// We found a target thread, resume normal execution and process this thread
+			m.state.Wakeup = exec.FutexEmptyAddr
+		} else {
+			// This is not the thread we're looking for, move on
+			traversingRight := m.state.TraverseRight
+			changedDirections := m.preemptThread(thread)
+			if traversingRight && changedDirections {
+				// We started the wakeup traversal walking left and we've now walked all the way right
+				// We have therefore visited all threads and can resume normal thread execution
+				m.state.Wakeup = exec.FutexEmptyAddr
+			}
+		}
+		return nil
+	}
+
+	if thread.Exited {
+		m.popThread()
+		m.stackTracker.DropThread(thread.ThreadId)
+		return nil
+	}
+
+	// check if thread is blocked on a futex
+	if thread.FutexAddr != exec.FutexEmptyAddr {
+		// if set, then check futex
+		// check timeout first
+		if m.state.Step > thread.FutexTimeoutStep {
+			// timeout! Allow execution
+			m.onWaitComplete(thread, true)
+			return nil
+		} else {
+			m.memoryTracker.TrackMemAccess(thread.FutexAddr)
+			mem := m.state.Memory.GetMemory(thread.FutexAddr)
+			if thread.FutexVal == mem {
+				// still got expected value, continue sleeping, try next thread.
+				m.preemptThread(thread)
+				return nil
+			} else {
+				// wake thread up, the value at its address changed!
+				// Userspace can turn thread back to sleep if it was too sporadic.
+				m.onWaitComplete(thread, false)
+				return nil
+			}
+		}
+	}
+
+	if m.state.StepsSinceLastContextSwitch >= exec.SchedQuantum {
+		// Force a context switch as this thread has been active too long
+		if m.state.threadCount() > 1 {
+			// Log if we're hitting our context switch limit - only matters if we have > 1 thread
+			if m.log.Enabled(context.Background(), log.LevelTrace) {
+				msg := fmt.Sprintf("Thread has reached maximum execution steps (%v) - preempting.", exec.SchedQuantum)
+				m.log.Trace(msg, "threadId", thread.ThreadId, "threadCount", m.state.threadCount(), "pc", thread.Cpu.PC)
+			}
+		}
+		m.preemptThread(thread)
+		return nil
+	}
+	m.state.StepsSinceLastContextSwitch += 1
+
+	//instruction fetch
+	insn, opcode, fun := exec.GetInstructionDetails(m.state.GetPC(), m.state.Memory)
+
+	// Handle syscall separately
+	// syscall (can read and write)
+	if opcode == 0 && fun == 0xC {
+		return m.handleSyscall()
+	}
+
+	// Exec the rest of the step logic
+	return exec.ExecMipsCoreStepLogic(m.state.getCpu(), m.state.GetRegisters(), m.state.Memory, insn, opcode, fun, m.memoryTracker, m.stackTracker)
+}
+
+func (m *InstrumentedState) onWaitComplete(thread *ThreadState, isTimedOut bool) {
+	// Clear the futex state
+	thread.FutexAddr = exec.FutexEmptyAddr
+	thread.FutexVal = 0
+	thread.FutexTimeoutStep = 0
+
+	// Complete the FUTEX_WAIT syscall
+	v0 := uint32(0)
+	v1 := uint32(0)
+	if isTimedOut {
+		v0 = exec.SysErrorSignal
+		v1 = exec.MipsETIMEDOUT
+	}
+	exec.HandleSyscallUpdates(&thread.Cpu, &thread.Registers, v0, v1)
+
+	// Clear wakeup signal
+	m.state.Wakeup = exec.FutexEmptyAddr
+}
+
+func (m *InstrumentedState) preemptThread(thread *ThreadState) bool {
+	// Pop thread from the current stack and push to the other stack
+	if m.state.TraverseRight {
+		rtThreadCnt := len(m.state.RightThreadStack)
+		if rtThreadCnt == 0 {
+			panic("empty right thread stack")
+		}
+		m.state.RightThreadStack = m.state.RightThreadStack[:rtThreadCnt-1]
+		m.state.LeftThreadStack = append(m.state.LeftThreadStack, thread)
+	} else {
+		lftThreadCnt := len(m.state.LeftThreadStack)
+		if lftThreadCnt == 0 {
+			panic("empty left thread stack")
+		}
+		m.state.LeftThreadStack = m.state.LeftThreadStack[:lftThreadCnt-1]
+		m.state.RightThreadStack = append(m.state.RightThreadStack, thread)
+	}
+
+	changeDirections := false
+	current := m.state.getActiveThreadStack()
+	if len(current) == 0 {
+		m.state.TraverseRight = !m.state.TraverseRight
+		changeDirections = true
+	}
+
+	m.state.StepsSinceLastContextSwitch = 0
+	return changeDirections
+}
+
+func (m *InstrumentedState) pushThread(thread *ThreadState) {
+	if m.state.TraverseRight {
+		m.state.RightThreadStack = append(m.state.RightThreadStack, thread)
+	} else {
+		m.state.LeftThreadStack = append(m.state.LeftThreadStack, thread)
+	}
+	m.state.StepsSinceLastContextSwitch = 0
+}
+
+func (m *InstrumentedState) popThread() {
+	if m.state.TraverseRight {
+		m.state.RightThreadStack = m.state.RightThreadStack[:len(m.state.RightThreadStack)-1]
+	} else {
+		m.state.LeftThreadStack = m.state.LeftThreadStack[:len(m.state.LeftThreadStack)-1]
+	}
+
+	current := m.state.getActiveThreadStack()
+	if len(current) == 0 {
+		m.state.TraverseRight = !m.state.TraverseRight
+	}
+	m.state.StepsSinceLastContextSwitch = 0
+}
+
+func (m *InstrumentedState) lastThreadRemaining() bool {
+	return m.state.threadCount() == 1
+}

cannon/mipsevm/multithreaded/stack.go

+package multithreaded
+
+import (
+	"errors"
+
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/exec"
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/program"
+)
+
+type ThreadedStackTracker interface {
+	exec.TraceableStackTracker
+	DropThread(threadId uint32)
+}
+
+type NoopThreadedStackTracker struct {
+	exec.NoopStackTracker
+}
+
+var _ ThreadedStackTracker = (*ThreadedStackTrackerImpl)(nil)
+
+func (n *NoopThreadedStackTracker) DropThread(threadId uint32) {}
+
+type ThreadedStackTrackerImpl struct {
+	meta               *program.Metadata
+	state              *State
+	trackersByThreadId map[uint32]exec.TraceableStackTracker
+}
+
+var _ ThreadedStackTracker = (*ThreadedStackTrackerImpl)(nil)
+
+func NewThreadedStackTracker(state *State, meta *program.Metadata) (*ThreadedStackTrackerImpl, error) {
+	if meta == nil {
+		return nil, errors.New("metadata is nil")
+	}
+
+	return &ThreadedStackTrackerImpl{
+		state:              state,
+		meta:               meta,
+		trackersByThreadId: make(map[uint32]exec.TraceableStackTracker),
+	}, nil
+}
+
+func (t *ThreadedStackTrackerImpl) PushStack(caller uint32, target uint32) {
+	t.getCurrentTracker().PushStack(caller, target)
+}
+
+func (t *ThreadedStackTrackerImpl) PopStack() {
+	t.getCurrentTracker().PopStack()
+}
+
+func (t *ThreadedStackTrackerImpl) Traceback() {
+	t.getCurrentTracker().Traceback()
+}
+
+func (t *ThreadedStackTrackerImpl) getCurrentTracker() exec.TraceableStackTracker {
+	thread := t.state.getCurrentThread()
+	tracker, exists := t.trackersByThreadId[thread.ThreadId]
+	if !exists {
+		tracker = exec.NewStackTrackerUnsafe(t.state, t.meta)
+		t.trackersByThreadId[thread.ThreadId] = tracker
+	}
+	return tracker
+}
+
+func (t *ThreadedStackTrackerImpl) DropThread(threadId uint32) {
+	delete(t.trackersByThreadId, threadId)
+}

cannon/mipsevm/multithreaded/state.go

@@ -9,64 +9,10 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/exec"
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/memory"
 )
 
-// SERIALIZED_THREAD_SIZE is the size of a serialized ThreadState object
-const SERIALIZED_THREAD_SIZE = 166
-
-// THREAD_WITNESS_SIZE is the size of a thread witness encoded in bytes.
-//
-//	It consists of the active thread serialized and concatenated with the
-//	32 byte hash onion of the active thread stack without the active thread
-const THREAD_WITNESS_SIZE = SERIALIZED_THREAD_SIZE + 32
-
-// The empty thread root - keccak256(bytes32(0) ++ bytes32(0))
-var EmptyThreadsRoot common.Hash = common.HexToHash("0xad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb5")
-
-type ThreadState struct {
-	ThreadId         uint32             `json:"threadId"`
-	ExitCode         uint8              `json:"exit"`
-	Exited           bool               `json:"exited"`
-	FutexAddr        uint32             `json:"futexAddr"`
-	FutexVal         uint32             `json:"futexVal"`
-	FutexTimeoutStep uint64             `json:"futexTimeoutStep"`
-	Cpu              mipsevm.CpuScalars `json:"cpu"`
-	Registers        [32]uint32         `json:"registers"`
-}
-
-func (t *ThreadState) serializeThread() []byte {
-	out := make([]byte, 0, SERIALIZED_THREAD_SIZE)
-
-	out = binary.BigEndian.AppendUint32(out, t.ThreadId)
-	out = append(out, t.ExitCode)
-	out = mipsevm.AppendBoolToWitness(out, t.Exited)
-	out = binary.BigEndian.AppendUint32(out, t.FutexAddr)
-	out = binary.BigEndian.AppendUint32(out, t.FutexVal)
-	out = binary.BigEndian.AppendUint64(out, t.FutexTimeoutStep)
-
-	out = binary.BigEndian.AppendUint32(out, t.Cpu.PC)
-	out = binary.BigEndian.AppendUint32(out, t.Cpu.NextPC)
-	out = binary.BigEndian.AppendUint32(out, t.Cpu.LO)
-	out = binary.BigEndian.AppendUint32(out, t.Cpu.HI)
-
-	for _, r := range t.Registers {
-		out = binary.BigEndian.AppendUint32(out, r)
-	}
-
-	return out
-}
-
-func computeThreadRoot(prevStackRoot common.Hash, threadToPush *ThreadState) common.Hash {
-	hashedThread := crypto.Keccak256Hash(threadToPush.serializeThread())
-
-	var hashData []byte
-	hashData = append(hashData, prevStackRoot[:]...)
-	hashData = append(hashData, hashedThread[:]...)
-
-	return crypto.Keccak256Hash(hashData)
-}
-
 // STATE_WITNESS_SIZE is the size of the state witness encoding in bytes.
 const STATE_WITNESS_SIZE = 163
 const (
@@ -100,10 +46,10 @@ type State struct {
 	StepsSinceLastContextSwitch uint64 `json:"stepsSinceLastContextSwitch"`
 	Wakeup                      uint32 `json:"wakeup"`
 
-	TraverseRight    bool          `json:"traverseRight"`
-	LeftThreadStack  []ThreadState `json:"leftThreadStack"`
-	RightThreadStack []ThreadState `json:"rightThreadStack"`
-	NextThreadId     uint32        `json:"nextThreadId"`
+	TraverseRight    bool           `json:"traverseRight"`
+	LeftThreadStack  []*ThreadState `json:"leftThreadStack"`
+	RightThreadStack []*ThreadState `json:"rightThreadStack"`
+	NextThreadId     uint32         `json:"nextThreadId"`
 
 	// LastHint is optional metadata, and not part of the VM state itself.
 	// It is used to remember the last pre-image hint,
@@ -116,23 +62,10 @@ type State struct {
 	LastHint hexutil.Bytes `json:"lastHint,omitempty"`
 }
 
+var _ mipsevm.FPVMState = (*State)(nil)
+
 func CreateEmptyState() *State {
-	initThreadId := uint32(0)
-	initThread := ThreadState{
-		ThreadId: initThreadId,
-		ExitCode: 0,
-		Exited:   false,
-		Cpu: mipsevm.CpuScalars{
-			PC:     0,
-			NextPC: 0,
-			LO:     0,
-			HI:     0,
-		},
-		FutexAddr:        ^uint32(0),
-		FutexVal:         0,
-		FutexTimeoutStep: 0,
-		Registers:        [32]uint32{},
-	}
+	initThread := CreateEmptyThread()
 
 	return &State{
 		Memory:           memory.NewMemory(),
@@ -140,11 +73,11 @@ func CreateEmptyState() *State {
 		ExitCode:         0,
 		Exited:           false,
 		Step:             0,
-		Wakeup:           ^uint32(0),
+		Wakeup:           exec.FutexEmptyAddr,
 		TraverseRight:    false,
-		LeftThreadStack:  []ThreadState{initThread},
-		RightThreadStack: []ThreadState{},
-		NextThreadId:     initThreadId + 1,
+		LeftThreadStack:  []*ThreadState{initThread},
+		RightThreadStack: []*ThreadState{},
+		NextThreadId:     initThread.ThreadId + 1,
 	}
 }
 
@@ -166,11 +99,11 @@ func (s *State) getCurrentThread() *ThreadState {
 		panic("Active thread stack is empty")
 	}
 
-	return &activeStack[activeStackSize-1]
+	return activeStack[activeStackSize-1]
 }
 
-func (s *State) getActiveThreadStack() []ThreadState {
-	var activeStack []ThreadState
+func (s *State) getActiveThreadStack() []*ThreadState {
+	var activeStack []*ThreadState
 	if s.TraverseRight {
 		activeStack = s.RightThreadStack
 	} else {
@@ -188,25 +121,15 @@ func (s *State) getLeftThreadStackRoot() common.Hash {
 	return s.calculateThreadStackRoot(s.LeftThreadStack)
 }
 
-func (s *State) calculateThreadStackRoot(stack []ThreadState) common.Hash {
+func (s *State) calculateThreadStackRoot(stack []*ThreadState) common.Hash {
 	curRoot := EmptyThreadsRoot
 	for _, thread := range stack {
-		curRoot = computeThreadRoot(curRoot, &thread)
+		curRoot = computeThreadRoot(curRoot, thread)
 	}
 
 	return curRoot
 }
 
-func (s *State) PreemptThread() {
-	// TODO(CP-903)
-	panic("Not Implemented")
-}
-
-func (s *State) PushThread(thread *ThreadState) {
-	// TODO(CP-903)
-	panic("Not Implemented")
-}
-
 func (s *State) GetPC() uint32 {
 	activeThread := s.getCurrentThread()
 	return activeThread.Cpu.PC
@@ -217,6 +140,10 @@ func (s *State) GetRegisters() *[32]uint32 {
 	return &activeThread.Registers
 }
 
+func (s *State) getCpu() *mipsevm.CpuScalars {
+	return &s.getCurrentThread().Cpu
+}
+
 func (s *State) GetExitCode() uint8 { return s.ExitCode }
 
 func (s *State) GetExited() bool { return s.Exited }
@@ -255,6 +182,29 @@ func (s *State) EncodeWitness() ([]byte, common.Hash) {
 	return out, stateHashFromWitness(out)
 }
 
+func (s *State) EncodeThreadProof() []byte {
+	activeStack := s.getActiveThreadStack()
+	threadCount := len(activeStack)
+	if threadCount == 0 {
+		panic("Invalid empty thread stack")
+	}
+
+	activeThread := activeStack[threadCount-1]
+	otherThreads := activeStack[:threadCount-1]
+	threadBytes := activeThread.serializeThread()
+	otherThreadsWitness := s.calculateThreadStackRoot(otherThreads)
+
+	out := make([]byte, 0, THREAD_WITNESS_SIZE)
+	out = append(out, threadBytes[:]...)
+	out = append(out, otherThreadsWitness[:]...)
+
+	return out
+}
+
+func (s *State) threadCount() int {
+	return len(s.LeftThreadStack) + len(s.RightThreadStack)
+}
+
 type StateWitness []byte
 
 func (sw StateWitness) StateHash() (common.Hash, error) {

cannon/mipsevm/multithreaded/state_test.go

@@ -6,9 +6,11 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/exec"
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/program"
 )
 
@@ -127,3 +129,93 @@ func TestState_EmptyThreadsRoot(t *testing.T) {
 
 	require.Equal(t, expectedEmptyRoot, EmptyThreadsRoot)
 }
+
+func TestState_EncodeThreadProof_SingleThread(t *testing.T) {
+	state := CreateEmptyState()
+	// Set some fields on the active thread
+	activeThread := state.getCurrentThread()
+	activeThread.Cpu.PC = 4
+	activeThread.Cpu.NextPC = 8
+	activeThread.Cpu.HI = 11
+	activeThread.Cpu.LO = 22
+	for i := 0; i < 32; i++ {
+		activeThread.Registers[i] = uint32(i)
+	}
+
+	expectedProof := append([]byte{}, activeThread.serializeThread()[:]...)
+	expectedProof = append(expectedProof, EmptyThreadsRoot[:]...)
+
+	actualProof := state.EncodeThreadProof()
+	require.Equal(t, THREAD_WITNESS_SIZE, len(actualProof))
+	require.Equal(t, expectedProof, actualProof)
+}
+
+func TestState_EncodeThreadProof_MultipleThreads(t *testing.T) {
+	state := CreateEmptyState()
+	// Add some more threads
+	require.Equal(t, state.TraverseRight, false, "sanity check")
+	state.LeftThreadStack = append(state.LeftThreadStack, CreateEmptyThread())
+	state.LeftThreadStack = append(state.LeftThreadStack, CreateEmptyThread())
+	require.Equal(t, 3, len(state.LeftThreadStack), "sanity check")
+
+	// Set some fields on our threads
+	for i := 0; i < 3; i++ {
+		curThread := state.LeftThreadStack[i]
+		curThread.Cpu.PC = uint32(4 * i)
+		curThread.Cpu.NextPC = curThread.Cpu.PC + 4
+		curThread.Cpu.HI = uint32(11 + i)
+		curThread.Cpu.LO = uint32(22 + i)
+		for j := 0; j < 32; j++ {
+			curThread.Registers[j] = uint32(j + i)
+		}
+	}
+
+	expectedRoot := EmptyThreadsRoot
+	for i := 0; i < 2; i++ {
+		curThread := state.LeftThreadStack[i]
+		hashedThread := crypto.Keccak256Hash(curThread.serializeThread())
+
+		// root = prevRoot ++ hash(curRoot)
+		hashData := append([]byte{}, expectedRoot[:]...)
+		hashData = append(hashData, hashedThread[:]...)
+		expectedRoot = crypto.Keccak256Hash(hashData)
+	}
+
+	expectedProof := append([]byte{}, state.getCurrentThread().serializeThread()[:]...)
+	expectedProof = append(expectedProof, expectedRoot[:]...)
+
+	actualProof := state.EncodeThreadProof()
+	require.Equal(t, THREAD_WITNESS_SIZE, len(actualProof))
+	require.Equal(t, expectedProof, actualProof)
+}
+
+func TestState_EncodeThreadProof_EmptyThreadStackPanic(t *testing.T) {
+	cases := []struct {
+		name          string
+		wakeupAddr    uint32
+		traverseRight bool
+	}{
+		{"traverse left during wakeup traversal", uint32(99), false},
+		{"traverse left during normal traversal", exec.FutexEmptyAddr, false},
+		{"traverse right during wakeup traversal", uint32(99), true},
+		{"traverse right during normal traversal", exec.FutexEmptyAddr, true},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			// Set up invalid state where the active stack is empty
+			state := CreateEmptyState()
+			state.Wakeup = c.wakeupAddr
+			state.TraverseRight = c.traverseRight
+			if c.traverseRight {
+				state.LeftThreadStack = []*ThreadState{CreateEmptyThread()}
+				state.RightThreadStack = []*ThreadState{}
+			} else {
+				state.LeftThreadStack = []*ThreadState{}
+				state.RightThreadStack = []*ThreadState{CreateEmptyThread()}
+			}
+
+			assert.PanicsWithValue(t, "Invalid empty thread stack", func() { state.EncodeThreadProof() })
+		})
+	}
+}

cannon/mipsevm/multithreaded/thread.go

+package multithreaded
+
+import (
+	"encoding/binary"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/exec"
+)
+
+// SERIALIZED_THREAD_SIZE is the size of a serialized ThreadState object
+const SERIALIZED_THREAD_SIZE = 166
+
+// THREAD_WITNESS_SIZE is the size of a thread witness encoded in bytes.
+//
+//	It consists of the active thread serialized and concatenated with the
+//	32 byte hash onion of the active thread stack without the active thread
+const THREAD_WITNESS_SIZE = SERIALIZED_THREAD_SIZE + 32
+
+// The empty thread root - keccak256(bytes32(0) ++ bytes32(0))
+var EmptyThreadsRoot common.Hash = common.HexToHash("0xad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb5")
+
+type ThreadState struct {
+	ThreadId         uint32             `json:"threadId"`
+	ExitCode         uint8              `json:"exit"`
+	Exited           bool               `json:"exited"`
+	FutexAddr        uint32             `json:"futexAddr"`
+	FutexVal         uint32             `json:"futexVal"`
+	FutexTimeoutStep uint64             `json:"futexTimeoutStep"`
+	Cpu              mipsevm.CpuScalars `json:"cpu"`
+	Registers        [32]uint32         `json:"registers"`
+}
+
+func CreateEmptyThread() *ThreadState {
+	initThreadId := uint32(0)
+	return &ThreadState{
+		ThreadId: initThreadId,
+		ExitCode: 0,
+		Exited:   false,
+		Cpu: mipsevm.CpuScalars{
+			PC:     0,
+			NextPC: 4,
+			LO:     0,
+			HI:     0,
+		},
+		FutexAddr:        exec.FutexEmptyAddr,
+		FutexVal:         0,
+		FutexTimeoutStep: 0,
+		Registers:        [32]uint32{},
+	}
+}
+
+func (t *ThreadState) serializeThread() []byte {
+	out := make([]byte, 0, SERIALIZED_THREAD_SIZE)
+
+	out = binary.BigEndian.AppendUint32(out, t.ThreadId)
+	out = append(out, t.ExitCode)
+	out = mipsevm.AppendBoolToWitness(out, t.Exited)
+	out = binary.BigEndian.AppendUint32(out, t.FutexAddr)
+	out = binary.BigEndian.AppendUint32(out, t.FutexVal)
+	out = binary.BigEndian.AppendUint64(out, t.FutexTimeoutStep)
+
+	out = binary.BigEndian.AppendUint32(out, t.Cpu.PC)
+	out = binary.BigEndian.AppendUint32(out, t.Cpu.NextPC)
+	out = binary.BigEndian.AppendUint32(out, t.Cpu.LO)
+	out = binary.BigEndian.AppendUint32(out, t.Cpu.HI)
+
+	for _, r := range t.Registers {
+		out = binary.BigEndian.AppendUint32(out, r)
+	}
+
+	return out
+}
+
+func computeThreadRoot(prevStackRoot common.Hash, threadToPush *ThreadState) common.Hash {
+	hashedThread := crypto.Keccak256Hash(threadToPush.serializeThread())
+
+	var hashData []byte
+	hashData = append(hashData, prevStackRoot[:]...)
+	hashData = append(hashData, hashedThread[:]...)
+
+	return crypto.Keccak256Hash(hashData)
+}

cannon/mipsevm/program/load.go

@@ -11,9 +11,9 @@ import (
 
 const HEAP_START = 0x05000000
 
-type CreateFPVMState[T mipsevm.FPVMState] func(pc, heapStart uint32) T
+type CreateInitialFPVMState[T mipsevm.FPVMState] func(pc, heapStart uint32) T
 
-func LoadELF[T mipsevm.FPVMState](f *elf.File, initState CreateFPVMState[T]) (T, error) {
+func LoadELF[T mipsevm.FPVMState](f *elf.File, initState CreateInitialFPVMState[T]) (T, error) {
 	var empty T
 	s := initState(uint32(f.Entry), HEAP_START)
 

cannon/mipsevm/program/metadata.go

@@ -50,7 +50,9 @@ func (m *Metadata) LookupSymbol(addr uint32) string {
 	return out.Name
 }
 
-func (m *Metadata) SymbolMatcher(name string) func(addr uint32) bool {
+type SymbolMatcher func(addr uint32) bool
+
+func (m *Metadata) CreateSymbolMatcher(name string) SymbolMatcher {
 	for _, s := range m.Symbols {
 		if s.Name == name {
 			start := s.Start

cannon/mipsevm/program/patch.go

@@ -10,6 +10,8 @@ import (
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/memory"
 )
 
+// TODO(cp-903) Consider breaking up go patching into performance and threading-related patches so we can
+// selectively apply the perf patching to MTCannon
 func PatchGo(f *elf.File, st mipsevm.FPVMState) error {
 	symbols, err := f.Symbols()
 	if err != nil {
@@ -45,15 +47,12 @@ func PatchGo(f *elf.File, st mipsevm.FPVMState) error {
 			})); err != nil {
 				return fmt.Errorf("failed to patch Go runtime.gcenable: %w", err)
 			}
-		case "runtime.MemProfileRate":
-			if err := st.GetMemory().SetMemoryRange(uint32(s.Value), bytes.NewReader(make([]byte, 4))); err != nil { // disable mem profiling, to avoid a lot of unnecessary floating point ops
-				return err
-			}
 		}
 	}
 	return nil
 }
 
+// TODO(cp-903) Consider setting envar "GODEBUG=memprofilerate=0" for go programs to disable memprofiling
 func PatchStack(st mipsevm.FPVMState) error {
 	// setup stack pointer
 	sp := uint32(0x7f_ff_d0_00)

cannon/mipsevm/singlethreaded/instrumented.go

@@ -10,6 +10,9 @@ import (
 )
 
 type InstrumentedState struct {
+	meta       *program.Metadata
+	sleepCheck program.SymbolMatcher
+
 	state *State
 
 	stdOut io.Writer
@@ -21,8 +24,19 @@ type InstrumentedState struct {
 	preimageOracle *exec.TrackingPreimageOracleReader
 }
 
-func NewInstrumentedState(state *State, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer) *InstrumentedState {
+var _ mipsevm.FPVM = (*InstrumentedState)(nil)
+
+func NewInstrumentedState(state *State, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, meta *program.Metadata) *InstrumentedState {
+	var sleepCheck program.SymbolMatcher
+	if meta == nil {
+		sleepCheck = func(addr uint32) bool { return false }
+	} else {
+		sleepCheck = meta.CreateSymbolMatcher("runtime.notesleep")
+	}
+
 	return &InstrumentedState{
+		meta:           meta,
+		sleepCheck:     sleepCheck,
 		state:          state,
 		stdOut:         stdOut,
 		stdErr:         stdErr,
@@ -32,16 +46,16 @@ func NewInstrumentedState(state *State, po mipsevm.PreimageOracle, stdOut, stdEr
 	}
 }
 
-func NewInstrumentedStateFromFile(stateFile string, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer) (*InstrumentedState, error) {
+func NewInstrumentedStateFromFile(stateFile string, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, meta *program.Metadata) (*InstrumentedState, error) {
 	state, err := jsonutil.LoadJSON[State](stateFile)
 	if err != nil {
 		return nil, err
 	}
-	return NewInstrumentedState(state, po, stdOut, stdErr), nil
+	return NewInstrumentedState(state, po, stdOut, stdErr, meta), nil
 }
 
-func (m *InstrumentedState) InitDebug(meta *program.Metadata) error {
-	stackTracker, err := exec.NewStackTracker(m.state, meta)
+func (m *InstrumentedState) InitDebug() error {
+	stackTracker, err := exec.NewStackTracker(m.state, m.meta)
 	if err != nil {
 		return err
 	}
@@ -80,6 +94,10 @@ func (m *InstrumentedState) Step(proof bool) (wit *mipsevm.StepWitness, err erro
 	return
 }
 
+func (m *InstrumentedState) CheckInfiniteLoop() bool {
+	return m.sleepCheck(m.state.GetPC())
+}
+
 func (m *InstrumentedState) LastPreimage() ([32]byte, []byte, uint32) {
 	return m.preimageOracle.LastPreimage()
 }

cannon/mipsevm/singlethreaded/instrumented_test.go

 package singlethreaded
 
 import (
-	"bytes"
 	"io"
-	"os"
-	"path"
 	"testing"
 
-	"github.com/stretchr/testify/require"
+	"github.com/ethereum/go-ethereum/log"
 
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
-	"github.com/ethereum-optimism/optimism/cannon/mipsevm/memory"
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/testutil"
 )
 
-func TestState(t *testing.T) {
-	testFiles, err := os.ReadDir("../tests/open_mips_tests/test/bin")
-	require.NoError(t, err)
-
-	for _, f := range testFiles {
-		t.Run(f.Name(), func(t *testing.T) {
-			oracle := testutil.SelectOracleFixture(t, f.Name())
-			// Short-circuit early for exit_group.bin
-			exitGroup := f.Name() == "exit_group.bin"
-
-			// TODO: currently tests are compiled as flat binary objects
-			// We can use more standard tooling to compile them to ELF files and get remove maketests.py
-			fn := path.Join("../tests/open_mips_tests/test/bin", f.Name())
-			//elfProgram, err := elf.Open()
-			//require.NoError(t, err, "must load test ELF binary")
-			//state, err := LoadELF(elfProgram)
-			//require.NoError(t, err, "must load ELF into state")
-			programMem, err := os.ReadFile(fn)
-			require.NoError(t, err)
-			state := &State{Cpu: mipsevm.CpuScalars{PC: 0, NextPC: 4}, Memory: memory.NewMemory()}
-			err = state.Memory.SetMemoryRange(0, bytes.NewReader(programMem))
-			require.NoError(t, err, "load program into state")
-
-			// set the return address ($ra) to jump into when test completes
-			state.Registers[31] = testutil.EndAddr
-
-			us := NewInstrumentedState(state, oracle, os.Stdout, os.Stderr)
-
-			for i := 0; i < 1000; i++ {
-				if us.state.Cpu.PC == testutil.EndAddr {
-					break
-				}
-				if exitGroup && us.state.Exited {
-					break
-				}
-				_, err := us.Step(false)
-				require.NoError(t, err)
-			}
-
-			if exitGroup {
-				require.NotEqual(t, uint32(testutil.EndAddr), us.state.Cpu.PC, "must not reach end")
-				require.True(t, us.state.Exited, "must set exited state")
-				require.Equal(t, uint8(1), us.state.ExitCode, "must exit with 1")
-			} else {
-				require.Equal(t, uint32(testutil.EndAddr), us.state.Cpu.PC, "must reach end")
-				done, result := state.Memory.GetMemory(testutil.BaseAddrEnd+4), state.Memory.GetMemory(testutil.BaseAddrEnd+8)
-				// inspect test result
-				require.Equal(t, done, uint32(1), "must be done")
-				require.Equal(t, result, uint32(1), "must have success result")
-			}
-		})
-	}
+func vmFactory(state *State, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, log log.Logger) mipsevm.FPVM {
+	return NewInstrumentedState(state, po, stdOut, stdErr, nil)
 }
 
-func TestHello(t *testing.T) {
-	state := testutil.LoadELFProgram(t, "../../example/bin/hello.elf", CreateInitialState)
-
-	var stdOutBuf, stdErrBuf bytes.Buffer
-	us := NewInstrumentedState(state, nil, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr))
-
-	for i := 0; i < 400_000; i++ {
-		if us.state.Exited {
-			break
-		}
-		_, err := us.Step(false)
-		require.NoError(t, err)
-	}
-
-	require.True(t, state.Exited, "must complete program")
-	require.Equal(t, uint8(0), state.ExitCode, "exit with 0")
-
-	require.Equal(t, "hello world!\n", stdOutBuf.String(), "stdout says hello")
-	require.Equal(t, "", stdErrBuf.String(), "stderr silent")
+func TestInstrumentedState_OpenMips(t *testing.T) {
+	testutil.RunVMTests_OpenMips(t, CreateEmptyState, vmFactory)
 }
 
-func TestClaim(t *testing.T) {
-	state := testutil.LoadELFProgram(t, "../../example/bin/claim.elf", CreateInitialState)
-
-	oracle, expectedStdOut, expectedStdErr := testutil.ClaimTestOracle(t)
-
-	var stdOutBuf, stdErrBuf bytes.Buffer
-	us := NewInstrumentedState(state, oracle, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr))
-
-	for i := 0; i < 2000_000; i++ {
-		if us.GetState().GetExited() {
-			break
-		}
-		_, err := us.Step(false)
-		require.NoError(t, err)
-	}
-
-	require.True(t, state.Exited, "must complete program")
-	require.Equal(t, uint8(0), state.ExitCode, "exit with 0")
-
-	require.Equal(t, expectedStdOut, stdOutBuf.String(), "stdout")
-	require.Equal(t, expectedStdErr, stdErrBuf.String(), "stderr")
+func TestInstrumentedState_Hello(t *testing.T) {
+	testutil.RunVMTest_Hello(t, CreateInitialState, vmFactory, true)
 }
 
-func TestAlloc(t *testing.T) {
-	t.Skip("TODO(client-pod#906): Currently fails on Single threaded Cannon. Re-enable for the MT FPVM")
-
-	state := testutil.LoadELFProgram(t, "../example/bin/alloc.elf", CreateInitialState)
-	const numAllocs = 100 // where each alloc is a 32 MiB chunk
-	oracle := testutil.AllocOracle(t, numAllocs)
-
-	// completes in ~870 M steps
-	us := NewInstrumentedState(state, oracle, os.Stdout, os.Stderr)
-	for i := 0; i < 20_000_000_000; i++ {
-		if us.GetState().GetExited() {
-			break
-		}
-		_, err := us.Step(false)
-		require.NoError(t, err)
-		if state.Step%10_000_000 == 0 {
-			t.Logf("Completed %d steps", state.Step)
-		}
-	}
-	t.Logf("Completed in %d steps", state.Step)
-	require.True(t, state.Exited, "must complete program")
-	require.Equal(t, uint8(0), state.ExitCode, "exit with 0")
-	require.Less(t, state.Memory.PageCount()*memory.PageSize, 1*1024*1024*1024, "must not allocate more than 1 GiB")
+func TestInstrumentedState_Claim(t *testing.T) {
+	testutil.RunVMTest_Claim(t, CreateInitialState, vmFactory, true)
 }

cannon/mipsevm/singlethreaded/mips.go

@@ -8,7 +8,7 @@ import (
 )
 
 func (m *InstrumentedState) handleSyscall() error {
-	syscallNum, a0, a1, a2 := exec.GetSyscallArgs(&m.state.Registers)
+	syscallNum, a0, a1, a2, _ := exec.GetSyscallArgs(&m.state.Registers)
 
 	v0 := uint32(0)
 	v1 := uint32(0)
@@ -20,7 +20,7 @@ func (m *InstrumentedState) handleSyscall() error {
 		v0, v1, newHeap = exec.HandleSysMmap(a0, a1, m.state.Heap)
 		m.state.Heap = newHeap
 	case exec.SysBrk:
-		v0 = 0x40000000
+		v0 = exec.BrkStart
 	case exec.SysClone: // clone (not supported)
 		v0 = 1
 	case exec.SysExitGroup:

cannon/mipsevm/singlethreaded/state.go

@@ -44,11 +44,13 @@ type State struct {
 	LastHint hexutil.Bytes `json:"lastHint,omitempty"`
 }
 
+var _ mipsevm.FPVMState = (*State)(nil)
+
 func CreateEmptyState() *State {
 	return &State{
 		Cpu: mipsevm.CpuScalars{
 			PC:     0,
-			NextPC: 0,
+			NextPC: 4,
 			LO:     0,
 			HI:     0,
 		},

cannon/mipsevm/tests/evm_test.go

@@ -2,7 +2,6 @@ package tests
 
 import (
 	"bytes"
-	"debug/elf"
 	"io"
 	"os"
 	"path"
@@ -17,7 +16,7 @@ import (
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/exec"
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/memory"
-	"github.com/ethereum-optimism/optimism/cannon/mipsevm/program"
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/multithreaded"
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/singlethreaded"
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/testutil"
 )
@@ -64,7 +63,7 @@ func TestEVM(t *testing.T) {
 			// set the return address ($ra) to jump into when test completes
 			state.Registers[31] = testutil.EndAddr
 
-			goState := singlethreaded.NewInstrumentedState(state, oracle, os.Stdout, os.Stderr)
+			goState := singlethreaded.NewInstrumentedState(state, oracle, os.Stdout, os.Stderr, nil)
 
 			for i := 0; i < 1000; i++ {
 				curStep := goState.GetState().GetStep()
@@ -101,6 +100,60 @@ func TestEVM(t *testing.T) {
 	}
 }
 
+func TestEVM_CloneFlags(t *testing.T) {
+	//contracts, addrs := testContractsSetup(t)
+	//var tracer vm.EVMLogger
+
+	cases := []struct {
+		name  string
+		flags uint32
+		valid bool
+	}{
+		{"the supported flags bitmask", exec.ValidCloneFlags, true},
+		{"no flags", 0, false},
+		{"all flags", ^uint32(0), false},
+		{"all unsupported flags", ^uint32(exec.ValidCloneFlags), false},
+		{"a few supported flags", exec.CloneFs | exec.CloneSysvsem, false},
+		{"one supported flag", exec.CloneFs, false},
+		{"mixed supported and unsupported flags", exec.CloneFs | exec.CloneParentSettid, false},
+		{"a single unsupported flag", exec.CloneUntraced, false},
+		{"multiple unsupported flags", exec.CloneUntraced | exec.CloneParentSettid, false},
+	}
+
+	const insn = uint32(0x00_00_00_0C) // syscall instruction
+	for _, tt := range cases {
+		t.Run(tt.name, func(t *testing.T) {
+			state := multithreaded.CreateEmptyState()
+			state.Memory.SetMemory(state.GetPC(), insn)
+			state.GetRegisters()[2] = exec.SysClone // Set syscall number
+			state.GetRegisters()[4] = tt.flags      // Set first argument
+			//curStep := state.Step
+
+			us := multithreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr, nil)
+			if !tt.valid {
+				// The VM should exit
+				_, err := us.Step(true)
+				require.NoError(t, err)
+				require.Equal(t, true, us.GetState().GetExited())
+				require.Equal(t, uint8(mipsevm.VMStatusPanic), us.GetState().GetExitCode())
+			} else {
+				/*stepWitness*/ _, err := us.Step(true)
+				require.NoError(t, err)
+			}
+
+			// TODO: Validate EVM execution once onchain implementation is ready
+			//evm := testutil.NewMIPSEVM(contracts, addrs)
+			//evm.SetTracer(tracer)
+			//testutil.LogStepFailureAtCleanup(t, evm)
+			//
+			//evmPost := evm.Step(t, stepWitness, curStep, singlethreaded.GetStateHashFn())
+			//goPost, _ := us.GetState().EncodeWitness()
+			//require.Equal(t, hexutil.Bytes(goPost).String(), hexutil.Bytes(evmPost).String(),
+			//	"mipsevm produced different state than EVM")
+		})
+	}
+}
+
 func TestEVMSingleStep(t *testing.T) {
 	contracts, addrs := testContractsSetup(t)
 	var tracer vm.EVMLogger
@@ -123,7 +176,7 @@ func TestEVMSingleStep(t *testing.T) {
 			state.Memory.SetMemory(tt.pc, tt.insn)
 			curStep := state.Step
 
-			us := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr)
+			us := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr, nil)
 			stepWitness, err := us.Step(true)
 			require.NoError(t, err)
 
@@ -303,7 +356,7 @@ func TestEVMSysWriteHint(t *testing.T) {
 			state.Memory.SetMemory(0, insn)
 			curStep := state.Step
 
-			us := singlethreaded.NewInstrumentedState(state, &oracle, os.Stdout, os.Stderr)
+			us := singlethreaded.NewInstrumentedState(state, &oracle, os.Stdout, os.Stderr, nil)
 			stepWitness, err := us.Step(true)
 			require.NoError(t, err)
 			require.Equal(t, tt.expectedHints, oracle.hints)
@@ -347,7 +400,7 @@ func TestEVMFault(t *testing.T) {
 			// set the return address ($ra) to jump into when test completes
 			state.Registers[31] = testutil.EndAddr
 
-			us := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr)
+			us := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr, nil)
 			require.Panics(t, func() { _, _ = us.Step(true) })
 
 			insnProof := initialState.Memory.MerkleProof(0)
@@ -374,18 +427,9 @@ func TestHelloEVM(t *testing.T) {
 	evm.SetTracer(tracer)
 	testutil.LogStepFailureAtCleanup(t, evm)
 
-	elfProgram, err := elf.Open("../../example/bin/hello.elf")
-	require.NoError(t, err, "open ELF file")
-
-	state, err := program.LoadELF(elfProgram, singlethreaded.CreateInitialState)
-	require.NoError(t, err, "load ELF into state")
-
-	err = program.PatchGo(elfProgram, state)
-	require.NoError(t, err, "apply Go runtime patches")
-	require.NoError(t, program.PatchStack(state), "add initial stack")
-
+	state := testutil.LoadELFProgram(t, "../../example/bin/hello.elf", singlethreaded.CreateInitialState, true)
 	var stdOutBuf, stdErrBuf bytes.Buffer
-	goState := singlethreaded.NewInstrumentedState(state, nil, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr))
+	goState := singlethreaded.NewInstrumentedState(state, nil, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr), nil)
 
 	start := time.Now()
 	for i := 0; i < 400_000; i++ {
@@ -425,20 +469,11 @@ func TestClaimEVM(t *testing.T) {
 	evm.SetTracer(tracer)
 	testutil.LogStepFailureAtCleanup(t, evm)
 
-	elfProgram, err := elf.Open("../../example/bin/claim.elf")
-	require.NoError(t, err, "open ELF file")
-
-	state, err := program.LoadELF(elfProgram, singlethreaded.CreateInitialState)
-	require.NoError(t, err, "load ELF into state")
-
-	err = program.PatchGo(elfProgram, state)
-	require.NoError(t, err, "apply Go runtime patches")
-	require.NoError(t, program.PatchStack(state), "add initial stack")
-
+	state := testutil.LoadELFProgram(t, "../../example/bin/claim.elf", singlethreaded.CreateInitialState, true)
 	oracle, expectedStdOut, expectedStdErr := testutil.ClaimTestOracle(t)
 
 	var stdOutBuf, stdErrBuf bytes.Buffer
-	goState := singlethreaded.NewInstrumentedState(state, oracle, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr))
+	goState := singlethreaded.NewInstrumentedState(state, oracle, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr), nil)
 
 	for i := 0; i < 2000_000; i++ {
 		curStep := goState.GetState().GetStep()

cannon/mipsevm/tests/fuzz_evm_test.go

@@ -47,7 +47,7 @@ func FuzzStateSyscallBrk(f *testing.F) {
 		expectedRegisters := state.Registers
 		expectedRegisters[2] = 0x4000_0000
 
-		goState := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr)
+		goState := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr, nil)
 		stepWitness, err := goState.Step(true)
 		require.NoError(t, err)
 		require.False(t, stepWitness.HasPreimage())
@@ -98,7 +98,7 @@ func FuzzStateSyscallClone(f *testing.F) {
 		expectedRegisters := state.Registers
 		expectedRegisters[2] = 0x1
 
-		goState := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr)
+		goState := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr, nil)
 		stepWitness, err := goState.Step(true)
 		require.NoError(t, err)
 		require.False(t, stepWitness.HasPreimage())
@@ -147,7 +147,7 @@ func FuzzStateSyscallMmap(f *testing.F) {
 		preStateRoot := state.Memory.MerkleRoot()
 		preStateRegisters := state.Registers
 
-		goState := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr)
+		goState := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr, nil)
 		stepWitness, err := goState.Step(true)
 		require.NoError(t, err)
 		require.False(t, stepWitness.HasPreimage())
@@ -210,7 +210,7 @@ func FuzzStateSyscallExitGroup(f *testing.F) {
 		preStateRoot := state.Memory.MerkleRoot()
 		preStateRegisters := state.Registers
 
-		goState := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr)
+		goState := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr, nil)
 		stepWitness, err := goState.Step(true)
 		require.NoError(t, err)
 		require.False(t, stepWitness.HasPreimage())
@@ -259,7 +259,7 @@ func FuzzStateSyscallFcntl(f *testing.F) {
 		preStateRoot := state.Memory.MerkleRoot()
 		preStateRegisters := state.Registers
 
-		goState := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr)
+		goState := singlethreaded.NewInstrumentedState(state, nil, os.Stdout, os.Stderr, nil)
 		stepWitness, err := goState.Step(true)
 		require.NoError(t, err)
 		require.False(t, stepWitness.HasPreimage())
@@ -330,7 +330,7 @@ func FuzzStateHintRead(f *testing.F) {
 		expectedRegisters[2] = count
 
 		oracle := testutil.StaticOracle(t, preimageData) // only used for hinting
-		goState := singlethreaded.NewInstrumentedState(state, oracle, os.Stdout, os.Stderr)
+		goState := singlethreaded.NewInstrumentedState(state, oracle, os.Stdout, os.Stderr, nil)
 		stepWitness, err := goState.Step(true)
 		require.NoError(t, err)
 		require.False(t, stepWitness.HasPreimage())
@@ -391,7 +391,7 @@ func FuzzStatePreimageRead(f *testing.F) {
 		}
 		oracle := testutil.StaticOracle(t, preimageData)
 
-		goState := singlethreaded.NewInstrumentedState(state, oracle, os.Stdout, os.Stderr)
+		goState := singlethreaded.NewInstrumentedState(state, oracle, os.Stdout, os.Stderr, nil)
 		stepWitness, err := goState.Step(true)
 		require.NoError(t, err)
 		require.True(t, stepWitness.HasPreimage())
@@ -458,7 +458,7 @@ func FuzzStateHintWrite(f *testing.F) {
 		expectedRegisters[2] = count
 
 		oracle := testutil.StaticOracle(t, preimageData) // only used for hinting
-		goState := singlethreaded.NewInstrumentedState(state, oracle, os.Stdout, os.Stderr)
+		goState := singlethreaded.NewInstrumentedState(state, oracle, os.Stdout, os.Stderr, nil)
 		stepWitness, err := goState.Step(true)
 		require.NoError(t, err)
 		require.False(t, stepWitness.HasPreimage())
@@ -514,7 +514,7 @@ func FuzzStatePreimageWrite(f *testing.F) {
 		expectedRegisters[2] = count
 
 		oracle := testutil.StaticOracle(t, preimageData)
-		goState := singlethreaded.NewInstrumentedState(state, oracle, os.Stdout, os.Stderr)
+		goState := singlethreaded.NewInstrumentedState(state, oracle, os.Stdout, os.Stderr, nil)
 		stepWitness, err := goState.Step(true)
 		require.NoError(t, err)
 		require.False(t, stepWitness.HasPreimage())

cannon/mipsevm/testutil/elf.go

@@ -10,15 +10,18 @@ import (
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/program"
 )
 
-func LoadELFProgram[T mipsevm.FPVMState](t *testing.T, name string, initState program.CreateFPVMState[T]) T {
+func LoadELFProgram[T mipsevm.FPVMState](t *testing.T, name string, initState program.CreateInitialFPVMState[T], doPatchGo bool) T {
 	elfProgram, err := elf.Open(name)
 	require.NoError(t, err, "open ELF file")
 
 	state, err := program.LoadELF(elfProgram, initState)
 	require.NoError(t, err, "load ELF into state")
 
-	err = program.PatchGo(elfProgram, state)
-	require.NoError(t, err, "apply Go runtime patches")
+	if doPatchGo {
+		err = program.PatchGo(elfProgram, state)
+		require.NoError(t, err, "apply Go runtime patches")
+	}
+
 	require.NoError(t, program.PatchStack(state), "add initial stack")
 	return state
 }

cannon/mipsevm/testutil/log.go

+package testutil
+
+import (
+	"os"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+func CreateLogger() log.Logger {
+	return log.NewLogger(log.LogfmtHandlerWithLevel(os.Stdout, log.LevelInfo))
+}

cannon/mipsevm/testutil/vmtests.go

+package testutil
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/program"
+)
+
+type VMFactory[T mipsevm.FPVMState] func(state T, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer, log log.Logger) mipsevm.FPVM
+type StateFactory[T mipsevm.FPVMState] func() T
+
+func RunVMTests_OpenMips[T mipsevm.FPVMState](t *testing.T, stateFactory StateFactory[T], vmFactory VMFactory[T], excludedTests ...string) {
+	testFiles, err := os.ReadDir("../tests/open_mips_tests/test/bin")
+	require.NoError(t, err)
+
+	for _, f := range testFiles {
+		t.Run(f.Name(), func(t *testing.T) {
+			for _, skipped := range excludedTests {
+				if f.Name() == skipped {
+					t.Skipf("Skipping explicitly excluded open_mips testcase: %v", f.Name())
+				}
+			}
+
+			oracle := SelectOracleFixture(t, f.Name())
+			// Short-circuit early for exit_group.bin
+			exitGroup := f.Name() == "exit_group.bin"
+
+			// TODO: currently tests are compiled as flat binary objects
+			// We can use more standard tooling to compile them to ELF files and get remove maketests.py
+			fn := path.Join("../tests/open_mips_tests/test/bin", f.Name())
+			//elfProgram, err := elf.Open()
+			//require.NoError(t, err, "must load test ELF binary")
+			//state, err := LoadELF(elfProgram)
+			//require.NoError(t, err, "must load ELF into state")
+			programMem, err := os.ReadFile(fn)
+			require.NoError(t, err)
+			state := stateFactory()
+			err = state.GetMemory().SetMemoryRange(0, bytes.NewReader(programMem))
+			require.NoError(t, err, "load program into state")
+
+			// set the return address ($ra) to jump into when test completes
+			state.GetRegisters()[31] = EndAddr
+
+			us := vmFactory(state, oracle, os.Stdout, os.Stderr, CreateLogger())
+
+			for i := 0; i < 1000; i++ {
+				if us.GetState().GetPC() == EndAddr {
+					break
+				}
+				if exitGroup && us.GetState().GetExited() {
+					break
+				}
+				_, err := us.Step(false)
+				require.NoError(t, err)
+			}
+
+			if exitGroup {
+				require.NotEqual(t, uint32(EndAddr), us.GetState().GetPC(), "must not reach end")
+				require.True(t, us.GetState().GetExited(), "must set exited state")
+				require.Equal(t, uint8(1), us.GetState().GetExitCode(), "must exit with 1")
+			} else {
+				require.Equal(t, uint32(EndAddr), us.GetState().GetPC(), "must reach end")
+				done, result := state.GetMemory().GetMemory(BaseAddrEnd+4), state.GetMemory().GetMemory(BaseAddrEnd+8)
+				// inspect test result
+				require.Equal(t, done, uint32(1), "must be done")
+				require.Equal(t, result, uint32(1), "must have success result")
+			}
+		})
+	}
+}
+
+func RunVMTest_Hello[T mipsevm.FPVMState](t *testing.T, initState program.CreateInitialFPVMState[T], vmFactory VMFactory[T], doPatchGo bool) {
+	state := LoadELFProgram(t, "../../example/bin/hello.elf", initState, doPatchGo)
+
+	var stdOutBuf, stdErrBuf bytes.Buffer
+	us := vmFactory(state, nil, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr), CreateLogger())
+
+	for i := 0; i < 400_000; i++ {
+		if us.GetState().GetExited() {
+			break
+		}
+		_, err := us.Step(false)
+		require.NoError(t, err)
+	}
+
+	require.True(t, state.GetExited(), "must complete program")
+	require.Equal(t, uint8(0), state.GetExitCode(), "exit with 0")
+
+	require.Equal(t, "hello world!\n", stdOutBuf.String(), "stdout says hello")
+	require.Equal(t, "", stdErrBuf.String(), "stderr silent")
+}
+
+func RunVMTest_Claim[T mipsevm.FPVMState](t *testing.T, initState program.CreateInitialFPVMState[T], vmFactory VMFactory[T], doPatchGo bool) {
+	state := LoadELFProgram(t, "../../example/bin/claim.elf", initState, doPatchGo)
+
+	oracle, expectedStdOut, expectedStdErr := ClaimTestOracle(t)
+
+	var stdOutBuf, stdErrBuf bytes.Buffer
+	us := vmFactory(state, oracle, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr), CreateLogger())
+
+	for i := 0; i < 2000_000; i++ {
+		if us.GetState().GetExited() {
+			break
+		}
+		_, err := us.Step(false)
+		require.NoError(t, err)
+	}
+
+	require.True(t, state.GetExited(), "must complete program")
+	require.Equal(t, uint8(0), state.GetExitCode(), "exit with 0")
+
+	require.Equal(t, expectedStdOut, stdOutBuf.String(), "stdout")
+	require.Equal(t, expectedStdErr, stdErrBuf.String(), "stderr")
+}

op-program/client/cmd/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"os"
+	"runtime"
 
 	"github.com/ethereum/go-ethereum/log"
 
@@ -9,6 +10,13 @@ import (
 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
 )
 
+var _ = func() bool {
+	// Disable mem profiling, to avoid a lot of unnecessary floating point ops
+	// TODO(cp-903) Consider cutting this in favor patching envar "GODEBUG=memprofilerate=0" into cannon go programs
+	runtime.MemProfileRate = 0
+	return true
+}()
+
 func main() {
 	// Default to a machine parsable but relatively human friendly log format.
 	// Don't do anything fancy to detect if color output is supported.