Merge branch 'gomips'

.github/workflows/test.yml

@@ -16,19 +16,20 @@ jobs:
         with:
           submodules: true
       - name: unicorn commit hash
+        working-directory: ./diffmips/unicorn
         run: |
           git rev-parse HEAD > /tmp/unicorn-commit-hash.txt
       - name: cached libunicorn
         uses: actions/cache@v3
         with:
           path: |
-            ./unicorn/build
+            ./diffmips/unicorn/build
           key:
             unicorn-build-{{ hashFiles('/tmp/unicorn-commit-hash.txt') }}
           restore-keys: |
             unicorn-build-{{ hashFiles('/tmp/unicorn-commit-hash.txt') }}
       - name: install libunicorn
-        working-directory: .
+        working-directory: ./diffmips
         run: make libunicorn
       - uses: actions/cache@v3
         with:
@@ -45,11 +46,17 @@ jobs:
           key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-${{ matrix.go-version }}-
-      - name: golangci-lint
+      - name: main golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
           version: v1.52
-          working-directory: mipsevm
+          working-directory: ./
+          skip-cache: true  # we already have go caching
+      - name: unicorntest golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.52
+          working-directory: ./diffmips/unicorntest
           skip-cache: true  # we already have go caching
       - name: Build examples
         working-directory: ./example
@@ -60,3 +67,6 @@ jobs:
       - name: mipsevm tests
         working-directory: ./mipsevm
         run: go test ./...
+      - name: diffmips tests
+        working-directory: ./diffmips/unicorntest
+        run: go test ./...

.gitignore

@@ -8,3 +8,4 @@ venv
 example/bin
 contracts/out
 state.json
+*.json

.gitmodules

 [submodule "unicorn"]
-	path = unicorn
+	path = diffmips/unicorn
 	url = https://github.com/unicorn-engine/unicorn.git

Makefile

 SHELL := /bin/bash
 
-build: submodules libunicorn contracts
+build: contracts
 .PHONY: build
 
-submodules:
-	# CI will checkout submodules on its own (and fails on these commands)
-	if [[ -z "$$GITHUB_ENV" ]]; then \
-		git submodule init; \
-		git submodule update; \
-	fi
-.PHONY: submodules
-
-# Approximation, use `make libunicorn_rebuild` to force.
-unicorn/build: unicorn/CMakeLists.txt
-	mkdir -p unicorn/build
-	cd unicorn/build && cmake .. -DUNICORN_ARCH=mips -DCMAKE_BUILD_TYPE=Release
-	# Not sure why, but the second invocation is needed for fresh installs on MacOS.
-	if [ "$(shell uname)" == "Darwin" ]; then \
-		cd unicorn/build && cmake .. -DUNICORN_ARCH=mips -DCMAKE_BUILD_TYPE=Release; \
-	fi
-
-# Rebuild whenever anything in the unicorn/ directory changes.
-unicorn/build/libunicorn.so: unicorn/build unicorn
-	cd unicorn/build && make -j8
-	# The Go linker / runtime expects dynamic libraries in the unicorn/ dir.
-	find ./unicorn/build -name "libunicorn.*" | xargs -L 1 -I {} cp {} ./unicorn/
-	# Update timestamp on libunicorn.so to make it more recent than the build/ dir.
-	# On Mac this will create a new empty file (dyn libraries are .dylib), but works
-	# fine for the purpose of avoiding recompilation.
-	touch unicorn/build/libunicorn.so
-
-libunicorn: unicorn/build/libunicorn.so
-.PHONY: libunicorn
-
-libunicorn_rebuild:
-	touch unicorn/CMakeLists.txt
-	make libunicorn
-.PHONY: libunicorn_rebuild
-
-# Must be a definition and not a rule, otherwise it gets only called once and
-# not before each test as we wish.
-define clear_cache
-	rm -rf /tmp/cannon
-	mkdir -p /tmp/cannon
-endef
-
-clear_cache:
-	$(call clear_cache)
-.PHONY: clear_cache
-
-clean:
-	rm -f unicorn/libunicorn.*
-.PHONY: clean
-
 contracts:
 	cd contracts && forge build
 .PHONY: contracts
 
-test: libunicorn
+test:
 	cd mipsevm && go test -v ./...
 .PHONY: test

README.md

@@ -24,18 +24,11 @@ contracts -- A MIPS emulator implementation, using merkleized state and a pre-im
 example   -- Example programs that can be run and proven with Cannon.
 extra     -- Extra scripts and legacy contracts, deprecated.
 mipsevm   -- Go tooling to test the onchain MIPS implementation, and generate proof data.
-unicorn   -- Sub-module, used by mipsevm for offchain MIPS emulation.
+diffmips  -- MIPS diff testing, to ensure correctness of the main Cannon implementation, with isolated dependencies. 
 ```
 
 ## Building
 
-### `unicorn`
-
-To build unicorn from source (git sub-module), run:
-```
-make libunicorn
-```
-
 ### `contracts`
 
 The contracts are compiled with [`forge`](https://github.com/foundry-rs/foundry).
@@ -43,17 +36,6 @@ The contracts are compiled with [`forge`](https://github.com/foundry-rs/foundry)
 make contracts
 ```
 
-### `mipsevm`
-
-This requires `unicorn` to be built, as well as the `contracts` for testing.
-
-To test:
-```
-make test
-```
-
-Also see notes in `mipsevm/go.mod` about the Unicorn dependency, if you wish to use Cannon as a Go library.
-
 ## License
 
 MIT, see [`LICENSE`](./LICENSE) file.

cmd/json.go

@@ -2,12 +2,16 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
 )
 
 func loadJSON[X any](inputPath string) (*X, error) {
+	if inputPath == "" {
+		return nil, errors.New("no path specified")
+	}
 	f, err := os.OpenFile(inputPath, os.O_RDONLY, 0)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open file %q: %w", inputPath, err)

cmd/load_elf.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/urfave/cli/v2"
 
-	"cannon/mipsevm"
+	"github.com/ethereum-optimism/cannon/mipsevm"
 )
 
 var (
@@ -28,6 +28,12 @@ var (
 		Value:    "state.json",
 		Required: false,
 	}
+	LoadELFMetaFlag = &cli.PathFlag{
+		Name:     "meta",
+		Usage:    "Write metadata file, for symbol lookup during program execution. None if empty.",
+		Value:    "meta.json",
+		Required: false,
+	}
 )
 
 func LoadELF(ctx *cli.Context) error {
@@ -36,6 +42,9 @@ func LoadELF(ctx *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to open ELF file %q: %w", elfPath, err)
 	}
+	if elfProgram.Machine != elf.EM_MIPS {
+		return fmt.Errorf("ELF is not big-endian MIPS R3000, but got %q", elfProgram.Machine.String())
+	}
 	state, err := mipsevm.LoadELF(elfProgram)
 	if err != nil {
 		return fmt.Errorf("failed to load ELF data into VM state: %w", err)
@@ -53,6 +62,13 @@ func LoadELF(ctx *cli.Context) error {
 			return fmt.Errorf("failed to apply patch %s: %w", typ, err)
 		}
 	}
+	meta, err := mipsevm.MakeMetadata(elfProgram)
+	if err != nil {
+		return fmt.Errorf("failed to compute program metadata: %w", err)
+	}
+	if err := writeJSON[*mipsevm.Metadata](ctx.Path(LoadELFMetaFlag.Name), meta, false); err != nil {
+		return fmt.Errorf("failed to output metadata: %w", err)
+	}
 	return writeJSON[*mipsevm.State](ctx.Path(LoadELFOutFlag.Name), state, true)
 }
 
@@ -65,5 +81,6 @@ var LoadELFCommand = &cli.Command{
 		LoadELFPathFlag,
 		LoadELFPatchFlag,
 		LoadELFOutFlag,
+		LoadELFMetaFlag,
 	},
 }

cmd/matcher.go

@@ -5,7 +5,7 @@ import (
 	"strconv"
 	"strings"
 
-	"cannon/mipsevm"
+	"github.com/ethereum-optimism/cannon/mipsevm"
 )
 
 type StepMatcher func(st *mipsevm.State) bool
@@ -15,6 +15,14 @@ type StepMatcherFlag struct {
 	matcher StepMatcher
 }
 
+func MustStepMatcherFlag(pattern string) *StepMatcherFlag {
+	out := new(StepMatcherFlag)
+	if err := out.Set(pattern); err != nil {
+		panic(err)
+	}
+	return out
+}
+
 func (m *StepMatcherFlag) Set(value string) error {
 	m.repr = value
 	if value == "" || value == "never" {

cmd/run.go

@@ -12,7 +12,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/urfave/cli/v2"
 
-	"cannon/mipsevm"
+	"github.com/ethereum-optimism/cannon/mipsevm"
 	"github.com/ethereum-optimism/cannon/preimage"
 )
 
@@ -62,6 +62,18 @@ var (
 		Value:    new(StepMatcherFlag),
 		Required: false,
 	}
+	RunMetaFlag = &cli.PathFlag{
+		Name:     "meta",
+		Usage:    "path to metadata file for symbol lookup for enhanced debugging info durign execution.",
+		Value:    "meta.json",
+		Required: false,
+	}
+	RunInfoAtFlag = &cli.GenericFlag{
+		Name:     "info-at",
+		Usage:    "step pattern to print info at: " + patternHelp,
+		Value:    MustStepMatcherFlag("%1000"),
+		Required: false,
+	}
 )
 
 type Proof struct {
@@ -170,13 +182,7 @@ func Run(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	mu, err := mipsevm.NewUnicorn()
-	if err != nil {
-		return fmt.Errorf("failed to create unicorn emulator: %w", err)
-	}
-	if err := mipsevm.LoadUnicorn(state, mu); err != nil {
-		return fmt.Errorf("failed to load state into unicorn emulator: %w", err)
-	}
+
 	l := Logger(os.Stderr, log.LvlInfo)
 	outLog := &mipsevm.LoggingWriter{Name: "program std-out", Log: l}
 	errLog := &mipsevm.LoggingWriter{Name: "program std-err", Log: l}
@@ -206,11 +212,21 @@ func Run(ctx *cli.Context) error {
 	stopAt := ctx.Generic(RunStopAtFlag.Name).(*StepMatcherFlag).Matcher()
 	proofAt := ctx.Generic(RunProofAtFlag.Name).(*StepMatcherFlag).Matcher()
 	snapshotAt := ctx.Generic(RunSnapshotAtFlag.Name).(*StepMatcherFlag).Matcher()
-
-	us, err := mipsevm.NewUnicornState(mu, state, po, outLog, errLog)
-	if err != nil {
-		return fmt.Errorf("failed to setup instrumented VM state: %w", err)
+	infoAt := ctx.Generic(RunInfoAtFlag.Name).(*StepMatcherFlag).Matcher()
+
+	var meta *mipsevm.Metadata
+	if metaPath := ctx.Path(RunMetaFlag.Name); metaPath == "" {
+		l.Info("no metadata file specified, defaulting to empty metadata")
+		meta = &mipsevm.Metadata{Symbols: nil} // provide empty metadata by default
+	} else {
+		if m, err := loadJSON[mipsevm.Metadata](metaPath); err != nil {
+			return fmt.Errorf("failed to load metadata: %w", err)
+		} else {
+			meta = m
+		}
 	}
+
+	us := mipsevm.NewInstrumentedState(state, po, outLog, errLog)
 	proofFmt := ctx.String(RunProofFmtFlag.Name)
 	snapshotFmt := ctx.String(RunSnapshotFmtFlag.Name)
 
@@ -222,12 +238,18 @@ func Run(ctx *cli.Context) error {
 	for !state.Exited {
 		step := state.Step
 
-		//if infoAt(state) {
-		//	s := lookupSymbol(state.PC)
-		//	var sy elf.Symbol
-		//	l.Info("", "insn", state.Memory.GetMemory(state.PC), "pc", state.PC, "symbol", sy.Name)
-		//	// print name
-		//}
+		name := meta.LookupSymbol(state.PC)
+		if infoAt(state) {
+			l.Info("processing",
+				"step", step,
+				"pc", mipsevm.HexU32(state.PC),
+				"insn", mipsevm.HexU32(state.Memory.GetMemory(state.PC)),
+				"name", name,
+			)
+		}
+		if name == "runtime.notesleep" { // don't loop forever when we get stuck because of an unexpected bad program
+			return fmt.Errorf("got stuck in Go sleep at step %d", step)
+		}
 
 		if stopAt(state) {
 			break
@@ -262,7 +284,7 @@ func Run(ctx *cli.Context) error {
 				return fmt.Errorf("failed to write proof data: %w", err)
 			}
 		} else {
-			_, err = us.Step(false)
+			_, err = stepFn(false)
 			if err != nil {
 				return fmt.Errorf("failed at step %d (PC: %08x): %w", step, state.PC, err)
 			}
@@ -288,5 +310,7 @@ var RunCommand = &cli.Command{
 		RunSnapshotAtFlag,
 		RunSnapshotFmtFlag,
 		RunStopAtFlag,
+		RunMetaFlag,
+		RunInfoAtFlag,
 	},
 }

diffmips/Makefile

+SHELL := /bin/bash
+
+build: submodules libunicorn
+.PHONY: build
+
+submodules:
+	# CI will checkout submodules on its own (and fails on these commands)
+	if [[ -z "$$GITHUB_ENV" ]]; then \
+		git submodule init; \
+		git submodule update; \
+	fi
+.PHONY: submodules
+
+# Approximation, use `make libunicorn_rebuild` to force.
+unicorn/build: unicorn/CMakeLists.txt
+	mkdir -p unicorn/build
+	cd unicorn/build && cmake .. -DUNICORN_ARCH=mips -DCMAKE_BUILD_TYPE=Release
+	# Not sure why, but the second invocation is needed for fresh installs on MacOS.
+	if [ "$(shell uname)" == "Darwin" ]; then \
+		cd unicorn/build && cmake .. -DUNICORN_ARCH=mips -DCMAKE_BUILD_TYPE=Release; \
+	fi
+
+# Rebuild whenever anything in the unicorn/ directory changes.
+unicorn/build/libunicorn.so: unicorn/build unicorn
+	cd unicorn/build && make -j8
+	# The Go linker / runtime expects dynamic libraries in the unicorn/ dir.
+	find ./unicorn/build -name "libunicorn.*" | xargs -L 1 -I {} cp {} ./unicorn/
+	# Update timestamp on libunicorn.so to make it more recent than the build/ dir.
+	# On Mac this will create a new empty file (dyn libraries are .dylib), but works
+	# fine for the purpose of avoiding recompilation.
+	touch unicorn/build/libunicorn.so
+
+libunicorn: unicorn/build/libunicorn.so
+.PHONY: libunicorn
+
+libunicorn_rebuild:
+	touch unicorn/CMakeLists.txt
+	make libunicorn
+.PHONY: libunicorn_rebuild
+
+clean:
+	rm -f unicorn/libunicorn.*
+.PHONY: clean
+
+test:
+	cd unicorntest && go test -v ./...
+.PHONY: test

diffmips/README.md

+# diffmips
+
+This is a collection of MIPS testing tools.
+The Unicorn emulator first backed Cannon directly, but has been separated as a testing-only tool,
+and is replaced with a Go implementation of the minimal MIPS functionality.
+
+Directory layout
+```
+unicorn     -- Sub-module, used by mipsevm for offchain MIPS emulation.
+unicorntest -- Go module with Go tests, diffed against the Cannon state. [Work in Progress]
+```
+
+### `unicorn`
+
+To build unicorn from source (git sub-module), run:
+```
+make libunicorn
+```
+
+### `unicorntest`
+
+This requires `unicorn` to be built, as well as the `contracts` for testing.
+
+To test:
+```
+make test
+```
+

diffmips/unicorntest/diff_test.go

+package unicorntest
+
+import (
+	"bytes"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum-optimism/cannon/mipsevm"
+)
+
+// baseAddrStart - baseAddrEnd is used in tests to write the results to
+const baseAddrEnd = 0xbf_ff_ff_f0
+const baseAddrStart = 0xbf_c0_00_00
+
+// endAddr is used as return-address for tests
+const endAddr = 0xa7ef00d0
+
+func TestState(t *testing.T) {
+	testFiles, err := os.ReadDir("../../mipsevm/open_mips_tests/test/bin")
+	require.NoError(t, err)
+
+	for _, f := range testFiles {
+		t.Run(f.Name(), func(t *testing.T) {
+			if f.Name() == "oracle.bin" {
+				t.Skip("oracle test needs to be updated to use syscall pre-image oracle")
+			}
+			fn := path.Join("../../mipsevm/open_mips_tests/test/bin", f.Name())
+
+			programMem, err := os.ReadFile(fn)
+			require.NoError(t, err)
+			state := &mipsevm.State{PC: 0, NextPC: 4, Memory: mipsevm.NewMemory()}
+			err = state.Memory.SetMemoryRange(0, bytes.NewReader(programMem))
+			require.NoError(t, err, "load program into state")
+
+			// set the return address ($ra) to jump into when test completes
+			state.Registers[31] = endAddr
+
+			mu, err := NewUnicorn()
+			require.NoError(t, err, "load unicorn")
+			defer mu.Close()
+
+			require.NoError(t, mu.MemMap(baseAddrStart, ((baseAddrEnd-baseAddrStart)&^mipsevm.PageAddrMask)+mipsevm.PageSize))
+			require.NoError(t, mu.MemMap(endAddr&^mipsevm.PageAddrMask, mipsevm.PageSize))
+
+			err = LoadUnicorn(state, mu)
+			require.NoError(t, err, "load state into unicorn")
+
+			us, err := NewUnicornState(mu, state, nil, os.Stdout, os.Stderr)
+			require.NoError(t, err, "hook unicorn to state")
+
+			for i := 0; i < 1000; i++ {
+				if us.state.PC == endAddr {
+					break
+				}
+				_, err := us.Step(false)
+				require.NoError(t, err)
+			}
+			require.Equal(t, uint32(endAddr), us.state.PC, "must reach end")
+			// inspect test result
+			done, result := state.Memory.GetMemory(baseAddrEnd+4), state.Memory.GetMemory(baseAddrEnd+8)
+			require.Equal(t, done, uint32(1), "must be done")
+			require.Equal(t, result, uint32(1), "must have success result")
+		})
+	}
+}

diffmips/unicorntest/go.mod

+module unicorntest
+
+go 1.20
+
+require (
+	github.com/ethereum-optimism/cannon v0.0.0
+	github.com/stretchr/testify v1.8.2
+	github.com/unicorn-engine/unicorn v0.0.0-20230207094436-7b8c63dfe650
+)
+
+require (
+	github.com/DataDog/zstd v1.5.2 // indirect
+	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
+	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cockroachdb/errors v1.9.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 // indirect
+	github.com/cockroachdb/redact v1.1.3 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/edsrzf/mmap-go v1.0.0 // indirect
+	github.com/ethereum-optimism/cannon/preimage v0.0.0 // indirect
+	github.com/ethereum/go-ethereum v1.11.5 // indirect
+	github.com/getsentry/sentry-go v0.18.0 // indirect
+	github.com/go-ole/go-ole v1.2.1 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
+	github.com/holiman/uint256 v1.2.0 // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.39.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
+	github.com/tklauser/go-sysconf v0.3.5 // indirect
+	github.com/tklauser/numcpus v0.2.2 // indirect
+	golang.org/x/crypto v0.8.0 // indirect
+	golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
+	golang.org/x/sys v0.7.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+replace github.com/ethereum-optimism/cannon v0.0.0 => ../../
+
+replace github.com/ethereum-optimism/cannon/preimage v0.0.0 => ../../preimage
+
+// We need to point to our local Unicorn clone for the shared object to be located correctly in all our Go commands.
+// See https://github.com/unicorn-engine/unicorn/blob/7b8c63dfe650b5d4d2bf684526161971925e6350/bindings/go/unicorn/unicorn.go#L11
+// The -L../../../ -lunicorn   points to the unicorn root directory relative to the unicorn Go bindings.
+// Run make libunicorn in the Cannon repo root to create this libunicorn.so dependency.
+//
+// If you are importing this as a library you will need to also point it to a unicorn clone with a `replace`
+// in your `go.mod` (or `go.work`), or use `go build -ldflags="-L../path/to/my/unicorn/build -lunicorn`,
+// or simply have it installed globally so Go can find it. The `replace` here will be ignored as library-user.
+replace github.com/unicorn-engine/unicorn v0.0.0-20230207094436-7b8c63dfe650 => ../unicorn

mipsevm/unicorn.go → diffmips/unicorntest/unicorn.go

-package mipsevm
+package unicorntest
 
 import (
 	"encoding/binary"
@@ -6,7 +6,8 @@ import (
 	"io"
 	"log"
 	"math"
-	"sync"
+
+	"github.com/ethereum-optimism/cannon/mipsevm"
 
 	uc "github.com/unicorn-engine/unicorn/bindings/go/unicorn"
 )
@@ -17,11 +18,9 @@ type PreimageOracle interface {
 }
 
 type UnicornState struct {
-	sync.Mutex
-
 	mu uc.Unicorn
 
-	state *State
+	state *mipsevm.State
 
 	stdOut io.Writer
 	stdErr io.Writer
@@ -55,7 +54,7 @@ const (
 	MipsEINVAL = 0x16
 )
 
-func NewUnicornState(mu uc.Unicorn, state *State, po PreimageOracle, stdOut, stdErr io.Writer) (*UnicornState, error) {
+func NewUnicornState(mu uc.Unicorn, state *mipsevm.State, po PreimageOracle, stdOut, stdErr io.Writer) (*UnicornState, error) {
 	m := &UnicornState{
 		mu:             mu,
 		state:          state,
@@ -109,8 +108,8 @@ func NewUnicornState(mu uc.Unicorn, state *State, po PreimageOracle, stdOut, std
 		switch syscallNum {
 		case 4090: // mmap
 			sz := a1
-			if sz&pageAddrMask != 0 { // adjust size to align with page size
-				sz += pageSize - (sz & pageAddrMask)
+			if sz&mipsevm.PageAddrMask != 0 { // adjust size to align with page size
+				sz += mipsevm.PageSize - (sz & mipsevm.PageAddrMask)
 			}
 			if a0 == 0 {
 				v0 = st.Heap
@@ -305,7 +304,7 @@ func NewUnicornState(mu uc.Unicorn, state *State, po PreimageOracle, stdOut, std
 	return m, nil
 }
 
-func (m *UnicornState) Step(proof bool) (wit *StepWitness, err error) {
+func (m *UnicornState) Step(proof bool) (wit *mipsevm.StepWitness, err error) {
 	defer func() { // pre-image oracle or emulator hooks might panic
 		if a := recover(); a != nil {
 			if ae, ok := a.(error); ok {
@@ -322,9 +321,9 @@ func (m *UnicornState) Step(proof bool) (wit *StepWitness, err error) {
 
 	if proof {
 		insnProof := m.state.Memory.MerkleProof(m.state.PC)
-		wit = &StepWitness{
-			state:    m.state.EncodeWitness(),
-			memProof: insnProof[:],
+		wit = &mipsevm.StepWitness{
+			State:    m.state.EncodeWitness(),
+			MemProof: insnProof[:],
 		}
 	}
 
@@ -377,11 +376,11 @@ func (m *UnicornState) Step(proof bool) (wit *StepWitness, err error) {
 	}
 
 	if proof {
-		wit.memProof = append(wit.memProof, m.memProof[:]...)
+		wit.MemProof = append(wit.MemProof, m.memProof[:]...)
 		if m.lastPreimageOffset != ^uint32(0) {
-			wit.preimageOffset = m.lastPreimageOffset
-			wit.preimageKey = m.lastPreimageKey
-			wit.preimageValue = m.lastPreimage
+			wit.PreimageOffset = m.lastPreimageOffset
+			wit.PreimageKey = m.lastPreimageKey
+			wit.PreimageValue = m.lastPreimage
 		}
 	}
 
@@ -421,11 +420,11 @@ func NewUnicorn() (uc.Unicorn, error) {
 	return uc.NewUnicorn(uc.ARCH_MIPS, uc.MODE_32|uc.MODE_BIG_ENDIAN)
 }
 
-func LoadUnicorn(st *State, mu uc.Unicorn) error {
+func LoadUnicorn(st *mipsevm.State, mu uc.Unicorn) error {
 	// mmap and write each page of memory state into unicorn
 	for pageIndex, page := range st.Memory.Pages {
-		addr := uint64(pageIndex) << pageAddrSize
-		if err := mu.MemMap(addr, pageSize); err != nil {
+		addr := uint64(pageIndex) << mipsevm.PageAddrSize
+		if err := mu.MemMap(addr, mipsevm.PageSize); err != nil {
 			return fmt.Errorf("failed to mmap page at addr 0x%x: %w", addr, err)
 		}
 		if err := mu.MemWrite(addr, page.Data[:]); err != nil {

mipsevm/unicorn_test.go → diffmips/unicorntest/unicorn_test.go

-package mipsevm
+package unicorntest
 
 import (
 	"testing"

go.mod

-module cannon
+module github.com/ethereum-optimism/cannon
 
 go 1.20
 
@@ -6,7 +6,6 @@ require (
 	github.com/ethereum-optimism/cannon/preimage v0.0.0
 	github.com/ethereum/go-ethereum v1.11.5
 	github.com/stretchr/testify v1.8.2
-	github.com/unicorn-engine/unicorn v0.0.0-20230207094436-7b8c63dfe650
 	github.com/urfave/cli/v2 v2.25.3
 )
 
@@ -65,13 +64,3 @@ require (
 )
 
 replace github.com/ethereum-optimism/cannon/preimage v0.0.0 => ./preimage
-
-// We need to point to our local Unicorn clone for the shared object to be located correctly in all our Go commands.
-// See https://github.com/unicorn-engine/unicorn/blob/7b8c63dfe650b5d4d2bf684526161971925e6350/bindings/go/unicorn/unicorn.go#L11
-// The -L../../../ -lunicorn   points to the unicorn root directory relative to the unicorn Go bindings.
-// Run make libunicorn in the Cannon repo root to create this libunicorn.so dependency.
-//
-// If you are importing Cannon as a library you will need to also point it to a unicorn clone with a `replace`
-// in your `go.mod` (or `go.work`), or use `go build -ldflags="-L../path/to/my/unicorn/build -lunicorn`,
-// or simply have it installed globally so Go can find it. The `replace` here will be ignored as library-user.
-replace github.com/unicorn-engine/unicorn v0.0.0-20230207094436-7b8c63dfe650 => ./unicorn

go.sum

@@ -275,8 +275,6 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa h1:5SqCsI/2Qya2bCzK15ozrqo2sZxkh0FHynJZOTVoV6Q=
-github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI=
 github.com/urfave/cli/v2 v2.25.3 h1:VJkt6wvEBOoSjPFQvOkv6iWIrsJyCrKGtCtxXWwmGeY=
 github.com/urfave/cli/v2 v2.25.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=

main.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/urfave/cli/v2"
 
-	"cannon/cmd"
+	"github.com/ethereum-optimism/cannon/cmd"
 )
 
 func main() {

mipsevm/README.md

@@ -16,9 +16,8 @@ Supported 55 instructions:
 To run:
 1. Load a program into a state, e.g. using `LoadELF`.
 2. Patch the program if necessary: e.g. using `PatchGo` for Go programs, `PatchStack` for empty initial stack, etc.
-3. Load the state into a MIPS-32 configured unicorn instance, using `NewUnicorn`, `LoadUnicorn`
 4. Implement the `PreimageOracle` interface
-5. Instrument the emulator with the state, and pre-image oracle, using `NewUnicornState`
+5. Instrument the emulator with the state, and pre-image oracle, using `NewInstrumentedState`
 6. Step through the instrumented state with `Step(proof)`,
    where `proof==true` if witness data should be generated. Steps are faster with `proof==false`.
 7. Optionally repeat the step on-chain by calling `MIPS.sol` and `Oracle.sol`, using the above witness data.

mipsevm/evm_test.go

@@ -64,18 +64,7 @@ func TestEVM(t *testing.T) {
 			// set the return address ($ra) to jump into when test completes
 			state.Registers[31] = endAddr
 
-			mu, err := NewUnicorn()
-			require.NoError(t, err, "load unicorn")
-			defer mu.Close()
-
-			require.NoError(t, mu.MemMap(baseAddrStart, ((baseAddrEnd-baseAddrStart)&^pageAddrMask)+pageSize))
-			require.NoError(t, mu.MemMap(endAddr&^pageAddrMask, pageSize))
-
-			err = LoadUnicorn(state, mu)
-			require.NoError(t, err, "load state into unicorn")
-
-			us, err := NewUnicornState(mu, state, nil, os.Stdout, os.Stderr)
-			require.NoError(t, err, "hook unicorn to state")
+			us := NewInstrumentedState(state, nil, os.Stdout, os.Stderr)
 
 			for i := 0; i < 1000; i++ {
 				if us.state.PC == endAddr {
@@ -133,14 +122,8 @@ func TestHelloEVM(t *testing.T) {
 	require.NoError(t, err, "apply Go runtime patches")
 	require.NoError(t, PatchStack(state), "add initial stack")
 
-	mu, err := NewUnicorn()
-	require.NoError(t, err, "load unicorn")
-	defer mu.Close()
-	err = LoadUnicorn(state, mu)
-	require.NoError(t, err, "load state into unicorn")
 	var stdOutBuf, stdErrBuf bytes.Buffer
-	us, err := NewUnicornState(mu, state, nil, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr))
-	require.NoError(t, err, "hook unicorn to state")
+	us := NewInstrumentedState(state, nil, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr))
 
 	env, evmState := NewEVMEnv(contracts, addrs)
 	env.Config.Debug = false
@@ -206,17 +189,10 @@ func TestClaimEVM(t *testing.T) {
 	require.NoError(t, err, "apply Go runtime patches")
 	require.NoError(t, PatchStack(state), "add initial stack")
 
-	mu, err := NewUnicorn()
-	require.NoError(t, err, "load unicorn")
-	defer mu.Close()
-	err = LoadUnicorn(state, mu)
-	require.NoError(t, err, "load state into unicorn")
-
 	oracle, expectedStdOut, expectedStdErr := claimTestOracle(t)
 
 	var stdOutBuf, stdErrBuf bytes.Buffer
-	us, err := NewUnicornState(mu, state, oracle, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr))
-	require.NoError(t, err, "hook unicorn to state")
+	us := NewInstrumentedState(state, oracle, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr))
 
 	env, evmState := NewEVMEnv(contracts, addrs)
 	env.Config.Debug = false

mipsevm/instrumented.go

+package mipsevm
+
+import (
+	"io"
+)
+
+type PreimageOracle interface {
+	Hint(v []byte)
+	GetPreimage(k [32]byte) []byte
+}
+
+type InstrumentedState struct {
+	state *State
+
+	stdOut io.Writer
+	stdErr io.Writer
+
+	lastMemAccess   uint32
+	memProofEnabled bool
+	memProof        [28 * 32]byte
+
+	preimageOracle PreimageOracle
+
+	// cached pre-image data, including 8 byte length prefix
+	lastPreimage []byte
+	// key for above preimage
+	lastPreimageKey [32]byte
+	// offset we last read from, or max uint32 if nothing is read this step
+	lastPreimageOffset uint32
+}
+
+const (
+	fdStdin         = 0
+	fdStdout        = 1
+	fdStderr        = 2
+	fdHintRead      = 3
+	fdHintWrite     = 4
+	fdPreimageRead  = 5
+	fdPreimageWrite = 6
+)
+
+const (
+	MipsEBADF  = 0x9
+	MipsEINVAL = 0x16
+)
+
+func NewInstrumentedState(state *State, po PreimageOracle, stdOut, stdErr io.Writer) *InstrumentedState {
+	return &InstrumentedState{
+		state:          state,
+		stdOut:         stdOut,
+		stdErr:         stdErr,
+		preimageOracle: po,
+	}
+}
+
+func (m *InstrumentedState) Step(proof bool) (wit *StepWitness, err error) {
+	m.memProofEnabled = proof
+	m.lastMemAccess = ^uint32(0)
+	m.lastPreimageOffset = ^uint32(0)
+
+	if proof {
+		insnProof := m.state.Memory.MerkleProof(m.state.PC)
+		wit = &StepWitness{
+			State:    m.state.EncodeWitness(),
+			MemProof: insnProof[:],
+		}
+	}
+	err = m.mipsStep()
+	if err != nil {
+		return nil, err
+	}
+
+	if proof {
+		wit.MemProof = append(wit.MemProof, m.memProof[:]...)
+		if m.lastPreimageOffset != ^uint32(0) {
+			wit.PreimageOffset = m.lastPreimageOffset
+			wit.PreimageKey = m.lastPreimageKey
+			wit.PreimageValue = m.lastPreimage
+		}
+	}
+	return
+}

mipsevm/memory.go

@@ -11,15 +11,15 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
+// Note: 2**12 = 4 KiB, the minimum page-size in Unicorn for mmap
+// as well as the Go runtime min phys page size.
 const (
-	// Note: 2**12 = 4 KiB, the minimum page-size in Unicorn for mmap
-	// as well as the Go runtime min phys page size.
-	pageAddrSize = 12
-	pageKeySize  = 32 - pageAddrSize
-	pageSize     = 1 << pageAddrSize
-	pageAddrMask = pageSize - 1
-	maxPageCount = 1 << pageKeySize
-	pageKeyMask  = maxPageCount - 1
+	PageAddrSize = 12
+	PageKeySize  = 32 - PageAddrSize
+	PageSize     = 1 << PageAddrSize
+	PageAddrMask = PageSize - 1
+	MaxPageCount = 1 << PageKeySize
+	PageKeyMask  = MaxPageCount - 1
 )
 
 func HashPair(left, right [32]byte) [32]byte {
@@ -62,12 +62,12 @@ func (m *Memory) Invalidate(addr uint32) {
 	}
 
 	// find page, and invalidate addr within it
-	if p, ok := m.Pages[addr>>pageAddrSize]; ok {
-		p.Invalidate(addr & pageAddrMask)
+	if p, ok := m.Pages[addr>>PageAddrSize]; ok {
+		p.Invalidate(addr & PageAddrMask)
 	}
 
 	// find the gindex of the first page covering the address
-	gindex := ((uint64(1) << 32) | uint64(addr)) >> pageAddrSize
+	gindex := ((uint64(1) << 32) | uint64(addr)) >> PageAddrSize
 
 	for gindex > 0 {
 		m.Nodes[gindex] = nil
@@ -80,9 +80,9 @@ func (m *Memory) MerkleizeSubtree(gindex uint64) [32]byte {
 	if l > 28 {
 		panic("gindex too deep")
 	}
-	if l > pageKeySize {
-		depthIntoPage := l - 1 - pageKeySize
-		pageIndex := (gindex >> depthIntoPage) & pageKeyMask
+	if l > PageKeySize {
+		depthIntoPage := l - 1 - PageKeySize
+		pageIndex := (gindex >> depthIntoPage) & PageKeyMask
 		if p, ok := m.Pages[uint32(pageIndex)]; ok {
 			pageGindex := (1 << depthIntoPage) | (gindex & ((1 << depthIntoPage) - 1))
 			return p.MerkleizeSubtree(pageGindex)
@@ -90,7 +90,7 @@ func (m *Memory) MerkleizeSubtree(gindex uint64) [32]byte {
 			return zeroHashes[28-l] // page does not exist
 		}
 	}
-	if l > pageKeySize+1 {
+	if l > PageKeySize+1 {
 		panic("cannot jump into intermediate node of page")
 	}
 	n, ok := m.Nodes[gindex]
@@ -147,8 +147,8 @@ func (m *Memory) SetMemory(addr uint32, v uint32) {
 		panic(fmt.Errorf("unaligned memory access: %x", addr))
 	}
 
-	pageIndex := addr >> pageAddrSize
-	pageAddr := addr & pageAddrMask
+	pageIndex := addr >> PageAddrSize
+	pageAddr := addr & PageAddrMask
 	p, ok := m.Pages[pageIndex]
 	if !ok {
 		// allocate the page if we have not already.
@@ -165,11 +165,11 @@ func (m *Memory) GetMemory(addr uint32) uint32 {
 	if addr&0x3 != 0 {
 		panic(fmt.Errorf("unaligned memory access: %x", addr))
 	}
-	p, ok := m.Pages[addr>>pageAddrSize]
+	p, ok := m.Pages[addr>>PageAddrSize]
 	if !ok {
 		return 0
 	}
-	pageAddr := addr & pageAddrMask
+	pageAddr := addr & PageAddrMask
 	return binary.BigEndian.Uint32(p.Data[pageAddr : pageAddr+4])
 }
 
@@ -177,7 +177,7 @@ func (m *Memory) AllocPage(pageIndex uint32) *CachedPage {
 	p := &CachedPage{Data: new(Page)}
 	m.Pages[pageIndex] = p
 	// make nodes to root
-	k := (1 << pageKeySize) | uint64(pageIndex)
+	k := (1 << PageKeySize) | uint64(pageIndex)
 	for k > 0 {
 		m.Nodes[k] = nil
 		k >>= 1
@@ -222,8 +222,8 @@ func (m *Memory) UnmarshalJSON(data []byte) error {
 
 func (m *Memory) SetMemoryRange(addr uint32, r io.Reader) error {
 	for {
-		pageIndex := addr >> pageAddrSize
-		pageAddr := addr & pageAddrMask
+		pageIndex := addr >> PageAddrSize
+		pageAddr := addr & PageAddrMask
 		p, ok := m.Pages[pageIndex]
 		if !ok {
 			p = m.AllocPage(pageIndex)
@@ -255,12 +255,12 @@ func (r *memReader) Read(dest []byte) (n int, err error) {
 	// It may wrap around the address range, and may not be aligned
 	endAddr := r.addr + r.count
 
-	pageIndex := r.addr >> pageAddrSize
-	start := r.addr & pageAddrMask
-	end := uint32(pageSize)
+	pageIndex := r.addr >> PageAddrSize
+	start := r.addr & PageAddrMask
+	end := uint32(PageSize)
 
-	if pageIndex == (endAddr >> pageAddrSize) {
-		end = endAddr & pageAddrMask
+	if pageIndex == (endAddr >> PageAddrSize) {
+		end = endAddr & PageAddrMask
 	}
 	p, ok := r.m.Pages[pageIndex]
 	if ok {

mipsevm/memory_test.go

@@ -72,20 +72,20 @@ func TestMemoryMerkleRoot(t *testing.T) {
 	})
 	t.Run("two empty pages", func(t *testing.T) {
 		m := NewMemory()
-		m.SetMemory(pageSize*3, 0)
-		m.SetMemory(pageSize*10, 0)
+		m.SetMemory(PageSize*3, 0)
+		m.SetMemory(PageSize*10, 0)
 		root := m.MerkleRoot()
 		require.Equal(t, zeroHashes[32-5], root, "zero still")
 	})
 	t.Run("random few pages", func(t *testing.T) {
 		m := NewMemory()
-		m.SetMemory(pageSize*3, 1)
-		m.SetMemory(pageSize*5, 42)
-		m.SetMemory(pageSize*6, 123)
-		p3 := m.MerkleizeSubtree((1 << pageKeySize) | 3)
-		p5 := m.MerkleizeSubtree((1 << pageKeySize) | 5)
-		p6 := m.MerkleizeSubtree((1 << pageKeySize) | 6)
-		z := zeroHashes[pageAddrSize-5]
+		m.SetMemory(PageSize*3, 1)
+		m.SetMemory(PageSize*5, 42)
+		m.SetMemory(PageSize*6, 123)
+		p3 := m.MerkleizeSubtree((1 << PageKeySize) | 3)
+		p5 := m.MerkleizeSubtree((1 << PageKeySize) | 5)
+		p6 := m.MerkleizeSubtree((1 << PageKeySize) | 6)
+		z := zeroHashes[PageAddrSize-5]
 		r1 := HashPair(
 			HashPair(
 				HashPair(z, z),  // 0,1
@@ -96,7 +96,7 @@ func TestMemoryMerkleRoot(t *testing.T) {
 				HashPair(p6, z), // 6,7
 			),
 		)
-		r2 := m.MerkleizeSubtree(1 << (pageKeySize - 3))
+		r2 := m.MerkleizeSubtree(1 << (PageKeySize - 3))
 		require.Equal(t, r1, r2, "expecting manual page combination to match subtree merkle func")
 	})
 	t.Run("invalidate page", func(t *testing.T) {

mipsevm/metadata.go

+package mipsevm
+
+import (
+	"debug/elf"
+	"fmt"
+	"sort"
+)
+
+type Symbol struct {
+	Name  string `json:"name"`
+	Start uint32 `json:"start"`
+	Size  uint32 `json:"size"`
+}
+
+type Metadata struct {
+	Symbols []Symbol `json:"symbols"`
+}
+
+func MakeMetadata(elfProgram *elf.File) (*Metadata, error) {
+	syms, err := elfProgram.Symbols()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load symbols table: %w", err)
+	}
+	// Make sure the table is sorted, Go outputs mostly sorted data, except some internal functions
+	sort.Slice(syms, func(i, j int) bool {
+		return syms[i].Value < syms[j].Value
+	})
+	out := &Metadata{Symbols: make([]Symbol, len(syms))}
+	for i, s := range syms {
+		out.Symbols[i] = Symbol{Name: s.Name, Start: uint32(s.Value), Size: uint32(s.Size)}
+	}
+	return out, nil
+}
+
+func (m *Metadata) LookupSymbol(addr uint32) string {
+	if len(m.Symbols) == 0 {
+		return "!unknown"
+	}
+	// find first symbol with higher start. Or n if no such symbol exists
+	i := sort.Search(len(m.Symbols), func(i int) bool {
+		return m.Symbols[i].Start > addr
+	})
+	if i == 0 {
+		return "!start"
+	}
+	out := &m.Symbols[i-1]
+	if out.Start+out.Size < addr { // addr may be pointing to a gap between symbols
+		return "!gap"
+	}
+	return out.Name
+}
+
+// HexU32 to lazy-format integer attributes for logging
+type HexU32 uint32
+
+func (v HexU32) String() string {
+	return fmt.Sprintf("%08x", uint32(v))
+}
+
+func (v HexU32) MarshalText() ([]byte, error) {
+	return []byte(v.String()), nil
+}

mipsevm/page.go

@@ -7,7 +7,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
-type Page [pageSize]byte
+type Page [PageSize]byte
 
 func (p *Page) MarshalText() ([]byte, error) {
 	dst := make([]byte, hex.EncodedLen(len(p)))
@@ -16,8 +16,8 @@ func (p *Page) MarshalText() ([]byte, error) {
 }
 
 func (p *Page) UnmarshalText(dat []byte) error {
-	if len(dat) != pageSize*2 {
-		return fmt.Errorf("expected %d hex chars, but got %d", pageSize*2, len(dat))
+	if len(dat) != PageSize*2 {
+		return fmt.Errorf("expected %d hex chars, but got %d", PageSize*2, len(dat))
 	}
 	_, err := hex.Decode(p[:], dat)
 	return err
@@ -26,16 +26,16 @@ func (p *Page) UnmarshalText(dat []byte) error {
 type CachedPage struct {
 	Data *Page
 	// intermediate nodes only
-	Cache [pageSize / 32][32]byte
+	Cache [PageSize / 32][32]byte
 	// true if the intermediate node is valid
-	Ok [pageSize / 32]bool
+	Ok [PageSize / 32]bool
 }
 
 func (p *CachedPage) Invalidate(pageAddr uint32) {
-	if pageAddr >= pageSize {
+	if pageAddr >= PageSize {
 		panic("invalid page addr")
 	}
-	k := (1 << pageAddrSize) | pageAddr
+	k := (1 << PageAddrSize) | pageAddr
 	// first cache layer caches nodes that has two 32 byte leaf nodes.
 	k >>= 5 + 1
 	for k > 0 {
@@ -45,13 +45,13 @@ func (p *CachedPage) Invalidate(pageAddr uint32) {
 }
 
 func (p *CachedPage) InvalidateFull() {
-	p.Ok = [pageSize / 32]bool{} // reset everything to false
+	p.Ok = [PageSize / 32]bool{} // reset everything to false
 }
 
 func (p *CachedPage) MerkleRoot() [32]byte {
 	// hash the bottom layer
-	for i := uint64(0); i < pageSize; i += 64 {
-		j := pageSize/32/2 + i/64
+	for i := uint64(0); i < PageSize; i += 64 {
+		j := PageSize/32/2 + i/64
 		if p.Ok[j] {
 			continue
 		}
@@ -61,7 +61,7 @@ func (p *CachedPage) MerkleRoot() [32]byte {
 	}
 
 	// hash the cache layers
-	for i := pageSize/32 - 2; i > 0; i -= 2 {
+	for i := PageSize/32 - 2; i > 0; i -= 2 {
 		j := i >> 1
 		if p.Ok[j] {
 			continue
@@ -75,12 +75,12 @@ func (p *CachedPage) MerkleRoot() [32]byte {
 
 func (p *CachedPage) MerkleizeSubtree(gindex uint64) [32]byte {
 	_ = p.MerkleRoot() // fill cache
-	if gindex >= pageSize/32 {
-		if gindex >= pageSize/32*2 {
+	if gindex >= PageSize/32 {
+		if gindex >= PageSize/32*2 {
 			panic("gindex too deep")
 		}
 		// it's pointing to a bottom node
-		nodeIndex := gindex & (pageAddrMask >> 5)
+		nodeIndex := gindex & (PageAddrMask >> 5)
 		return *(*[32]byte)(p.Data[nodeIndex*32 : nodeIndex*32+32])
 	}
 	return p.Cache[gindex]

mipsevm/page_test.go

@@ -11,7 +11,7 @@ func TestCachedPage(t *testing.T) {
 	p := &CachedPage{Data: new(Page)}
 	p.Data[42] = 0xab
 
-	gindex := ((uint64(1) << pageAddrSize) | 42) >> 5
+	gindex := ((uint64(1) << PageAddrSize) | 42) >> 5
 	node := common.Hash(p.MerkleizeSubtree(gindex))
 	expectedLeaf := common.Hash{10: 0xab}
 	require.Equal(t, expectedLeaf, node, "leaf nodes should not be hashed")

mipsevm/patch.go

@@ -65,6 +65,16 @@ func PatchGo(f *elf.File, st *State) error {
 			"runtime.main.func1",        // patch out: main.func() { newm(sysmon, ....) }
 			"runtime.deductSweepCredit", // uses floating point nums and interacts with gc we disabled
 			"runtime.(*gcControllerState).commit",
+			// these prometheus packages rely on concurrent background things. We cannot run those.
+			"github.com/prometheus/client_golang/prometheus.init",
+			"github.com/prometheus/client_golang/prometheus.init.0",
+			"github.com/prometheus/procfs.init",
+			"github.com/prometheus/common/model.init",
+			"github.com/prometheus/client_model/go.init",
+			"github.com/prometheus/client_model/go.init.0",
+			"github.com/prometheus/client_model/go.init.1",
+			// skip flag pkg init, we need to debug arg-processing more to see why this fails
+			"flag.init",
 			// We need to patch this out, we don't pass float64nan because we don't support floats
 			"runtime.check":
 			// MIPS32 patch: ret (pseudo instruction)
@@ -89,7 +99,7 @@ func PatchStack(st *State) error {
 	// setup stack pointer
 	sp := uint32(0x7f_ff_d0_00)
 	// allocate 1 page for the initial stack data, and 16KB = 4 pages for the stack to grow
-	if err := st.Memory.SetMemoryRange(sp-4*pageSize, bytes.NewReader(make([]byte, 5*pageSize))); err != nil {
+	if err := st.Memory.SetMemoryRange(sp-4*PageSize, bytes.NewReader(make([]byte, 5*PageSize))); err != nil {
 		return fmt.Errorf("failed to allocate page for stack content")
 	}
 	st.Registers[29] = sp

mipsevm/state_test.go

@@ -19,9 +19,8 @@ import (
 	"github.com/ethereum-optimism/cannon/preimage"
 )
 
-// baseAddrStart - baseAddrEnd is used in tests to write the results to
+// 0xbf_c0_00_00 ... baseAddrEnd is used in tests to write the results to
 const baseAddrEnd = 0xbf_ff_ff_f0
-const baseAddrStart = 0xbf_c0_00_00
 
 // endAddr is used as return-address for tests
 const endAddr = 0xa7ef00d0
@@ -51,24 +50,7 @@ func TestState(t *testing.T) {
 			// set the return address ($ra) to jump into when test completes
 			state.Registers[31] = endAddr
 
-			//err = state.SetMemoryRange(baseAddr&^pageAddrMask, bytes.NewReader(make([]byte, pageSize)))
-			//require.NoError(t, err, "must allocate page for the result data")
-			//
-			//err = state.SetMemoryRange(endAddr&^pageAddrMask, bytes.NewReader(make([]byte, pageSize)))
-			//require.NoError(t, err, "must allocate page to return to")
-
-			mu, err := NewUnicorn()
-			require.NoError(t, err, "load unicorn")
-			defer mu.Close()
-
-			require.NoError(t, mu.MemMap(baseAddrStart, ((baseAddrEnd-baseAddrStart)&^pageAddrMask)+pageSize))
-			require.NoError(t, mu.MemMap(endAddr&^pageAddrMask, pageSize))
-
-			err = LoadUnicorn(state, mu)
-			require.NoError(t, err, "load state into unicorn")
-
-			us, err := NewUnicornState(mu, state, nil, os.Stdout, os.Stderr)
-			require.NoError(t, err, "hook unicorn to state")
+			us := NewInstrumentedState(state, nil, os.Stdout, os.Stderr)
 
 			for i := 0; i < 1000; i++ {
 				if us.state.PC == endAddr {
@@ -97,14 +79,8 @@ func TestHello(t *testing.T) {
 	require.NoError(t, err, "apply Go runtime patches")
 	require.NoError(t, PatchStack(state), "add initial stack")
 
-	mu, err := NewUnicorn()
-	require.NoError(t, err, "load unicorn")
-	defer mu.Close()
-	err = LoadUnicorn(state, mu)
-	require.NoError(t, err, "load state into unicorn")
 	var stdOutBuf, stdErrBuf bytes.Buffer
-	us, err := NewUnicornState(mu, state, nil, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr))
-	require.NoError(t, err, "hook unicorn to state")
+	us := NewInstrumentedState(state, nil, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr))
 
 	for i := 0; i < 400_000; i++ {
 		if us.state.Exited {
@@ -201,17 +177,10 @@ func TestClaim(t *testing.T) {
 	require.NoError(t, err, "apply Go runtime patches")
 	require.NoError(t, PatchStack(state), "add initial stack")
 
-	mu, err := NewUnicorn()
-	require.NoError(t, err, "load unicorn")
-	defer mu.Close()
-	err = LoadUnicorn(state, mu)
-	require.NoError(t, err, "load state into unicorn")
-
 	oracle, expectedStdOut, expectedStdErr := claimTestOracle(t)
 
 	var stdOutBuf, stdErrBuf bytes.Buffer
-	us, err := NewUnicornState(mu, state, oracle, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr))
-	require.NoError(t, err, "hook unicorn to state")
+	us := NewInstrumentedState(state, oracle, io.MultiWriter(&stdOutBuf, os.Stdout), io.MultiWriter(&stdErrBuf, os.Stderr))
 
 	for i := 0; i < 2000_000; i++ {
 		if us.state.Exited {

mipsevm/witness.go

@@ -11,13 +11,14 @@ import (
 )
 
 type StepWitness struct {
-	state []byte
+	// encoded state witness
+	State []byte
 
-	memProof []byte
+	MemProof []byte
 
-	preimageKey    [32]byte // zeroed when no pre-image is accessed
-	preimageValue  []byte   // including the 8-byte length prefix
-	preimageOffset uint32
+	PreimageKey    [32]byte // zeroed when no pre-image is accessed
+	PreimageValue  []byte   // including the 8-byte length prefix
+	PreimageOffset uint32
 }
 
 func uint32ToBytes32(v uint32) []byte {
@@ -27,30 +28,30 @@ func uint32ToBytes32(v uint32) []byte {
 }
 
 func (wit *StepWitness) EncodeStepInput() []byte {
-	stateHash := crypto.Keccak256Hash(wit.state)
+	stateHash := crypto.Keccak256Hash(wit.State)
 	var input []byte
 	input = append(input, StepBytes4...)
 	input = append(input, stateHash[:]...)
 	input = append(input, uint32ToBytes32(32*3)...)                           // state data offset in bytes
-	input = append(input, uint32ToBytes32(32*3+32+uint32(len(wit.state)))...) // proof data offset in bytes
+	input = append(input, uint32ToBytes32(32*3+32+uint32(len(wit.State)))...) // proof data offset in bytes
 
-	input = append(input, uint32ToBytes32(uint32(len(wit.state)))...) // state data length in bytes
-	input = append(input, wit.state[:]...)
-	input = append(input, uint32ToBytes32(uint32(len(wit.memProof)))...) // proof data length in bytes
-	input = append(input, wit.memProof[:]...)
+	input = append(input, uint32ToBytes32(uint32(len(wit.State)))...) // state data length in bytes
+	input = append(input, wit.State[:]...)
+	input = append(input, uint32ToBytes32(uint32(len(wit.MemProof)))...) // proof data length in bytes
+	input = append(input, wit.MemProof[:]...)
 	return input
 }
 
 func (wit *StepWitness) HasPreimage() bool {
-	return wit.preimageKey != ([32]byte{})
+	return wit.PreimageKey != ([32]byte{})
 }
 
 func (wit *StepWitness) EncodePreimageOracleInput() ([]byte, error) {
-	if wit.preimageKey == ([32]byte{}) {
+	if wit.PreimageKey == ([32]byte{}) {
 		return nil, errors.New("cannot encode pre-image oracle input, witness has no pre-image to proof")
 	}
 
-	switch preimage.KeyType(wit.preimageKey[0]) {
+	switch preimage.KeyType(wit.PreimageKey[0]) {
 	case preimage.LocalKeyType:
 		// We have no on-chain form of preparing the bootstrap pre-images onchain yet.
 		// So instead we cheat them in.
@@ -58,25 +59,25 @@ func (wit *StepWitness) EncodePreimageOracleInput() ([]byte, error) {
 		// rather than going through the global keccak256 oracle.
 		var input []byte
 		input = append(input, CheatBytes4...)
-		input = append(input, uint32ToBytes32(wit.preimageOffset)...)
-		input = append(input, wit.preimageKey[:]...)
+		input = append(input, uint32ToBytes32(wit.PreimageOffset)...)
+		input = append(input, wit.PreimageKey[:]...)
 		var tmp [32]byte
-		copy(tmp[:], wit.preimageValue[wit.preimageOffset:])
+		copy(tmp[:], wit.PreimageValue[wit.PreimageOffset:])
 		input = append(input, tmp[:]...)
-		input = append(input, uint32ToBytes32(uint32(len(wit.preimageValue))-8)...)
+		input = append(input, uint32ToBytes32(uint32(len(wit.PreimageValue))-8)...)
 		// TODO: do we want to pad the end to a multiple of 32 bytes?
 		return input, nil
 	case preimage.Keccak256KeyType:
 		var input []byte
 		input = append(input, LoadKeccak256PreimagePartBytes4...)
-		input = append(input, uint32ToBytes32(wit.preimageOffset)...)
+		input = append(input, uint32ToBytes32(wit.PreimageOffset)...)
 		input = append(input, uint32ToBytes32(32+32)...) // partOffset, calldata offset
-		input = append(input, uint32ToBytes32(uint32(len(wit.preimageValue))-8)...)
-		input = append(input, wit.preimageValue[8:]...)
+		input = append(input, uint32ToBytes32(uint32(len(wit.PreimageValue))-8)...)
+		input = append(input, wit.PreimageValue[8:]...)
 		// TODO: do we want to pad the end to a multiple of 32 bytes?
 		return input, nil
 	default:
 		return nil, fmt.Errorf("unsupported pre-image type %d, cannot prepare preimage with key %x offset %d for oracle",
-			wit.preimageKey[0], wit.preimageKey, wit.preimageOffset)
+			wit.PreimageKey[0], wit.PreimageKey, wit.PreimageOffset)
 	}
 }