Skip to content

N
nebula
Commit 1741146c authored by protolambda's avatar protolambda
Browse files
Options

mipsevm,cmd: optimize steps

parent d16685bd
Hide whitespace changes
Inline Side-by-side
Showing with 122 additions and 31 deletions
.gitignore
View file @ 1741146c
......@@ -9,3 +9,5 @@ example/bin
contracts/out
state.json
*.json
*.pprof
*.out
cmd/run.go
View file @ 1741146c
......@@ -12,6 +12,8 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/urfave/cli/v2"
"github.com/pkg/profile"
"github.com/ethereum-optimism/cannon/mipsevm"
"github.com/ethereum-optimism/cannon/preimage"
)
......@@ -71,7 +73,7 @@ var (
RunInfoAtFlag = &cli.GenericFlag{
Name: "info-at",
Usage: "step pattern to print info at: " + patternHelp,
Value: MustStepMatcherFlag("%1000"),
Value: MustStepMatcherFlag("%100000"),
Required: false,
}
)
......@@ -190,6 +192,8 @@ func Guard(proc *os.ProcessState, fn StepFn) StepFn {
var _ mipsevm.PreimageOracle = (*ProcessPreimageOracle)(nil)
func Run(ctx *cli.Context) error {
defer profile.Start(profile.NoShutdownHook, profile.ProfilePath("."), profile.CPUProfile).Stop()
state, err := loadJSON[mipsevm.State](ctx.Path(RunInputFlag.Name))
if err != nil {
return err
......@@ -253,14 +257,18 @@ func Run(ctx *cli.Context) error {
start := time.Now()
startStep := state.Step
// avoid symbol lookups every instruction by preparing a matcher func
sleepCheck := meta.SymbolMatcher("runtime.notesleep")
for !state.Exited {
if err := ctx.Context.Err(); err != nil {
return err
if state.Step%100 == 0 { // don't do the ctx err check (includes lock) too often
if err := ctx.Context.Err(); err != nil {
return err
}
}
step := state.Step
name := meta.LookupSymbol(state.PC)
if infoAt(state) {
delta := time.Since(start)
l.Info("processing",
......@@ -268,12 +276,13 @@ func Run(ctx *cli.Context) error {
"pc", mipsevm.HexU32(state.PC),
"insn", mipsevm.HexU32(state.Memory.GetMemory(state.PC)),
"ips", float64(step-startStep)/(float64(delta)/float64(time.Second)),
"pages", len(state.Memory.Pages),
"pages", state.Memory.PageCount(),
"mem", state.Memory.Usage(),
"name", name,
"name", meta.LookupSymbol(state.PC),
)
}
if name == "runtime.notesleep" { // don't loop forever when we get stuck because of an unexpected bad program
if sleepCheck(state.PC) { // don't loop forever when we get stuck because of an unexpected bad program
return fmt.Errorf("got stuck in Go sleep at step %d", step)
}
......
diffmips/unicorntest/unicorn.go
View file @ 1741146c
......@@ -422,15 +422,19 @@ func NewUnicorn() (uc.Unicorn, error) {
func LoadUnicorn(st *mipsevm.State, mu uc.Unicorn) error {
// mmap and write each page of memory state into unicorn
for pageIndex, page := range st.Memory.Pages {
if err := st.Memory.ForEachPage(func(pageIndex uint32, page *mipsevm.Page) error {
addr := uint64(pageIndex) << mipsevm.PageAddrSize
if err := mu.MemMap(addr, mipsevm.PageSize); err != nil {
return fmt.Errorf("failed to mmap page at addr 0x%x: %w", addr, err)
}
if err := mu.MemWrite(addr, page.Data[:]); err != nil {
if err := mu.MemWrite(addr, page[:]); err != nil {
return fmt.Errorf("failed to write page at addr 0x%x: %w", addr, err)
}
return nil
}); err != nil {
return err
}
// write all registers into unicorn, including PC, LO, HI
regValues := make([]uint64, 32+3)
// TODO: do we have to sign-extend registers before writing them to unicorn, or are the trailing bits unused?
......
go.mod
View file @ 1741146c
......@@ -5,6 +5,7 @@ go 1.20
require (
github.com/ethereum-optimism/cannon/preimage v0.0.0
github.com/ethereum/go-ethereum v1.11.5
github.com/pkg/profile v1.7.0
github.com/stretchr/testify v1.8.2
github.com/urfave/cli/v2 v2.25.3
)
......@@ -25,6 +26,7 @@ require (
github.com/deckarep/golang-set/v2 v2.1.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/edsrzf/mmap-go v1.0.0 // indirect
github.com/felixge/fgprof v0.9.3 // indirect
github.com/getsentry/sentry-go v0.18.0 // indirect
github.com/go-ole/go-ole v1.2.1 // indirect
github.com/go-stack/stack v1.8.1 // indirect
......@@ -32,6 +34,7 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
github.com/holiman/uint256 v1.2.0 // indirect
......
go.sum
View file @ 1741146c
......@@ -25,6 +25,9 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA=
......@@ -70,6 +73,8 @@ github.com/ethereum/go-ethereum v1.11.5 h1:3M1uan+LAUvdn+7wCEFrcMM4LJTeuxDrPTg/f
github.com/ethereum/go-ethereum v1.11.5/go.mod h1:it7x0DWnTDMfVFdXcU6Ti4KEFQynLHVRarcSlPr0HBo=
github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
......@@ -128,6 +133,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
......@@ -141,6 +148,7 @@ github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM
github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE=
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
......@@ -221,6 +229,8 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
......
mipsevm/memory.go
View file @ 1741146c
......@@ -39,20 +39,39 @@ var zeroHashes = func() [256][32]byte {
type Memory struct {
// generalized index -> merkle root or nil if invalidated
Nodes map[uint64]*[32]byte
nodes map[uint64]*[32]byte
// pageIndex -> cached page
Pages map[uint32]*CachedPage
pages map[uint32]*CachedPage
// Note: since we don't de-alloc pages, we don't do ref-counting.
// Once a page exists, it doesn't leave memory
// two caches: we often read instructions from one page, and do memory things with another page.
// this prevents map lookups each instruction
lastPageKeys [2]uint32
lastPage [2]*CachedPage
}
func NewMemory() *Memory {
return &Memory{
Nodes: make(map[uint64]*[32]byte),
Pages: make(map[uint32]*CachedPage),
nodes: make(map[uint64]*[32]byte),
pages: make(map[uint32]*CachedPage),
lastPageKeys: [2]uint32{^uint32(0), ^uint32(0)}, // default to invalid keys, to not match any pages
}
}
func (m *Memory) PageCount() int {
return len(m.pages)
}
func (m *Memory) ForEachPage(fn func(pageIndex uint32, page *Page) error) error {
for pageIndex, cachedPage := range m.pages {
if err := fn(pageIndex, cachedPage.Data); err != nil {
return err
}
}
return nil
}
func (m *Memory) Invalidate(addr uint32) {
......@@ -62,15 +81,21 @@ func (m *Memory) Invalidate(addr uint32) {
}
// find page, and invalidate addr within it
if p, ok := m.Pages[addr>>PageAddrSize]; ok {
if p, ok := m.pageLookup(addr >> PageAddrSize); ok {
prevValid := p.Ok[1]
p.Invalidate(addr & PageAddrMask)
if !prevValid { // if the page was already invalid before, then nodes to mem-root will also still be.
return
}
} else { // no page? nothing to invalidate
return
}
// find the gindex of the first page covering the address
gindex := ((uint64(1) << 32) | uint64(addr)) >> PageAddrSize
for gindex > 0 {
m.Nodes[gindex] = nil
m.nodes[gindex] = nil
gindex >>= 1
}
}
......@@ -83,7 +108,7 @@ func (m *Memory) MerkleizeSubtree(gindex uint64) [32]byte {
if l > PageKeySize {
depthIntoPage := l - 1 - PageKeySize
pageIndex := (gindex >> depthIntoPage) & PageKeyMask
if p, ok := m.Pages[uint32(pageIndex)]; ok {
if p, ok := m.pages[uint32(pageIndex)]; ok {
pageGindex := (1 << depthIntoPage) | (gindex & ((1 << depthIntoPage) - 1))
return p.MerkleizeSubtree(pageGindex)
} else {
......@@ -93,7 +118,7 @@ func (m *Memory) MerkleizeSubtree(gindex uint64) [32]byte {
if l > PageKeySize+1 {
panic("cannot jump into intermediate node of page")
}
n, ok := m.Nodes[gindex]
n, ok := m.nodes[gindex]
if !ok {
// if the node doesn't exist, the whole sub-tree is zeroed
return zeroHashes[28-l]
......@@ -104,7 +129,7 @@ func (m *Memory) MerkleizeSubtree(gindex uint64) [32]byte {
left := m.MerkleizeSubtree(gindex << 1)
right := m.MerkleizeSubtree((gindex << 1) | 1)
r := HashPair(left, right)
m.Nodes[gindex] = &r
m.nodes[gindex] = &r
return r
}
......@@ -141,6 +166,27 @@ func (m *Memory) MerkleRoot() [32]byte {
return m.MerkleizeSubtree(1)
}
func (m *Memory) pageLookup(pageIndex uint32) (*CachedPage, bool) {
// hit caches
if pageIndex == m.lastPageKeys[0] {
return m.lastPage[0], true
}
if pageIndex == m.lastPageKeys[1] {
return m.lastPage[1], true
}
p, ok := m.pages[pageIndex]
// only cache existing pages.
if ok {
m.lastPageKeys[1] = m.lastPageKeys[0]
m.lastPage[1] = m.lastPage[0]
m.lastPageKeys[0] = pageIndex
m.lastPage[0] = p
}
return p, ok
}
func (m *Memory) SetMemory(addr uint32, v uint32) {
// addr must be aligned to 4 bytes
if addr&0x3 != 0 {
......@@ -149,7 +195,7 @@ func (m *Memory) SetMemory(addr uint32, v uint32) {
pageIndex := addr >> PageAddrSize
pageAddr := addr & PageAddrMask
p, ok := m.Pages[pageIndex]
p, ok := m.pageLookup(pageIndex)
if !ok {
// allocate the page if we have not already.
// Go may mmap relatively large ranges, but we only allocate the pages just in time.
......@@ -165,7 +211,7 @@ func (m *Memory) GetMemory(addr uint32) uint32 {
if addr&0x3 != 0 {
panic(fmt.Errorf("unaligned memory access: %x", addr))
}
p, ok := m.Pages[addr>>PageAddrSize]
p, ok := m.pageLookup(addr >> PageAddrSize)
if !ok {
return 0
}
......@@ -175,11 +221,11 @@ func (m *Memory) GetMemory(addr uint32) uint32 {
func (m *Memory) AllocPage(pageIndex uint32) *CachedPage {
p := &CachedPage{Data: new(Page)}
m.Pages[pageIndex] = p
m.pages[pageIndex] = p
// make nodes to root
k := (1 << PageKeySize) | uint64(pageIndex)
for k > 0 {
m.Nodes[k] = nil
m.nodes[k] = nil
k >>= 1
}
return p
......@@ -191,8 +237,8 @@ type pageEntry struct {
}
func (m *Memory) MarshalJSON() ([]byte, error) {
pages := make([]pageEntry, 0, len(m.Pages))
for k, p := range m.Pages {
pages := make([]pageEntry, 0, len(m.pages))
for k, p := range m.pages {
pages = append(pages, pageEntry{
Index: k,
Data: p.Data,
......@@ -209,13 +255,15 @@ func (m *Memory) UnmarshalJSON(data []byte) error {
if err := json.Unmarshal(data, &pages); err != nil {
return err
}
m.Nodes = make(map[uint64]*[32]byte)
m.Pages = make(map[uint32]*CachedPage)
m.nodes = make(map[uint64]*[32]byte)
m.pages = make(map[uint32]*CachedPage)
m.lastPageKeys = [2]uint32{^uint32(0), ^uint32(0)}
m.lastPage = [2]*CachedPage{nil, nil}
for i, p := range pages {
if _, ok := m.Pages[p.Index]; ok {
if _, ok := m.pages[p.Index]; ok {
return fmt.Errorf("cannot load duplicate page, entry %d, page index %d", i, p.Index)
}
m.Pages[p.Index] = &CachedPage{Data: p.Data}
m.AllocPage(p.Index).Data = p.Data
}
return nil
}
......@@ -224,7 +272,7 @@ func (m *Memory) SetMemoryRange(addr uint32, r io.Reader) error {
for {
pageIndex := addr >> PageAddrSize
pageAddr := addr & PageAddrMask
p, ok := m.Pages[pageIndex]
p, ok := m.pageLookup(pageIndex)
if !ok {
p = m.AllocPage(pageIndex)
}
......@@ -262,7 +310,7 @@ func (r *memReader) Read(dest []byte) (n int, err error) {
if pageIndex == (endAddr >> PageAddrSize) {
end = endAddr & PageAddrMask
}
p, ok := r.m.Pages[pageIndex]
p, ok := r.m.pageLookup(pageIndex)
if ok {
n = copy(dest, p.Data[start:end])
} else {
......@@ -278,7 +326,7 @@ func (m *Memory) ReadMemoryRange(addr uint32, count uint32) io.Reader {
}
func (m *Memory) Usage() string {
total := uint64(len(m.Pages)) * PageSize
total := uint64(len(m.pages)) * PageSize
const unit = 1024
if total < unit {
return fmt.Sprintf("%d B", total)
......
mipsevm/metadata.go
View file @ 1741146c
......@@ -50,6 +50,21 @@ func (m *Metadata) LookupSymbol(addr uint32) string {
return out.Name
}
func (m *Metadata) SymbolMatcher(name string) func(addr uint32) bool {
for _, s := range m.Symbols {
if s.Name == name {
start := s.Start
end := s.Start + s.Size
return func(addr uint32) bool {
return addr >= start && addr < end
}
}
}
return func(addr uint32) bool {
return false
}
}
// HexU32 to lazy-format integer attributes for logging
type HexU32 uint32
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment