Add binary deserialization to asterisc's state converter trace function (#12238)

* Move serialize from cannon to op-service

* Add binary deserialization to asterisc's state converter trace function

* Use Asterisc's witness cmd to generate state data

* Fix op-challenger tests

* remove unnecessary logic from state_converter, and only test witness subcommand

op-challenger/game/fault/register_task.go

@@ -85,7 +85,7 @@ func NewCannonRegisterTask(gameType faultTypes.GameType, cfg *config.Config, m c
 }
 
 func NewAsteriscRegisterTask(gameType faultTypes.GameType, cfg *config.Config, m caching.Metrics, serverExecutor vm.OracleServerExecutor) *RegisterTask {
-	stateConverter := asterisc.NewStateConverter()
+	stateConverter := asterisc.NewStateConverter(cfg.Asterisc)
 	return &RegisterTask{
 		gameType: gameType,
 		getPrestateProvider: cachePrestates(
@@ -117,7 +117,7 @@ func NewAsteriscRegisterTask(gameType faultTypes.GameType, cfg *config.Config, m
 }
 
 func NewAsteriscKonaRegisterTask(gameType faultTypes.GameType, cfg *config.Config, m caching.Metrics, serverExecutor vm.OracleServerExecutor) *RegisterTask {
-	stateConverter := asterisc.NewStateConverter()
+	stateConverter := asterisc.NewStateConverter(cfg.Asterisc)
 	return &RegisterTask{
 		gameType: gameType,
 		getPrestateProvider: cachePrestates(

op-challenger/game/fault/trace/asterisc/provider.go

@@ -49,7 +49,7 @@ func NewTraceProvider(logger log.Logger, m vm.Metricer, cfg vm.Config, vmCfg vm.
 			return kvstore.NewDiskKV(logger, vm.PreimageDir(dir), kvtypes.DataFormatFile)
 		}),
 		PrestateProvider: prestateProvider,
-		stateConverter:   NewStateConverter(),
+		stateConverter:   NewStateConverter(cfg),
 		cfg:              cfg,
 	}
 }
@@ -173,7 +173,7 @@ func NewTraceProviderForTest(logger log.Logger, m vm.Metricer, cfg *config.Confi
 		preimageLoader: utils.NewPreimageLoader(func() (utils.PreimageSource, error) {
 			return kvstore.NewDiskKV(logger, vm.PreimageDir(dir), kvtypes.DataFormatFile)
 		}),
-		stateConverter: NewStateConverter(),
+		stateConverter: NewStateConverter(cfg.Asterisc),
 		cfg:            cfg.Asterisc,
 	}
 	return &AsteriscTraceProviderForTest{p}

op-challenger/game/fault/trace/asterisc/provider_test.go

@@ -23,6 +23,7 @@ import (
 
 //go:embed test_data
 var testData embed.FS
+var asteriscWitnessLen = 362
 
 func PositionFromTraceIndex(provider *AsteriscTraceProvider, idx *big.Int) types.Position {
 	return types.NewPosition(provider.gameDepth, idx)
@@ -226,7 +227,7 @@ func setupWithTestData(t *testing.T, dataDir string, prestate string) (*Asterisc
 		generator:      generator,
 		prestate:       filepath.Join(dataDir, prestate),
 		gameDepth:      63,
-		stateConverter: &StateConverter{},
+		stateConverter: generator,
 	}, generator
 }
 
@@ -234,6 +235,20 @@ type stubGenerator struct {
 	generated  []int // Using int makes assertions easier
 	finalState *VMState
 	proof      *utils.ProofData
+
+	finalStatePath string
+}
+
+func (e *stubGenerator) ConvertStateToProof(ctx context.Context, statePath string) (*utils.ProofData, uint64, bool, error) {
+	if statePath == e.finalStatePath {
+		return &utils.ProofData{
+			ClaimValue: e.finalState.StateHash,
+			StateData:  e.finalState.Witness,
+			ProofData:  []byte{},
+		}, e.finalState.Step, e.finalState.Exited, nil
+	} else {
+		return nil, 0, false, fmt.Errorf("loading unexpected state: %s, only support: %s", statePath, e.finalStatePath)
+	}
 }
 
 func (e *stubGenerator) GenerateProof(ctx context.Context, dir string, i uint64) error {
@@ -244,6 +259,7 @@ func (e *stubGenerator) GenerateProof(ctx context.Context, dir string, i uint64)
 	if e.finalState != nil && e.finalState.Step <= i {
 		// Requesting a trace index past the end of the trace
 		proofFile = vm.FinalStatePath(dir, false)
+		e.finalStatePath = proofFile
 		data, err = json.Marshal(e.finalState)
 		if err != nil {
 			return err

op-challenger/game/fault/trace/asterisc/state_converter.go

 package asterisc
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
+	"os/exec"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 
-	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/utils"
-	"github.com/ethereum-optimism/optimism/op-service/ioutil"
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/vm"
 )
 
-var asteriscWitnessLen = 362
-
 // The state struct will be read from json.
 // other fields included in json are specific to FPVM implementation, and not required for trace provider.
 type VMState struct {
 	PC        uint64        `json:"pc"`
 	Exited    bool          `json:"exited"`
 	Step      uint64        `json:"step"`
-	Witness   []byte      `json:"witness"`
+	Witness   hexutil.Bytes `json:"witness"`
 	StateHash common.Hash   `json:"stateHash"`
 }
 
-func (state *VMState) validateStateHash() error {
-	exitCode := state.StateHash[0]
-	if exitCode >= 4 {
-		return fmt.Errorf("invalid stateHash: unknown exitCode %d", exitCode)
-	}
-	if (state.Exited && exitCode == mipsevm.VMStatusUnfinished) || (!state.Exited && exitCode != mipsevm.VMStatusUnfinished) {
-		return fmt.Errorf("invalid stateHash: invalid exitCode %d", exitCode)
-	}
-	return nil
-}
-
-func (state *VMState) validateWitness() error {
-	witnessLen := len(state.Witness)
-	if witnessLen != asteriscWitnessLen {
-		return fmt.Errorf("invalid witness: Length must be 362 but got %d", witnessLen)
-	}
-	return nil
+type StateConverter struct {
+	vmConfig    vm.Config
+	cmdExecutor func(ctx context.Context, binary string, args ...string) (stdOut string, stdErr string, err error)
 }
 
-// validateState performs verification of state; it is not perfect.
-// It does not recalculate whether witness nor stateHash is correctly set from state.
-func (state *VMState) validateState() error {
-	if err := state.validateStateHash(); err != nil {
-		return err
-	}
-	if err := state.validateWitness(); err != nil {
-		return err
+func NewStateConverter(vmConfig vm.Config) *StateConverter {
+	return &StateConverter{
+		vmConfig:    vmConfig,
+		cmdExecutor: runCmd,
 	}
-	return nil
 }
 
-// parseState parses state from json and goes on state validation
-func parseState(path string) (*VMState, error) {
-	file, err := ioutil.OpenDecompressed(path)
+func (c *StateConverter) ConvertStateToProof(ctx context.Context, statePath string) (*utils.ProofData, uint64, bool, error) {
+	stdOut, stdErr, err := c.cmdExecutor(ctx, c.vmConfig.VmBin, "witness", "--input", statePath)
 	if err != nil {
-		return nil, fmt.Errorf("cannot open state file (%v): %w", path, err)
-	}
-	return parseStateFromReader(file)
-}
-
-func parseStateFromReader(in io.ReadCloser) (*VMState, error) {
-	defer in.Close()
-	var state VMState
-	if err := json.NewDecoder(in).Decode(&state); err != nil {
-		return nil, fmt.Errorf("invalid asterisc VM state %w", err)
+		return nil, 0, false, fmt.Errorf("state conversion failed: %w (%s)", err, stdErr)
 	}
-	if err := state.validateState(); err != nil {
-		return nil, fmt.Errorf("invalid asterisc VM state %w", err)
-	}
-	return &state, nil
-}
-
-type StateConverter struct {
-}
-
-func NewStateConverter() *StateConverter {
-	return &StateConverter{}
-}
-
-func (c *StateConverter) ConvertStateToProof(_ context.Context, statePath string) (*utils.ProofData, uint64, bool, error) {
-	state, err := parseState(statePath)
-	if err != nil {
-		return nil, 0, false, fmt.Errorf("cannot read final state: %w", err)
+	var data VMState
+	if err := json.Unmarshal([]byte(stdOut), &data); err != nil {
+		return nil, 0, false, fmt.Errorf("failed to parse state data: %w", err)
 	}
 	// Extend the trace out to the full length using a no-op instruction that doesn't change any state
 	// No execution is done, so no proof-data or oracle values are required.
 	return &utils.ProofData{
-		ClaimValue:   state.StateHash,
-		StateData:    state.Witness,
+		ClaimValue:   data.StateHash,
+		StateData:    data.Witness,
 		ProofData:    []byte{},
 		OracleKey:    nil,
 		OracleValue:  nil,
 		OracleOffset: 0,
-	}, state.Step, state.Exited, nil
+	}, data.Step, data.Exited, nil
+}
+
+func runCmd(ctx context.Context, binary string, args ...string) (stdOut string, stdErr string, err error) {
+	var outBuf bytes.Buffer
+	var errBuf bytes.Buffer
+	cmd := exec.CommandContext(ctx, binary, args...)
+	cmd.Stdout = &outBuf
+	cmd.Stderr = &errBuf
+	err = cmd.Run()
+	stdOut = outBuf.String()
+	stdErr = errBuf.String()
+	return
 }

op-challenger/game/fault/trace/asterisc/state_converter_test.go

 package asterisc
 
 import (
-	"compress/gzip"
-	_ "embed"
+	"context"
 	"encoding/json"
-	"os"
-	"path/filepath"
+	"errors"
 	"testing"
 
+	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/vm"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/stretchr/testify/require"
 )
 
-//go:embed test_data/state.json
-var testState []byte
+const testBinary = "./somewhere/asterisc"
 
-func TestLoadState(t *testing.T) {
-	t.Run("Uncompressed", func(t *testing.T) {
-		dir := t.TempDir()
-		path := filepath.Join(dir, "state.json")
-		require.NoError(t, os.WriteFile(path, testState, 0644))
-
-		state, err := parseState(path)
-		require.NoError(t, err)
-
-		var expected VMState
-		require.NoError(t, json.Unmarshal(testState, &expected))
-		require.Equal(t, &expected, state)
-	})
+func TestStateConverter(t *testing.T) {
+	setup := func(t *testing.T) (*StateConverter, *capturingExecutor) {
+		vmCfg := vm.Config{
+			VmBin: testBinary,
+		}
+		executor := &capturingExecutor{}
+		converter := NewStateConverter(vmCfg)
+		converter.cmdExecutor = executor.exec
+		return converter, executor
+	}
 
-	t.Run("Gzipped", func(t *testing.T) {
-		dir := t.TempDir()
-		path := filepath.Join(dir, "state.json.gz")
-		f, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
-		require.NoError(t, err)
-		defer f.Close()
-		writer := gzip.NewWriter(f)
-		_, err = writer.Write(testState)
+	t.Run("Valid", func(t *testing.T) {
+		converter, executor := setup(t)
+		data := VMState{
+			Witness:   []byte{1, 2, 3, 4},
+			StateHash: common.Hash{0xab},
+			Step:      42,
+			Exited:    true,
+			PC:        11,
+		}
+		ser, err := json.Marshal(data)
 		require.NoError(t, err)
-		require.NoError(t, writer.Close())
-
-		state, err := parseState(path)
+		executor.stdOut = string(ser)
+		proof, step, exited, err := converter.ConvertStateToProof(context.Background(), "foo.json")
 		require.NoError(t, err)
+		require.Equal(t, data.Exited, exited)
+		require.Equal(t, data.Step, step)
+		require.Equal(t, data.StateHash, proof.ClaimValue)
+		require.Equal(t, data.Witness, proof.StateData)
+		require.NotNil(t, proof.ProofData, "later validations require this to be non-nil")
 
-		var expected VMState
-		require.NoError(t, json.Unmarshal(testState, &expected))
-		require.Equal(t, &expected, state)
+		require.Equal(t, testBinary, executor.binary)
+		require.Equal(t, []string{"witness", "--input", "foo.json"}, executor.args)
 	})
 
-	t.Run("InvalidStateWitness", func(t *testing.T) {
-		invalidWitnessLen := asteriscWitnessLen - 1
-		state := &VMState{
-			Step:    10,
-			Exited:  true,
-			Witness: make([]byte, invalidWitnessLen),
-		}
-		err := state.validateState()
-		require.ErrorContains(t, err, "invalid witness")
+	t.Run("CommandError", func(t *testing.T) {
+		converter, executor := setup(t)
+		executor.err = errors.New("boom")
+		_, _, _, err := converter.ConvertStateToProof(context.Background(), "foo.json")
+		require.ErrorIs(t, err, executor.err)
 	})
 
-	t.Run("InvalidStateHash", func(t *testing.T) {
-		state := &VMState{
-			Step:    10,
-			Exited:  true,
-			Witness: make([]byte, asteriscWitnessLen),
-		}
-		// Unknown exit code
-		state.StateHash[0] = 37
-		err := state.validateState()
-		require.ErrorContains(t, err, "invalid stateHash: unknown exitCode")
-		// Exited but ExitCode is VMStatusUnfinished
-		state.StateHash[0] = 3
-		err = state.validateState()
-		require.ErrorContains(t, err, "invalid stateHash: invalid exitCode")
-		// Not Exited but ExitCode is not VMStatusUnfinished
-		state.Exited = false
-		for exitCode := 0; exitCode < 3; exitCode++ {
-			state.StateHash[0] = byte(exitCode)
-			err = state.validateState()
-			require.ErrorContains(t, err, "invalid stateHash: invalid exitCode")
-		}
+	t.Run("InvalidOutput", func(t *testing.T) {
+		converter, executor := setup(t)
+		executor.stdOut = "blah blah"
+		_, _, _, err := converter.ConvertStateToProof(context.Background(), "foo.json")
+		require.ErrorContains(t, err, "failed to parse state data")
 	})
 }
+
+type capturingExecutor struct {
+	binary string
+	args   []string
+
+	stdOut string
+	stdErr string
+	err    error
+}
+
+func (c *capturingExecutor) exec(_ context.Context, binary string, args ...string) (string, string, error) {
+	c.binary = binary
+	c.args = args
+	return c.stdOut, c.stdErr, c.err
+}

op-challenger/runner/factory.go

@@ -40,7 +40,7 @@ func createTraceProvider(
 		return cannon.NewTraceProvider(logger, m, cfg.Cannon, serverExecutor, prestateProvider, prestate, localInputs, dir, 42), nil
 	case types.TraceTypeAsterisc:
 		serverExecutor := vm.NewOpProgramServerExecutor()
-		stateConverter := asterisc.NewStateConverter()
+		stateConverter := asterisc.NewStateConverter(cfg.Asterisc)
 		prestate, err := getPrestate(ctx, prestateHash, cfg.AsteriscAbsolutePreStateBaseURL, cfg.AsteriscAbsolutePreState, dir, stateConverter)
 		if err != nil {
 			return nil, err
@@ -49,7 +49,7 @@ func createTraceProvider(
 		return asterisc.NewTraceProvider(logger, m, cfg.Asterisc, serverExecutor, prestateProvider, prestate, localInputs, dir, 42), nil
 	case types.TraceTypeAsteriscKona:
 		serverExecutor := vm.NewKonaExecutor()
-		stateConverter := asterisc.NewStateConverter()
+		stateConverter := asterisc.NewStateConverter(cfg.Asterisc)
 		prestate, err := getPrestate(ctx, prestateHash, cfg.AsteriscKonaAbsolutePreStateBaseURL, cfg.AsteriscKonaAbsolutePreState, dir, stateConverter)
 		if err != nil {
 			return nil, err