op-program: Switch fpp-verify back to running in separate process. (#12201)

Trying to reproduce the missing prestate failure.  Also now runs hourly instead of 4 hourly to increase chances of finding a failing case.

.circleci/config.yml

@@ -1677,7 +1677,7 @@ workflows:
 
   scheduled-fpp:
     when:
-      equal: [ build_four_hours, <<pipeline.schedule.name>> ]
+      equal: [ build_hourly, <<pipeline.schedule.name>> ]
     jobs:
       - fpp-verify:
           context:

op-program/verify/verify.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math/big"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -26,6 +27,8 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
+const runInProcess = false
+
 type Runner struct {
 	l1RpcUrl    string
 	l1RpcKind   string
@@ -99,7 +102,7 @@ func (r *Runner) RunBetweenBlocks(ctx context.Context, l1Head common.Hash, start
 		return fmt.Errorf("failed to find ending block info: %w", err)
 	}
 
-	return r.run(l1Head, agreedBlockInfo, agreedOutputRoot, claimedOutputRoot, claimedBlockInfo)
+	return r.run(ctx, l1Head, agreedBlockInfo, agreedOutputRoot, claimedOutputRoot, claimedBlockInfo)
 }
 
 func (r *Runner) createL2Client(ctx context.Context) (*sources.L2Client, error) {
@@ -157,10 +160,10 @@ func (r *Runner) RunToFinalized(ctx context.Context) error {
 		return fmt.Errorf("failed to find ending block info: %w", err)
 	}
 
-	return r.run(l1Head.Hash(), agreedBlockInfo, agreedOutputRoot, claimedOutputRoot, claimedBlockInfo)
+	return r.run(ctx, l1Head.Hash(), agreedBlockInfo, agreedOutputRoot, claimedOutputRoot, claimedBlockInfo)
 }
 
-func (r *Runner) run(l1Head common.Hash, agreedBlockInfo eth.BlockInfo, agreedOutputRoot common.Hash, claimedOutputRoot common.Hash, claimedBlockInfo eth.BlockInfo) error {
+func (r *Runner) run(ctx context.Context, l1Head common.Hash, agreedBlockInfo eth.BlockInfo, agreedOutputRoot common.Hash, claimedOutputRoot common.Hash, claimedBlockInfo eth.BlockInfo) error {
 	var err error
 	if r.dataDir == "" {
 		r.dataDir, err = os.MkdirTemp("", "oracledata")
@@ -199,31 +202,64 @@ func (r *Runner) run(l1Head common.Hash, agreedBlockInfo eth.BlockInfo, agreedOu
 	}
 	fmt.Printf("Configuration: %s\n", argsStr)
 
-	offlineCfg := config.NewConfig(
-		r.rollupCfg, r.chainCfg, l1Head, agreedBlockInfo.Hash(), agreedOutputRoot, claimedOutputRoot, claimedBlockInfo.NumberU64())
-	offlineCfg.DataDir = r.dataDir
-	onlineCfg := *offlineCfg
-	onlineCfg.L1URL = r.l1RpcUrl
-	onlineCfg.L1BeaconURL = r.l1BeaconUrl
-	onlineCfg.L2URL = r.l2RpcUrl
-	if r.l1RpcKind != "" {
-		onlineCfg.L1RPCKind = sources.RPCProviderKind(r.l1RpcKind)
-	}
+	if runInProcess {
+		offlineCfg := config.NewConfig(
+			r.rollupCfg, r.chainCfg, l1Head, agreedBlockInfo.Hash(), agreedOutputRoot, claimedOutputRoot, claimedBlockInfo.NumberU64())
+		offlineCfg.DataDir = r.dataDir
 
-	fmt.Println("Running in online mode")
-	err = host.Main(oplog.NewLogger(os.Stderr, r.logCfg), &onlineCfg)
-	if err != nil {
-		return fmt.Errorf("online mode failed: %w", err)
-	}
+		onlineCfg := *offlineCfg
+		onlineCfg.L1URL = r.l1RpcUrl
+		onlineCfg.L1BeaconURL = r.l1BeaconUrl
+		onlineCfg.L2URL = r.l2RpcUrl
+		if r.l1RpcKind != "" {
+			onlineCfg.L1RPCKind = sources.RPCProviderKind(r.l1RpcKind)
+		}
 
-	fmt.Println("Running in offline mode")
-	err = host.Main(oplog.NewLogger(os.Stderr, r.logCfg), offlineCfg)
-	if err != nil {
-		return fmt.Errorf("offline mode failed: %w", err)
+		fmt.Println("Running in online mode")
+		err = host.Main(oplog.NewLogger(os.Stderr, r.logCfg), &onlineCfg)
+		if err != nil {
+			return fmt.Errorf("online mode failed: %w", err)
+		}
+
+		fmt.Println("Running in offline mode")
+		err = host.Main(oplog.NewLogger(os.Stderr, r.logCfg), offlineCfg)
+		if err != nil {
+			return fmt.Errorf("offline mode failed: %w", err)
+		}
+	} else {
+		fmt.Println("Running in online mode")
+		onlineArgs := make([]string, len(args))
+		copy(onlineArgs, args)
+		onlineArgs = append(onlineArgs,
+			"--l1", r.l1RpcUrl,
+			"--l1.beacon", r.l1BeaconUrl,
+			"--l2", r.l2RpcUrl)
+		if r.l1RpcKind != "" {
+			onlineArgs = append(onlineArgs, "--l1.rpckind", r.l1RpcKind)
+		}
+		err = runFaultProofProgram(ctx, onlineArgs)
+		if err != nil {
+			return fmt.Errorf("online mode failed: %w", err)
+		}
+
+		fmt.Println("Running in offline mode")
+		err = runFaultProofProgram(ctx, args)
+		if err != nil {
+			return fmt.Errorf("offline mode failed: %w", err)
+		}
 	}
 	return nil
 }
 
+func runFaultProofProgram(ctx context.Context, args []string) error {
+	ctx, cancel := context.WithTimeout(ctx, 60*time.Minute)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "./bin/op-program", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
 func outputAtBlockNum(ctx context.Context, l2Client *sources.L2Client, blockNum uint64) (eth.BlockInfo, common.Hash, error) {
 	startBlockInfo, err := l2Client.InfoByNumber(ctx, blockNum)
 	if err != nil {