op-node: attempt recovery from odd forkchoice states (#11462)

* op-node: attempt recovery from odd forkchoice states

* op-node: adjust recovery heuristic to express distance in number of sequence windows

op-node/rollup/sync/start.go

@@ -54,6 +54,13 @@ var TooDeepReorgErr = errors.New("reorg is too deep")
 
 const MaxReorgSeqWindows = 5
 
+// RecoverMinSeqWindows is the number of sequence windows
+// between the unsafe head L1 origin, and the finalized block, while finality is still at genesis,
+// that need to elapse to heuristically recover from a missing forkchoice state.
+// Note that in healthy node circumstances finality should have been forced a long time ago,
+// since blocks are force-inserted after a full sequence window.
+const RecoverMinSeqWindows = 14
+
 type FindHeadsResult struct {
 	Unsafe    eth.L2BlockRef
 	Safe      eth.L2BlockRef
@@ -114,6 +121,25 @@ func FindL2Heads(ctx context.Context, cfg *rollup.Config, l1 L1Chain, l2 L2Chain
 	lgr.Info("Loaded current L2 heads", "unsafe", result.Unsafe, "safe", result.Safe, "finalized", result.Finalized,
 		"unsafe_origin", result.Unsafe.L1Origin, "safe_origin", result.Safe.L1Origin)
 
+	// Check if the execution engine completed sync, but left the forkchoice finalized & safe heads at genesis.
+	// This is the equivalent of the "syncStatusFinishedELButNotFinalized" post-processing in the engine controller.
+	if result.Finalized.Hash == cfg.Genesis.L2.Hash &&
+		result.Safe.Hash == cfg.Genesis.L2.Hash &&
+		result.Unsafe.Number > cfg.Genesis.L2.Number &&
+		result.Unsafe.L1Origin.Number > cfg.Genesis.L1.Number+(RecoverMinSeqWindows*cfg.SeqWindowSize) {
+		lgr.Warn("Attempting recovery from sync state without finality.", "head", result.Unsafe)
+		return &FindHeadsResult{Unsafe: result.Unsafe, Safe: result.Unsafe, Finalized: result.Unsafe}, nil
+	}
+
+	// Check if the execution engine corrupted, and forkchoice is ahead of the remaining chain:
+	// in this case we must go back to the prior head, to reprocess the pruned finalized/safe data.
+	if result.Unsafe.Number < result.Finalized.Number || result.Unsafe.Number < result.Safe.Number {
+		lgr.Error("Unsafe head is behind known finalized/safe blocks, "+
+			"execution-engine chain must have been rewound without forkchoice update. Attempting recovery now.",
+			"unsafe_head", result.Unsafe, "safe_head", result.Safe, "finalized_head", result.Finalized)
+		return &FindHeadsResult{Unsafe: result.Unsafe, Safe: result.Unsafe, Finalized: result.Unsafe}, nil
+	}
+
 	// Remember original unsafe block to determine reorg depth
 	prevUnsafe := result.Unsafe