op-chain-ops: better rollover checks

Improve the rollover script for waiting for the final deposits
to be ingested into `l2geth` by ensuring that it handles the case
that not all deposits have been ingested in a safe way. Previously
the error message was ominous, which leads to confusion when running
the script. Now instead of erroring, reset the loop and attempt to
look for the final deposit again. Also add a bunch of comments
to make the code easier to understand.

This script is used as part of the upgrade to bedrock.

op-chain-ops/cmd/rollover/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"math/big"
 	"os"
@@ -112,6 +111,7 @@ func main() {
 				}
 				log.Info("Searching backwards for final deposit", "start", blockNumber)
 
+				// Walk backards through the blocks until we find the final deposit.
 				for {
 					bn := new(big.Int).SetUint64(blockNumber)
 					log.Info("Checking L2 block", "number", bn)
@@ -131,18 +131,35 @@ func main() {
 					if err != nil {
 						return err
 					}
+					// If the queue origin is l1, then it is a deposit.
 					if json.QueueOrigin == "l1" {
 						if json.QueueIndex == nil {
-							// This should never happen
-							return errors.New("queue index is nil")
+							// This should never happen.
+							return fmt.Errorf("queue index is nil for tx %s at height %d", hash.Hex(), blockNumber)
 						}
 						queueIndex := uint64(*json.QueueIndex)
+						// Check to see if the final deposit was ingested. Subtract 1 here to handle zero
+						// indexing.
 						if queueIndex == queueLength.Uint64()-1 {
 							log.Info("Found final deposit in l2geth", "queue-index", queueIndex)
 							break
 						}
+						// If the queue index is less than the queue length, then not all deposits have
+						// been ingested by l2geth yet. This means that we need to reset the blocknumber
+						// to the latest block number to restart walking backwards to find deposits that
+						// have yet to be ingested.
 						if queueIndex < queueLength.Uint64() {
-							return errors.New("missed final deposit")
+							log.Info("Not all deposits ingested", "queue-index", queueIndex, "queue-length", queueLength.Uint64())
+							time.Sleep(time.Second * 3)
+							blockNumber, err = clients.L2Client.BlockNumber(context.Background())
+							if err != nil {
+								return err
+							}
+							continue
+						}
+						// The queueIndex should never be greater than the queue length.
+						if queueIndex > queueLength.Uint64() {
+							log.Warn("Queue index is greater than queue length", "queue-index", queueIndex, "queue-length", queueLength.Uint64())
 						}
 					}
 					blockNumber--