Merge branch 'develop' into fix/rename-deployment

op-chain-ops/cmd/rollover/main.go

@@ -51,6 +51,8 @@ func main() {
 					return err
 				}
 
+				log.Info("Requires an archive node")
+
 				log.Info("Connecting to AddressManager", "address", addresses.AddressManager)
 				addressManager, err := bindings.NewAddressManager(addresses.AddressManager, clients.L1Client)
 				if err != nil {
@@ -70,28 +72,42 @@ func main() {
 					time.Sleep(3 * time.Second)
 				}
 
+				shutoffBlock, err := addressManager.GetAddress(&bind.CallOpts{}, "DTL_SHUTOFF_BLOCK")
+				if err != nil {
+					return err
+				}
+				shutoffHeight := shutoffBlock.Big()
+
 				log.Info("Connecting to CanonicalTransactionChain", "address", addresses.CanonicalTransactionChain)
 				ctc, err := legacy_bindings.NewCanonicalTransactionChain(addresses.CanonicalTransactionChain, clients.L1Client)
 				if err != nil {
 					return err
 				}
 
-				queueLength, err := ctc.GetQueueLength(&bind.CallOpts{})
+				queueLength, err := ctc.GetQueueLength(&bind.CallOpts{
+					BlockNumber: shutoffHeight,
+				})
 				if err != nil {
 					return err
 				}
 
-				totalElements, err := ctc.GetTotalElements(&bind.CallOpts{})
+				totalElements, err := ctc.GetTotalElements(&bind.CallOpts{
+					BlockNumber: shutoffHeight,
+				})
 				if err != nil {
 					return err
 				}
 
-				totalBatches, err := ctc.GetTotalBatches(&bind.CallOpts{})
+				totalBatches, err := ctc.GetTotalBatches(&bind.CallOpts{
+					BlockNumber: shutoffHeight,
+				})
 				if err != nil {
 					return err
 				}
 
-				pending, err := ctc.GetNumPendingQueueElements(&bind.CallOpts{})
+				pending, err := ctc.GetNumPendingQueueElements(&bind.CallOpts{
+					BlockNumber: shutoffHeight,
+				})
 				if err != nil {
 					return err
 				}
@@ -131,6 +147,7 @@ func main() {
 					if err != nil {
 						return err
 					}
+
 					// If the queue origin is l1, then it is a deposit.
 					if json.QueueOrigin == "l1" {
 						if json.QueueIndex == nil {
@@ -138,12 +155,26 @@ func main() {
 							return fmt.Errorf("queue index is nil for tx %s at height %d", hash.Hex(), blockNumber)
 						}
 						queueIndex := uint64(*json.QueueIndex)
+						if json.L1BlockNumber == nil {
+							// This should never happen.
+							return fmt.Errorf("L1 block number is nil for tx %s at height %d", hash.Hex(), blockNumber)
+						}
+						l1BlockNumber := json.L1BlockNumber.ToInt()
+						log.Info("Deposit found", "l2-block", blockNumber, "l1-block", l1BlockNumber, "queue-index", queueIndex)
+
+						// This should never happen
+						if json.L1BlockNumber.ToInt().Uint64() > shutoffHeight.Uint64() {
+							log.Warn("Lost deposit")
+							return fmt.Errorf("Lost deposit: %s", hash.Hex())
+						}
+
 						// Check to see if the final deposit was ingested. Subtract 1 here to handle zero
 						// indexing.
 						if queueIndex == queueLength.Uint64()-1 {
 							log.Info("Found final deposit in l2geth", "queue-index", queueIndex)
 							break
 						}
+
 						// If the queue index is less than the queue length, then not all deposits have
 						// been ingested by l2geth yet. This means that we need to reset the blocknumber
 						// to the latest block number to restart walking backwards to find deposits that
@@ -258,7 +289,7 @@ func waitForTotalElements(wg *sync.WaitGroup, contract RollupContract, client *e
 		log.Info(
 			"Waiting for elements to be submitted",
 			"name", name,
-			"count", totalElements.Uint64()-bn,
+			"count", bn-totalElements.Uint64(),
 			"height", bn,
 			"total-elements", totalElements.Uint64(),
 		)

op-wheel/cheat/cheat.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/ethereum-optimism/optimism/op-node/eth"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus/beacon"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
@@ -379,6 +380,13 @@ func SetBalance(addr common.Address, amount *big.Int) HeadFn {
 	}
 }
 
+func SetCode(addr common.Address, code hexutil.Bytes) HeadFn {
+	return func(headState *state.StateDB) error {
+		headState.SetCode(addr, code)
+		return nil
+	}
+}
+
 func SetNonce(addr common.Address, nonce uint64) HeadFn {
 	return func(headState *state.StateDB) error {
 		headState.SetNonce(addr, nonce)

op-wheel/commands.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -179,6 +180,10 @@ func addrFlag(name string, usage string) cli.GenericFlag {
 	return textFlag[*common.Address](name, usage, new(common.Address))
 }
 
+func bytesFlag(name string, usage string) cli.GenericFlag {
+	return textFlag[*hexutil.Bytes](name, usage, new(hexutil.Bytes))
+}
+
 func hashFlag(name string, usage string) cli.GenericFlag {
 	return textFlag[*common.Hash](name, usage, new(common.Hash))
 }
@@ -191,6 +196,10 @@ func addrFlagValue(name string, ctx *cli.Context) common.Address {
 	return *ctx.Generic(name).(*TextFlag[*common.Address]).Value
 }
 
+func bytesFlagValue(name string, ctx *cli.Context) hexutil.Bytes {
+	return *ctx.Generic(name).(*TextFlag[*hexutil.Bytes]).Value
+}
+
 func hashFlagValue(name string, ctx *cli.Context) common.Hash {
 	return *ctx.Generic(name).(*TextFlag[*common.Hash]).Value
 }
@@ -271,6 +280,17 @@ var (
 			return ch.RunAndClose(cheat.SetBalance(addrFlagValue("address", ctx), bigFlagValue("balance", ctx)))
 		}),
 	}
+	CheatSetCodeCmd = cli.Command{
+		Name: "code",
+		Flags: []cli.Flag{
+			DataDirFlag,
+			addrFlag("address", "Address to change code of"),
+			bytesFlag("code", "New code of the account"),
+		},
+		Action: CheatAction(false, func(ctx *cli.Context, ch *cheat.Cheater) error {
+			return ch.RunAndClose(cheat.SetCode(addrFlagValue("address", ctx), bytesFlagValue("code", ctx)))
+		}),
+	}
 	CheatSetNonceCmd = cli.Command{
 		Name: "nonce",
 		Flags: []cli.Flag{
@@ -440,6 +460,7 @@ var CheatCmd = cli.Command{
 	Subcommands: []cli.Command{
 		CheatStorageCmd,
 		CheatSetBalanceCmd,
+		CheatSetCodeCmd,
 		CheatSetNonceCmd,
 		CheatOvmOwnersCmd,
 		CheatPrintHeadBlock,

ops/docker/ci-builder/Dockerfile

@@ -97,3 +97,11 @@ RUN echo "downloading and verifying Codecov uploader" && \
   cp codecov /usr/local/bin/codecov && \
   chmod +x /usr/local/bin/codecov  && \
   rm codecov
+
+RUN echo "downloading mockery tool" && \
+  mkdir -p mockery-tmp-dir && \
+  curl -o mockery-tmp-dir/mockery.tar.gz -sL https://github.com/vektra/mockery/releases/download/v2.28.1/mockery_2.28.1_Linux_x86_64.tar.gz && \
+  tar -xzvf mockery-tmp-dir/mockery.tar.gz -C mockery-tmp-dir && \
+  cp mockery-tmp-dir/mockery /usr/local/bin/mockery && \
+  chmod +x /usr/local/bin/mockery && \
+  rm -rf mockery-tmp-dir

packages/contracts-bedrock/contracts/test/invariants/Encoding.t.sol

+// SPDX-License-Identifier: MIT
+pragma solidity 0.8.15;
+
+import { Test } from "forge-std/Test.sol";
+import { StdInvariant } from "forge-std/StdInvariant.sol";
+import { Encoding } from "../../libraries/Encoding.sol";
+
+contract Encoding_Converter {
+    bool public failedRoundtripAToB;
+    bool public failedRoundtripBToA;
+
+    /**
+     * @notice Takes a pair of integers to be encoded into a versioned nonce with the
+     *         Encoding library and then decoded and updates the test contract's state
+     *         indicating if the round trip encoding failed.
+     */
+    function convertRoundTripAToB(uint240 _nonce, uint16 _version) external {
+        // Encode the nonce and version
+        uint256 encodedVersionedNonce = Encoding.encodeVersionedNonce(_nonce, _version);
+
+        // Decode the nonce and version
+        uint240 decodedNonce;
+        uint16 decodedVersion;
+
+        (decodedNonce, decodedVersion) = Encoding.decodeVersionedNonce(encodedVersionedNonce);
+
+        // If our round trip encoding did not return the original result, set our state.
+        if ((decodedNonce != _nonce) || (decodedVersion != _version)) {
+            failedRoundtripAToB = true;
+        }
+    }
+
+    /**
+     * @notice Takes an integer representing a packed version and nonce and attempts
+     *         to decode them using the Encoding library before re-encoding and updates
+     *         the test contract's state indicating if the round trip encoding failed.
+     */
+    function convertRoundTripBToA(uint256 _versionedNonce) external {
+        // Decode the nonce and version
+        uint240 decodedNonce;
+        uint16 decodedVersion;
+
+        (decodedNonce, decodedVersion) = Encoding.decodeVersionedNonce(_versionedNonce);
+
+        // Encode the nonce and version
+        uint256 encodedVersionedNonce = Encoding.encodeVersionedNonce(decodedNonce, decodedVersion);
+
+        // If our round trip encoding did not return the original result, set our state.
+        if (encodedVersionedNonce != _versionedNonce) {
+            failedRoundtripBToA = true;
+        }
+    }
+}
+
+contract Encoding_Invariant is StdInvariant, Test {
+    Encoding_Converter internal actor;
+
+    function setUp() public {
+        // Create a converter actor.
+        actor = new Encoding_Converter();
+
+        targetContract(address(actor));
+
+        bytes4[] memory selectors = new bytes4[](2);
+        selectors[0] = actor.convertRoundTripAToB.selector;
+        selectors[1] = actor.convertRoundTripBToA.selector;
+        FuzzSelector memory selector = FuzzSelector({ addr: address(actor), selectors: selectors });
+        targetSelector(selector);
+    }
+
+    /**
+     * @custom:invariant `convertRoundTripAToB` never fails.
+     *
+     * Asserts that a raw versioned nonce can be encoded / decoded to reach the same raw value.
+     */
+    function invariant_round_trip_encoding_AToB() external {
+        // ASSERTION: The round trip encoding done in testRoundTripAToB(...)
+        assertEq(actor.failedRoundtripAToB(), false);
+    }
+
+    /**
+     * @custom:invariant `convertRoundTripBToA` never fails.
+     *
+     * Asserts that an encoded versioned nonce can always be decoded / re-encoded to reach
+     * the same encoded value.
+     */
+    function invariant_round_trip_encoding_BToA() external {
+        // ASSERTION: The round trip encoding done in testRoundTripBToA should never
+        // fail.
+        assertEq(actor.failedRoundtripBToA(), false);
+    }
+}

packages/contracts-bedrock/invariant-docs/Encoding.md

 # `Encoding` Invariants
 
+## `convertRoundTripAToB` never fails.
+**Test:** [`Encoding.t.sol#L76`](../contracts/test/invariants/Encoding.t.sol#L76)
+
+Asserts that a raw versioned nonce can be encoded / decoded to reach the same raw value. 
+
+
+## `convertRoundTripBToA` never fails.
+**Test:** [`Encoding.t.sol#L87`](../contracts/test/invariants/Encoding.t.sol#L87)
+
+Asserts that an encoded versioned nonce can always be decoded / re-encoded to reach the same encoded value. 
+
+
 ## `testRoundTripAToB` never fails.
 **Test:** [`FuzzEncoding.sol#L56`](../contracts/echidna/FuzzEncoding.sol#L56)
 

proxyd/integration_tests/consensus_test.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path"
 	"testing"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
 
@@ -630,6 +631,55 @@ func TestConsensus(t *testing.T) {
 		require.Equal(t, len(nodes["node2"].mockBackend.Requests()), 0, msg)
 	})
 
+	t.Run("load balancing should not hit if node is degraded", func(t *testing.T) {
+		reset()
+		useOnlyNode1()
+
+		// replace node1 handler with one that adds a 500ms delay
+		oldHandler := nodes["node1"].mockBackend.handler
+		defer func() { nodes["node1"].mockBackend.handler = oldHandler }()
+
+		nodes["node1"].mockBackend.SetHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			time.Sleep(500 * time.Millisecond)
+			oldHandler.ServeHTTP(w, r)
+		}))
+
+		update()
+
+		// send 10 requests to make node1 degraded
+		numberReqs := 10
+		for numberReqs > 0 {
+			_, statusCode, err := client.SendRPC("eth_getBlockByNumber", []interface{}{"0x101", false})
+			require.NoError(t, err)
+			require.Equal(t, 200, statusCode)
+			numberReqs--
+		}
+
+		// bring back node2
+		nodes["node2"].handler.ResetOverrides()
+		update()
+
+		// reset request counts
+		nodes["node1"].mockBackend.Reset()
+		nodes["node2"].mockBackend.Reset()
+
+		require.Equal(t, 0, len(nodes["node1"].mockBackend.Requests()))
+		require.Equal(t, 0, len(nodes["node2"].mockBackend.Requests()))
+
+		numberReqs = 10
+		for numberReqs > 0 {
+			_, statusCode, err := client.SendRPC("eth_getBlockByNumber", []interface{}{"0x101", false})
+			require.NoError(t, err)
+			require.Equal(t, 200, statusCode)
+			numberReqs--
+		}
+
+		msg := fmt.Sprintf("n1 %d, n2 %d",
+			len(nodes["node1"].mockBackend.Requests()), len(nodes["node2"].mockBackend.Requests()))
+		require.Equal(t, 0, len(nodes["node1"].mockBackend.Requests()), msg)
+		require.Equal(t, 10, len(nodes["node2"].mockBackend.Requests()), msg)
+	})
+
 	t.Run("rewrite response of eth_blockNumber", func(t *testing.T) {
 		reset()
 		update()

proxyd/integration_tests/testdata/consensus.toml

@@ -3,6 +3,7 @@ rpc_port = 8545
 
 [backend]
 response_timeout_seconds = 1
+max_degraded_latency_threshold = "30ms"
 
 [backends]
 [backends.node1]

proxyd/metrics.go

@@ -358,6 +358,14 @@ var (
 		"backend_name",
 	})
 
+	degradedBackends = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: MetricsNamespace,
+		Name:      "backend_degraded",
+		Help:      "Bool gauge for degraded backends",
+	}, []string{
+		"backend_name",
+	})
+
 	networkErrorRateBackend = promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: MetricsNamespace,
 		Name:      "backend_error_rate",
@@ -493,6 +501,7 @@ func RecordConsensusBackendUpdateDelay(b *Backend, lastUpdate time.Time) {
 
 func RecordBackendNetworkLatencyAverageSlidingWindow(b *Backend, avgLatency time.Duration) {
 	avgLatencyBackend.WithLabelValues(b.Name).Set(float64(avgLatency.Milliseconds()))
+	degradedBackends.WithLabelValues(b.Name).Set(boolToFloat64(b.IsDegraded()))
 }
 
 func RecordBackendNetworkErrorRateSlidingWindow(b *Backend, rate float64) {