Merge pull request #5873 from ethereum-optimism/refcell/backoffclient

feat(op-service): Retrying client

op-program/host/host.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"math"
 	"os"
 	"os/exec"
 
@@ -21,10 +22,13 @@ import (
 	oppio "github.com/ethereum-optimism/optimism/op-program/io"
 	"github.com/ethereum-optimism/optimism/op-program/preimage"
 	opservice "github.com/ethereum-optimism/optimism/op-service"
+	opclient "github.com/ethereum-optimism/optimism/op-service/client"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 )
 
+const maxRPCRetries = math.MaxInt
+
 type L2Source struct {
 	*sources.L2Client
 	*sources.DebugClient
@@ -199,16 +203,21 @@ func makePrefetcher(ctx context.Context, logger log.Logger, kv kvstore.KV, cfg *
 		return nil, fmt.Errorf("failed to setup L2 RPC: %w", err)
 	}
 
+	l1Backoff := opclient.NewRetryingClient(l1RPC, maxRPCRetries)
+	l2Backoff := opclient.NewRetryingClient(l2RPC, maxRPCRetries)
+
 	l1ClCfg := sources.L1ClientDefaultConfig(cfg.Rollup, cfg.L1TrustRPC, cfg.L1RPCKind)
-	l2ClCfg := sources.L2ClientDefaultConfig(cfg.Rollup, true)
-	l1Cl, err := sources.NewL1Client(l1RPC, logger, nil, l1ClCfg)
+	l1Cl, err := sources.NewL1Client(l1Backoff, logger, nil, l1ClCfg)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create L1 client: %w", err)
 	}
-	l2Cl, err := sources.NewL2Client(l2RPC, logger, nil, l2ClCfg)
+
+	l2ClCfg := sources.L2ClientDefaultConfig(cfg.Rollup, true)
+	l2Cl, err := sources.NewL2Client(l2Backoff, logger, nil, l2ClCfg)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create L2 client: %w", err)
 	}
+
 	l2DebugCl := &L2Source{L2Client: l2Cl, DebugClient: sources.NewDebugClient(l2RPC.CallContext)}
 	return prefetcher.NewPrefetcher(logger, l1Cl, l2DebugCl, kv), nil
 }

op-program/host/prefetcher/prefetcher.go

@@ -42,8 +42,8 @@ type Prefetcher struct {
 func NewPrefetcher(logger log.Logger, l1Fetcher L1Source, l2Fetcher L2Source, kvStore kvstore.KV) *Prefetcher {
 	return &Prefetcher{
 		logger:    logger,
-		l1Fetcher: NewRetryingL1Source(logger, l1Fetcher),
-		l2Fetcher: NewRetryingL2Source(logger, l2Fetcher),
+		l1Fetcher: l1Fetcher,
+		l2Fetcher: l2Fetcher,
 		kvStore:   kvStore,
 	}
 }

op-program/host/prefetcher/retry.go

-package prefetcher
-
-import (
-	"context"
-	"math"
-
-	"github.com/ethereum-optimism/optimism/op-node/eth"
-	"github.com/ethereum-optimism/optimism/op-service/backoff"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/log"
-)
-
-const maxAttempts = math.MaxInt // Succeed or die trying
-
-type RetryingL1Source struct {
-	logger   log.Logger
-	source   L1Source
-	strategy backoff.Strategy
-}
-
-func NewRetryingL1Source(logger log.Logger, source L1Source) *RetryingL1Source {
-	return &RetryingL1Source{
-		logger:   logger,
-		source:   source,
-		strategy: backoff.Exponential(),
-	}
-}
-
-func (s *RetryingL1Source) InfoByHash(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, error) {
-	var info eth.BlockInfo
-	err := backoff.DoCtx(ctx, maxAttempts, s.strategy, func() error {
-		res, err := s.source.InfoByHash(ctx, blockHash)
-		if err != nil {
-			s.logger.Warn("Failed to retrieve info", "hash", blockHash, "err", err)
-			return err
-		}
-		info = res
-		return nil
-	})
-	return info, err
-}
-
-func (s *RetryingL1Source) InfoAndTxsByHash(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, types.Transactions, error) {
-	var info eth.BlockInfo
-	var txs types.Transactions
-	err := backoff.DoCtx(ctx, maxAttempts, s.strategy, func() error {
-		i, t, err := s.source.InfoAndTxsByHash(ctx, blockHash)
-		if err != nil {
-			s.logger.Warn("Failed to retrieve l1 info and txs", "hash", blockHash, "err", err)
-			return err
-		}
-		info = i
-		txs = t
-		return nil
-	})
-	return info, txs, err
-}
-
-func (s *RetryingL1Source) FetchReceipts(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, types.Receipts, error) {
-	var info eth.BlockInfo
-	var rcpts types.Receipts
-	err := backoff.DoCtx(ctx, maxAttempts, s.strategy, func() error {
-		i, r, err := s.source.FetchReceipts(ctx, blockHash)
-		if err != nil {
-			s.logger.Warn("Failed to fetch receipts", "hash", blockHash, "err", err)
-			return err
-		}
-		info = i
-		rcpts = r
-		return nil
-	})
-	return info, rcpts, err
-}
-
-var _ L1Source = (*RetryingL1Source)(nil)
-
-type RetryingL2Source struct {
-	logger   log.Logger
-	source   L2Source
-	strategy backoff.Strategy
-}
-
-func (s *RetryingL2Source) InfoAndTxsByHash(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, types.Transactions, error) {
-	var info eth.BlockInfo
-	var txs types.Transactions
-	err := backoff.DoCtx(ctx, maxAttempts, s.strategy, func() error {
-		i, t, err := s.source.InfoAndTxsByHash(ctx, blockHash)
-		if err != nil {
-			s.logger.Warn("Failed to retrieve l2 info and txs", "hash", blockHash, "err", err)
-			return err
-		}
-		info = i
-		txs = t
-		return nil
-	})
-	return info, txs, err
-}
-
-func (s *RetryingL2Source) NodeByHash(ctx context.Context, hash common.Hash) ([]byte, error) {
-	var node []byte
-	err := backoff.DoCtx(ctx, maxAttempts, s.strategy, func() error {
-		n, err := s.source.NodeByHash(ctx, hash)
-		if err != nil {
-			s.logger.Warn("Failed to retrieve node", "hash", hash, "err", err)
-			return err
-		}
-		node = n
-		return nil
-	})
-	return node, err
-}
-
-func (s *RetryingL2Source) CodeByHash(ctx context.Context, hash common.Hash) ([]byte, error) {
-	var code []byte
-	err := backoff.DoCtx(ctx, maxAttempts, s.strategy, func() error {
-		c, err := s.source.CodeByHash(ctx, hash)
-		if err != nil {
-			s.logger.Warn("Failed to retrieve code", "hash", hash, "err", err)
-			return err
-		}
-		code = c
-		return nil
-	})
-	return code, err
-}
-
-func NewRetryingL2Source(logger log.Logger, source L2Source) *RetryingL2Source {
-	return &RetryingL2Source{
-		logger:   logger,
-		source:   source,
-		strategy: backoff.Exponential(),
-	}
-}
-
-var _ L2Source = (*RetryingL2Source)(nil)

op-program/host/prefetcher/retry_test.go

-package prefetcher
-
-import (
-	"context"
-	"errors"
-	"testing"
-
-	"github.com/ethereum-optimism/optimism/op-node/eth"
-	"github.com/ethereum-optimism/optimism/op-node/testlog"
-	"github.com/ethereum-optimism/optimism/op-node/testutils"
-	"github.com/ethereum-optimism/optimism/op-service/backoff"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/stretchr/testify/mock"
-	"github.com/stretchr/testify/require"
-)
-
-func TestRetryingL1Source(t *testing.T) {
-	ctx := context.Background()
-	hash := common.Hash{0xab}
-	info := &testutils.MockBlockInfo{InfoHash: hash}
-	// The mock really doesn't like returning nil for a eth.BlockInfo so return a value we expect to be ignored instead
-	wrongInfo := &testutils.MockBlockInfo{InfoHash: common.Hash{0x99}}
-	txs := types.Transactions{
-		&types.Transaction{},
-	}
-	rcpts := types.Receipts{
-		&types.Receipt{},
-	}
-
-	t.Run("InfoByHash Success", func(t *testing.T) {
-		source, mock := createL1Source(t)
-		defer mock.AssertExpectations(t)
-		mock.ExpectInfoByHash(hash, info, nil)
-
-		result, err := source.InfoByHash(ctx, hash)
-		require.NoError(t, err)
-		require.Equal(t, info, result)
-	})
-
-	t.Run("InfoByHash Error", func(t *testing.T) {
-		source, mock := createL1Source(t)
-		defer mock.AssertExpectations(t)
-		expectedErr := errors.New("boom")
-		mock.ExpectInfoByHash(hash, wrongInfo, expectedErr)
-		mock.ExpectInfoByHash(hash, info, nil)
-
-		result, err := source.InfoByHash(ctx, hash)
-		require.NoError(t, err)
-		require.Equal(t, info, result)
-	})
-
-	t.Run("InfoAndTxsByHash Success", func(t *testing.T) {
-		source, mock := createL1Source(t)
-		defer mock.AssertExpectations(t)
-		mock.ExpectInfoAndTxsByHash(hash, info, txs, nil)
-
-		actualInfo, actualTxs, err := source.InfoAndTxsByHash(ctx, hash)
-		require.NoError(t, err)
-		require.Equal(t, info, actualInfo)
-		require.Equal(t, txs, actualTxs)
-	})
-
-	t.Run("InfoAndTxsByHash Error", func(t *testing.T) {
-		source, mock := createL1Source(t)
-		defer mock.AssertExpectations(t)
-		expectedErr := errors.New("boom")
-		mock.ExpectInfoAndTxsByHash(hash, wrongInfo, nil, expectedErr)
-		mock.ExpectInfoAndTxsByHash(hash, info, txs, nil)
-
-		actualInfo, actualTxs, err := source.InfoAndTxsByHash(ctx, hash)
-		require.NoError(t, err)
-		require.Equal(t, info, actualInfo)
-		require.Equal(t, txs, actualTxs)
-	})
-
-	t.Run("FetchReceipts Success", func(t *testing.T) {
-		source, mock := createL1Source(t)
-		defer mock.AssertExpectations(t)
-		mock.ExpectFetchReceipts(hash, info, rcpts, nil)
-
-		actualInfo, actualRcpts, err := source.FetchReceipts(ctx, hash)
-		require.NoError(t, err)
-		require.Equal(t, info, actualInfo)
-		require.Equal(t, rcpts, actualRcpts)
-	})
-
-	t.Run("FetchReceipts Error", func(t *testing.T) {
-		source, mock := createL1Source(t)
-		defer mock.AssertExpectations(t)
-		expectedErr := errors.New("boom")
-		mock.ExpectFetchReceipts(hash, wrongInfo, nil, expectedErr)
-		mock.ExpectFetchReceipts(hash, info, rcpts, nil)
-
-		actualInfo, actualRcpts, err := source.FetchReceipts(ctx, hash)
-		require.NoError(t, err)
-		require.Equal(t, info, actualInfo)
-		require.Equal(t, rcpts, actualRcpts)
-	})
-}
-
-func createL1Source(t *testing.T) (*RetryingL1Source, *testutils.MockL1Source) {
-	logger := testlog.Logger(t, log.LvlDebug)
-	mock := &testutils.MockL1Source{}
-	source := NewRetryingL1Source(logger, mock)
-	// Avoid sleeping in tests by using a fixed backoff strategy with no delay
-	source.strategy = backoff.Fixed(0)
-	return source, mock
-}
-
-func TestRetryingL2Source(t *testing.T) {
-	ctx := context.Background()
-	hash := common.Hash{0xab}
-	info := &testutils.MockBlockInfo{InfoHash: hash}
-	// The mock really doesn't like returning nil for a eth.BlockInfo so return a value we expect to be ignored instead
-	wrongInfo := &testutils.MockBlockInfo{InfoHash: common.Hash{0x99}}
-	txs := types.Transactions{
-		&types.Transaction{},
-	}
-	data := []byte{1, 2, 3, 4, 5}
-
-	t.Run("InfoAndTxsByHash Success", func(t *testing.T) {
-		source, mock := createL2Source(t)
-		defer mock.AssertExpectations(t)
-		mock.ExpectInfoAndTxsByHash(hash, info, txs, nil)
-
-		actualInfo, actualTxs, err := source.InfoAndTxsByHash(ctx, hash)
-		require.NoError(t, err)
-		require.Equal(t, info, actualInfo)
-		require.Equal(t, txs, actualTxs)
-	})
-
-	t.Run("InfoAndTxsByHash Error", func(t *testing.T) {
-		source, mock := createL2Source(t)
-		defer mock.AssertExpectations(t)
-		expectedErr := errors.New("boom")
-		mock.ExpectInfoAndTxsByHash(hash, wrongInfo, nil, expectedErr)
-		mock.ExpectInfoAndTxsByHash(hash, info, txs, nil)
-
-		actualInfo, actualTxs, err := source.InfoAndTxsByHash(ctx, hash)
-		require.NoError(t, err)
-		require.Equal(t, info, actualInfo)
-		require.Equal(t, txs, actualTxs)
-	})
-
-	t.Run("NodeByHash Success", func(t *testing.T) {
-		source, mock := createL2Source(t)
-		defer mock.AssertExpectations(t)
-		mock.ExpectNodeByHash(hash, data, nil)
-
-		actual, err := source.NodeByHash(ctx, hash)
-		require.NoError(t, err)
-		require.Equal(t, data, actual)
-	})
-
-	t.Run("NodeByHash Error", func(t *testing.T) {
-		source, mock := createL2Source(t)
-		defer mock.AssertExpectations(t)
-		expectedErr := errors.New("boom")
-		mock.ExpectNodeByHash(hash, nil, expectedErr)
-		mock.ExpectNodeByHash(hash, data, nil)
-
-		actual, err := source.NodeByHash(ctx, hash)
-		require.NoError(t, err)
-		require.Equal(t, data, actual)
-	})
-
-	t.Run("CodeByHash Success", func(t *testing.T) {
-		source, mock := createL2Source(t)
-		defer mock.AssertExpectations(t)
-		mock.ExpectCodeByHash(hash, data, nil)
-
-		actual, err := source.CodeByHash(ctx, hash)
-		require.NoError(t, err)
-		require.Equal(t, data, actual)
-	})
-
-	t.Run("CodeByHash Error", func(t *testing.T) {
-		source, mock := createL2Source(t)
-		defer mock.AssertExpectations(t)
-		expectedErr := errors.New("boom")
-		mock.ExpectCodeByHash(hash, nil, expectedErr)
-		mock.ExpectCodeByHash(hash, data, nil)
-
-		actual, err := source.CodeByHash(ctx, hash)
-		require.NoError(t, err)
-		require.Equal(t, data, actual)
-	})
-}
-
-func createL2Source(t *testing.T) (*RetryingL2Source, *MockL2Source) {
-	logger := testlog.Logger(t, log.LvlDebug)
-	mock := &MockL2Source{}
-	source := NewRetryingL2Source(logger, mock)
-	// Avoid sleeping in tests by using a fixed backoff strategy with no delay
-	source.strategy = backoff.Fixed(0)
-	return source, mock
-}
-
-type MockL2Source struct {
-	mock.Mock
-}
-
-func (m *MockL2Source) InfoAndTxsByHash(ctx context.Context, blockHash common.Hash) (eth.BlockInfo, types.Transactions, error) {
-	out := m.Mock.MethodCalled("InfoAndTxsByHash", blockHash)
-	return out[0].(eth.BlockInfo), out[1].(types.Transactions), *out[2].(*error)
-}
-
-func (m *MockL2Source) NodeByHash(ctx context.Context, hash common.Hash) ([]byte, error) {
-	out := m.Mock.MethodCalled("NodeByHash", hash)
-	return out[0].([]byte), *out[1].(*error)
-}
-
-func (m *MockL2Source) CodeByHash(ctx context.Context, hash common.Hash) ([]byte, error) {
-	out := m.Mock.MethodCalled("CodeByHash", hash)
-	return out[0].([]byte), *out[1].(*error)
-}
-
-func (m *MockL2Source) ExpectInfoAndTxsByHash(blockHash common.Hash, info eth.BlockInfo, txs types.Transactions, err error) {
-	m.Mock.On("InfoAndTxsByHash", blockHash).Once().Return(info, txs, &err)
-}
-
-func (m *MockL2Source) ExpectNodeByHash(hash common.Hash, node []byte, err error) {
-	m.Mock.On("NodeByHash", hash).Once().Return(node, &err)
-}
-
-func (m *MockL2Source) ExpectCodeByHash(hash common.Hash, code []byte, err error) {
-	m.Mock.On("CodeByHash", hash).Once().Return(code, &err)
-}
-
-var _ L2Source = (*MockL2Source)(nil)

op-service/client/retry.go

+package client
+
+import (
+	"context"
+	"time"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/rpc"
+
+	"github.com/ethereum-optimism/optimism/op-node/client"
+	"github.com/ethereum-optimism/optimism/op-service/backoff"
+)
+
+var (
+	// ExponentialBackoff is the default backoff strategy.
+	ExponentialBackoff = backoff.Exponential()
+)
+
+// retryingClient wraps a [client.RPC] with a backoff strategy.
+type retryingClient struct {
+	c             client.RPC
+	retryAttempts int
+	strategy      backoff.Strategy
+}
+
+// NewRetryingClient creates a new retrying client.
+// The backoff strategy is optional, if not provided, the default exponential backoff strategy is used.
+func NewRetryingClient(c client.RPC, retries int, strategy ...backoff.Strategy) *retryingClient {
+	if len(strategy) == 0 {
+		strategy = []backoff.Strategy{ExponentialBackoff}
+	}
+	return &retryingClient{
+		c:             c,
+		retryAttempts: retries,
+		strategy:      strategy[0],
+	}
+}
+
+// BackoffStrategy returns the [backoff.Strategy] used by the client.
+func (b *retryingClient) BackoffStrategy() backoff.Strategy {
+	return b.strategy
+}
+
+func (b *retryingClient) Close() {
+	b.c.Close()
+}
+
+func (b *retryingClient) CallContext(ctx context.Context, result any, method string, args ...any) error {
+	return backoff.DoCtx(ctx, b.retryAttempts, b.strategy, func() error {
+		cCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		defer cancel()
+		return b.c.CallContext(cCtx, result, method, args...)
+	})
+}
+
+func (b *retryingClient) BatchCallContext(ctx context.Context, batch []rpc.BatchElem) error {
+	return backoff.DoCtx(ctx, b.retryAttempts, b.strategy, func() error {
+		cCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
+		defer cancel()
+		err := b.c.BatchCallContext(cCtx, batch)
+		return err
+	})
+}
+
+func (b *retryingClient) EthSubscribe(ctx context.Context, channel any, args ...any) (ethereum.Subscription, error) {
+	var sub ethereum.Subscription
+	err := backoff.DoCtx(ctx, b.retryAttempts, b.strategy, func() error {
+		var err error
+		sub, err = b.c.EthSubscribe(ctx, channel, args...)
+		return err
+	})
+	return sub, err
+}

op-service/client/retry_test.go

+package client_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/rpc"
+
+	"github.com/ethereum-optimism/optimism/op-service/backoff"
+	"github.com/ethereum-optimism/optimism/op-service/client"
+
+	opclient "github.com/ethereum-optimism/optimism/op-node/client"
+)
+
+type MockRPC struct {
+	mock.Mock
+}
+
+func (m *MockRPC) Close() {
+	m.Called()
+}
+
+func (m *MockRPC) CallContext(ctx context.Context, result any, method string, args ...any) error {
+	out := m.Mock.MethodCalled("CallContext", ctx, result, method, args)
+	return *out[0].(*error)
+}
+
+func (m *MockRPC) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error {
+	out := m.Mock.MethodCalled("BatchCallContext", ctx, b)
+	return *out[0].(*error)
+}
+
+func (m *MockRPC) EthSubscribe(ctx context.Context, channel any, args ...any) (ethereum.Subscription, error) {
+	out := m.Mock.MethodCalled("EthSubscribe", ctx, channel, args)
+	return *out[0].(*ethereum.Subscription), *out[1].(*error)
+}
+
+func (m *MockRPC) ExpectCallContext(err error, result any, method string, arg string) {
+	m.On("CallContext", mock.Anything, result, method, []interface{}{arg}).Return(&err)
+}
+
+func (m *MockRPC) ExpectBatchCallContext(err error, b []rpc.BatchElem) {
+	m.On("BatchCallContext", mock.Anything, b).Return(&err)
+}
+
+func (m *MockRPC) ExpectEthSubscribe(sub ethereum.Subscription, err error, channel any, args ...any) {
+	m.On("EthSubscribe", mock.Anything, channel, args).Return(&sub, &err)
+}
+
+var _ opclient.RPC = (*MockRPC)(nil)
+
+func TestClient_BackoffClient_Strategy(t *testing.T) {
+	mockRpc := &MockRPC{}
+	backoffClient := client.NewRetryingClient(mockRpc, 0)
+	require.Equal(t, backoffClient.BackoffStrategy(), client.ExponentialBackoff)
+
+	fixedStrategy := &backoff.FixedStrategy{}
+	backoffClient = client.NewRetryingClient(mockRpc, 0, fixedStrategy)
+	require.Equal(t, backoffClient.BackoffStrategy(), fixedStrategy)
+}
+
+func TestClient_BackoffClient_Close(t *testing.T) {
+	mockRpc := &MockRPC{}
+	mockRpc.On("Close").Return()
+	backoffClient := client.NewRetryingClient(mockRpc, 0)
+	backoffClient.Close()
+	require.True(t, mockRpc.AssertCalled(t, "Close"))
+}
+
+func TestClient_BackoffClient_CallContext(t *testing.T) {
+	mockRpc := &MockRPC{}
+	mockRpc.ExpectCallContext(nil, nil, "foo", "bar")
+	backoffClient := client.NewRetryingClient(mockRpc, 1)
+	err := backoffClient.CallContext(context.Background(), nil, "foo", "bar")
+	require.NoError(t, err)
+	require.True(t, mockRpc.AssertCalled(t, "CallContext", mock.Anything, nil, "foo", []interface{}{"bar"}))
+}
+
+func TestClient_BackoffClient_CallContext_WithRetries(t *testing.T) {
+	mockRpc := &MockRPC{}
+	mockRpc.ExpectCallContext(errors.New("foo"), nil, "foo", "bar")
+	backoffClient := client.NewRetryingClient(mockRpc, 2)
+	err := backoffClient.CallContext(context.Background(), nil, "foo", "bar")
+	require.Error(t, err)
+	require.True(t, mockRpc.AssertNumberOfCalls(t, "CallContext", 2))
+}
+
+func TestClient_BackoffClient_BatchCallContext(t *testing.T) {
+	mockRpc := &MockRPC{}
+	mockRpc.ExpectBatchCallContext(nil, []rpc.BatchElem(nil))
+	backoffClient := client.NewRetryingClient(mockRpc, 1)
+	err := backoffClient.BatchCallContext(context.Background(), nil)
+	require.NoError(t, err)
+	require.True(t, mockRpc.AssertCalled(t, "BatchCallContext", mock.Anything, []rpc.BatchElem(nil)))
+}
+
+func TestClient_BackoffClient_BatchCallContext_WithRetries(t *testing.T) {
+	mockRpc := &MockRPC{}
+	mockRpc.ExpectBatchCallContext(errors.New("foo"), []rpc.BatchElem(nil))
+	backoffClient := client.NewRetryingClient(mockRpc, 2)
+	err := backoffClient.BatchCallContext(context.Background(), nil)
+	require.Error(t, err)
+	require.True(t, mockRpc.AssertNumberOfCalls(t, "BatchCallContext", 2))
+}
+
+func TestClient_BackoffClient_EthSubscribe(t *testing.T) {
+	mockRpc := &MockRPC{}
+	mockRpc.ExpectEthSubscribe(ethereum.Subscription(nil), nil, nil, "foo", "bar")
+	backoffClient := client.NewRetryingClient(mockRpc, 1)
+	_, err := backoffClient.EthSubscribe(context.Background(), nil, "foo", "bar")
+	require.NoError(t, err)
+	require.True(t, mockRpc.AssertCalled(t, "EthSubscribe", mock.Anything, nil, []interface{}{"foo", "bar"}))
+}
+
+func TestClient_BackoffClient_EthSubscribe_WithRetries(t *testing.T) {
+	mockRpc := &MockRPC{}
+	mockRpc.ExpectEthSubscribe(ethereum.Subscription(nil), errors.New("foo"), nil, "foo", "bar")
+	backoffClient := client.NewRetryingClient(mockRpc, 2)
+	_, err := backoffClient.EthSubscribe(context.Background(), nil, "foo", "bar")
+	require.Error(t, err)
+	require.True(t, mockRpc.AssertNumberOfCalls(t, "EthSubscribe", 2))
+}