Skip to content

N
nebula
Commit a3ac6a90 authored by Adrian Sutton's avatar Adrian Sutton Committed by GitHub
Browse files
Options

op-program: Update verification tests to not depend on L2OO (#9821) 

* op-program: Update fpp-verify to not depend on L2OO

Add verify test for the first sepolia testnet block

* op-program: Capture first mainnet block

* op-program: Capture ecotone section of sepolia

* op-program: Remove logging of l1 head again to make testing easier
parent 53a235c9
Hide whitespace changes
Inline Side-by-side
Showing with 214 additions and 84 deletions
op-program/Makefile
View file @ a3ac6a90
......@@ -38,24 +38,25 @@ clean:
test:
go test -v ./...
verify-goerli: op-program-host op-program-client
env GO111MODULE=on go run ./verify/goerli/cmd/goerli.go --l1 $$L1URL --l2 $$L2URL
verify-sepolia: op-program-host op-program-client
env GO111MODULE=on go run ./verify/sepolia/cmd/sepolia.go --l1 $$SEPOLIA_L1URL --l1.beacon $$SEPOLIA_BEACON_URL --l2 $$SEPOLIA_L2URL --datadir /tmp/test-sepolia
run-goerli-verify: op-program-host op-program-client
mkdir -p "$(COMPAT_DIR)"
curl -L -o "$(COMPAT_DIR)/goerli.tar.bz" https://github.com/ethereum-optimism/chain-test-data/releases/download/2023-10-11/goerli.tar.bz
tar jxf "$(COMPAT_DIR)/goerli.tar.bz" -C "$(COMPAT_DIR)"
./bin/op-program `cat "$(COMPAT_DIR)/goerli/args.txt"`
capture-mainnet-genesis: op-program-host op-program-client
rm -rf "$(COMPAT_DIR)/mainnet-genesis" "$(COMPAT_DIR)/mainnet-genesis.tar.bz"
env GO111MODULE=on go run ./verify/mainnet/cmd/mainnet.go --l1 $$MAINNET_L1URL --l1.beacon $$MAINNET_BEACON_URL --l2 $$MAINNET_L2URL --datadir "$(COMPAT_DIR)/mainnet-genesis" --l1.head "0x4903424f6cc2cfba7c2bf8c8f48ca46721c963fa64b411cfee3697b781e3e5f1" --l2.start "105235063" --l2.end "105235064"
tar jcf "$(COMPAT_DIR)/mainnet-genesis.tar.bz" -C "$(COMPAT_DIR)" mainnet-genesis
verify-sepolia: op-program-host op-program-client
env GO111MODULE=on go run ./verify/sepolia/cmd/sepolia.go --l1 $$SEPOLIA_L1URL --l1.beacon $$SEPOLIA_BEACON_URL --l2 $$SEPOLIA_L2URL
capture-sepolia-delta: op-program-host op-program-client
rm -rf "$(COMPAT_DIR)/sepolia-delta" "$(COMPAT_DIR)/sepolia-delta.tar.bz"
env GO111MODULE=on go run ./verify/sepolia/cmd/sepolia.go --l1 $$SEPOLIA_L1URL --l1.beacon $$SEPOLIA_BEACON_URL --l2 $$SEPOLIA_L2URL --datadir "$(COMPAT_DIR)/sepolia-delta" --l1.head "0x935428728bcfcfeb2e5ba9175fd2890e52831dae221aa4d5dcffed8320edc001" --l2.start "8728200" --l2.end "8728320"
tar jcf "$(COMPAT_DIR)/sepolia-delta.tar.bz" -C "$(COMPAT_DIR)" sepolia-delta
capture-sepolia-verify: op-program-host op-program-client
rm -rf "$(COMPAT_DIR)/sepolia" "$(COMPAT_DIR)/sepolia.tar.bz"
env GO111MODULE=on go run ./verify/sepolia/cmd/sepolia.go --l1 $$SEPOLIA_L1URL --l1.beacon $$SEPOLIA_BEACON_URL --l2 $$SEPOLIA_L2URL --datadir "$(COMPAT_DIR)/sepolia"
tar jcf "$(COMPAT_DIR)/sepolia.tar.bz" -C "$(COMPAT_DIR)" sepolia
capture-sepolia-ecotone: op-program-host op-program-client
rm -rf "$(COMPAT_DIR)/sepolia-ecotone" "$(COMPAT_DIR)/sepolia-ecotone.tar.bz"
env GO111MODULE=on go run ./verify/sepolia/cmd/sepolia.go --l1 $$SEPOLIA_L1URL --l1.beacon $$SEPOLIA_BEACON_URL --l2 $$SEPOLIA_L2URL --datadir "$(COMPAT_DIR)/sepolia-ecotone" --l1.head "0x5d491a8c1e728a4e70720c09fefdaa083681a9421cd365af85220cf8bd4448a3" --l2.start "9205715" --l2.end "9205815"
tar jcf "$(COMPAT_DIR)/sepolia-ecotone.tar.bz" -C "$(COMPAT_DIR)" sepolia-ecotone
capture-chain-test-data: capture-sepolia-verify
capture-chain-test-data: capture-mainnet-genesis capture-sepolia-delta
run-sepolia-verify: op-program-host op-program-client
mkdir -p "$(COMPAT_DIR)"
......@@ -70,10 +71,11 @@ run-sepolia-verify: op-program-host op-program-client
op-program-client-mips \
clean \
test \
verify-goerli \
capture-goerli-verify \
verify-sepolia \
capture-sepolia-verify \
capture-mainnet-genesis \
capture-sepolia-delta \
capture-sepolia-ecotone \
capture-chain-test-data \
run-goerli-verify \
run-sepolia-verify
op-program/verify/goerli/cmd/goerli.go → op-program/verify/mainnet/cmd/mainnet.go
View file @ a3ac6a90
package main
import (
"context"
"flag"
"fmt"
"os"
......@@ -16,12 +17,18 @@ func main() {
var l1BeaconUrl string
var l2RpcUrl string
var dataDir string
var l1HashStr string
var l2Start uint64
var l2End uint64
flag.StringVar(&l1RpcUrl, "l1", "", "L1 RPC URL to use")
flag.StringVar(&l1BeaconUrl, "l1.beacon", "", "L1 Beacon URL to use")
flag.StringVar(&l1RpcKind, "l1-rpckind", "", "L1 RPC kind")
flag.StringVar(&l2RpcUrl, "l2", "", "L2 RPC URL to use")
flag.StringVar(&dataDir, "datadir", "",
"Directory to use for storing pre-images. If not set a temporary directory will be used.")
flag.StringVar(&l1HashStr, "l1.head", "", "Hash of L1 block to use")
flag.Uint64Var(&l2Start, "l2.start", 0, "Block number of agreed L2 block")
flag.Uint64Var(&l2End, "l2.end", 0, "Block number of claimed L2 block")
flag.Parse()
if l1RpcUrl == "" {
......@@ -37,8 +44,22 @@ func main() {
os.Exit(2)
}
goerliOutputAddress := common.HexToAddress("0xE6Dfba0953616Bacab0c9A8ecb3a9BBa77FC15c0")
err := verify.Run(l1RpcUrl, l1RpcKind, l1BeaconUrl, l2RpcUrl, goerliOutputAddress, dataDir, "goerli", chainconfig.OPGoerliChainConfig)
runner, err := verify.NewRunner(l1RpcUrl, l1RpcKind, l1BeaconUrl, l2RpcUrl, dataDir, "op-mainnet", chainconfig.OPMainnetChainConfig)
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "Failed to create runner: %v\n", err.Error())
os.Exit(1)
}
if l1HashStr == "" && l2Start == 0 && l2End == 0 {
err = runner.RunToFinalized(context.Background())
} else {
l1Hash := common.HexToHash(l1HashStr)
if l1Hash == (common.Hash{}) {
_, _ = fmt.Fprintf(os.Stderr, "Invalid --l1.head: %v\n", l1HashStr)
os.Exit(2)
}
err = runner.RunBetweenBlocks(context.Background(), l1Hash, l2Start, l2End)
}
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "Failed: %v\n", err.Error())
os.Exit(1)
......
op-program/verify/sepolia/cmd/sepolia.go
View file @ a3ac6a90
package main
import (
"context"
"flag"
"fmt"
"os"
......@@ -16,12 +17,18 @@ func main() {
var l1BeaconUrl string
var l2RpcUrl string
var dataDir string
var l1HashStr string
var l2Start uint64
var l2End uint64
flag.StringVar(&l1RpcUrl, "l1", "", "L1 RPC URL to use")
flag.StringVar(&l1BeaconUrl, "l1.beacon", "", "L1 Beacon URL to use")
flag.StringVar(&l1RpcKind, "l1-rpckind", "", "L1 RPC kind")
flag.StringVar(&l2RpcUrl, "l2", "", "L2 RPC URL to use")
flag.StringVar(&dataDir, "datadir", "",
"Directory to use for storing pre-images. If not set a temporary directory will be used.")
flag.StringVar(&l1HashStr, "l1.head", "", "Hash of L1 block to use")
flag.Uint64Var(&l2Start, "l2.start", 0, "Block number of agreed L2 block")
flag.Uint64Var(&l2End, "l2.end", 0, "Block number of claimed L2 block")
flag.Parse()
if l1RpcUrl == "" {
......@@ -37,8 +44,22 @@ func main() {
os.Exit(2)
}
sepoliaOutputAddress := common.HexToAddress("0x90E9c4f8a994a250F6aEfd61CAFb4F2e895D458F")
err := verify.Run(l1RpcUrl, l1RpcKind, l1BeaconUrl, l2RpcUrl, sepoliaOutputAddress, dataDir, "sepolia", chainconfig.OPSepoliaChainConfig)
runner, err := verify.NewRunner(l1RpcUrl, l1RpcKind, l1BeaconUrl, l2RpcUrl, dataDir, "sepolia", chainconfig.OPSepoliaChainConfig)
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "Failed to create runner: %v\n", err.Error())
os.Exit(1)
}
if l1HashStr == "" && l2Start == 0 && l2End == 0 {
err = runner.RunToFinalized(context.Background())
} else {
l1Hash := common.HexToHash(l1HashStr)
if l1Hash == (common.Hash{}) {
_, _ = fmt.Fprintf(os.Stderr, "Invalid --l1.head: %v\n", l1HashStr)
os.Exit(2)
}
err = runner.RunBetweenBlocks(context.Background(), l1Hash, l2Start, l2End)
}
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "Failed: %v\n", err.Error())
os.Exit(1)
......
op-program/verify/verify.go
View file @ a3ac6a90
......@@ -6,18 +6,19 @@ import (
"math/big"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/ethereum-optimism/optimism/op-bindings/bindings"
"github.com/ethereum-optimism/optimism/op-node/rollup"
"github.com/ethereum-optimism/optimism/op-program/host"
"github.com/ethereum-optimism/optimism/op-program/host/config"
"github.com/ethereum-optimism/optimism/op-service/client"
"github.com/ethereum-optimism/optimism/op-service/dial"
"github.com/ethereum-optimism/optimism/op-service/eth"
oplog "github.com/ethereum-optimism/optimism/op-service/log"
"github.com/ethereum-optimism/optimism/op-service/retry"
"github.com/ethereum-optimism/optimism/op-service/sources"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
......@@ -25,134 +26,219 @@ import (
"github.com/ethereum/go-ethereum/rpc"
)
func Run(l1RpcUrl string, l1RpcKind string, l1BeaconUrl string, l2RpcUrl string, l2OracleAddr common.Address, dataDir string, network string, chainCfg *params.ChainConfig) error {
type Runner struct {
l1RpcUrl string
l1RpcKind string
l1BeaconUrl string
l2RpcUrl string
dataDir string
network string
chainCfg *params.ChainConfig
l2Client *sources.L2Client
logCfg oplog.CLIConfig
setupLog log.Logger
rollupCfg *rollup.Config
}
func NewRunner(l1RpcUrl string, l1RpcKind string, l1BeaconUrl string, l2RpcUrl string, dataDir string, network string, chainCfg *params.ChainConfig) (*Runner, error) {
ctx := context.Background()
logger := oplog.DefaultCLIConfig()
logger.Level = log.LevelDebug
logCfg := oplog.DefaultCLIConfig()
logCfg.Level = log.LevelDebug
setupLog := oplog.NewLogger(os.Stderr, logCfg)
setupLog := oplog.NewLogger(os.Stderr, logger)
l1Client, err := dial.DialEthClientWithTimeout(ctx, dial.DefaultDialTimeout, setupLog, l1RpcUrl)
l2RawRpc, err := dial.DialRPCClientWithTimeout(ctx, dial.DefaultDialTimeout, setupLog, l2RpcUrl)
if err != nil {
return fmt.Errorf("dial L1 client: %w", err)
return nil, fmt.Errorf("dial L2 client: %w", err)
}
l2Client, err := dial.DialEthClientWithTimeout(ctx, dial.DefaultDialTimeout, setupLog, l2RpcUrl)
rollupCfg, err := rollup.LoadOPStackRollupConfig(chainCfg.ChainID.Uint64())
if err != nil {
return fmt.Errorf("dial L2 client: %w", err)
return nil, fmt.Errorf("failed to load rollup config: %w", err)
}
outputOracle, err := bindings.NewL2OutputOracle(l2OracleAddr, l1Client)
l2ClientCfg := sources.L2ClientDefaultConfig(rollupCfg, false)
l2RPC := client.NewBaseRPCClient(l2RawRpc)
l2Client, err := sources.NewL2Client(l2RPC, setupLog, nil, l2ClientCfg)
if err != nil {
return fmt.Errorf("create output oracle bindings: %w", err)
return nil, fmt.Errorf("failed to create L2 client: %w", err)
}
// Find L1 finalized block. Can't be re-orged.
l1BlockNum := big.NewInt(int64(rpc.FinalizedBlockNumber))
l1HeadBlock, err := retryOp(ctx, func() (*types.Block, error) {
return l1Client.BlockByNumber(ctx, l1BlockNum)
})
return &Runner{
l1RpcUrl: l1RpcUrl,
l1RpcKind: l1RpcKind,
l1BeaconUrl: l1BeaconUrl,
l2RpcUrl: l2RpcUrl,
dataDir: dataDir,
network: network,
chainCfg: chainCfg,
logCfg: logCfg,
setupLog: setupLog,
l2Client: l2Client,
rollupCfg: rollupCfg,
}, nil
}
func (r *Runner) RunBetweenBlocks(ctx context.Context, l1Head common.Hash, startBlockNum uint64, endBlockNumber uint64) error {
if startBlockNum >= endBlockNumber {
return fmt.Errorf("start block number %v must be less than end block number %v", startBlockNum, endBlockNumber)
}
l2Client, err := r.createL2Client(ctx)
if err != nil {
return fmt.Errorf("find L1 head: %w", err)
return err
}
fmt.Printf("Found l1 head block number: %v hash: %v\n", l1HeadBlock.NumberU64(), l1HeadBlock.Hash())
defer l2Client.Close()
l1CallOpts := &bind.CallOpts{Context: ctx, BlockNumber: l1BlockNum}
agreedBlockInfo, agreedOutputRoot, err := outputAtBlockNum(ctx, l2Client, startBlockNum)
if err != nil {
return fmt.Errorf("failed to find starting block info: %w", err)
}
claimedBlockInfo, claimedOutputRoot, err := outputAtBlockNum(ctx, l2Client, endBlockNumber)
if err != nil {
return fmt.Errorf("failed to find ending block info: %w", err)
}
// Find the latest output root published in this finalized block
latestOutputIndex, err := retryOp(ctx, func() (*big.Int, error) {
return outputOracle.LatestOutputIndex(l1CallOpts)
})
return r.run(l1Head, agreedBlockInfo, agreedOutputRoot, claimedOutputRoot, claimedBlockInfo)
}
func (r *Runner) createL2Client(ctx context.Context) (*sources.L2Client, error) {
l2RawRpc, err := dial.DialRPCClientWithTimeout(ctx, dial.DefaultDialTimeout, r.setupLog, r.l2RpcUrl)
if err != nil {
return nil, fmt.Errorf("dial L2 client: %w", err)
}
l2ClientCfg := sources.L2ClientDefaultConfig(r.rollupCfg, false)
l2RPC := client.NewBaseRPCClient(l2RawRpc)
l2Client, err := sources.NewL2Client(l2RPC, r.setupLog, nil, l2ClientCfg)
if err != nil {
return nil, fmt.Errorf("failed to create L2 client: %w", err)
}
return l2Client, nil
}
func (r *Runner) RunToFinalized(ctx context.Context) error {
l1Client, err := dial.DialEthClientWithTimeout(ctx, dial.DefaultDialTimeout, r.setupLog, r.l1RpcUrl)
if err != nil {
return fmt.Errorf("failed to dial L1 client: %w", err)
}
l2Client, err := r.createL2Client(ctx)
if err != nil {
return fmt.Errorf("fetch latest output index: %w", err)
return err
}
output, err := retryOp(ctx, func() (bindings.TypesOutputProposal, error) {
return outputOracle.GetL2Output(l1CallOpts, latestOutputIndex)
defer l2Client.Close()
l2Finalized, err := retryOp(ctx, func() (eth.BlockInfo, error) {
return l2Client.InfoByLabel(ctx, eth.Finalized)
})
if err != nil {
return fmt.Errorf("fetch l2 output %v: %w", latestOutputIndex, err)
return fmt.Errorf("failed to retrieve finalized L2 block: %w", err)
}
// Use the previous output as the agreed starting point
agreedOutput, err := retryOp(ctx, func() (bindings.TypesOutputProposal, error) {
return outputOracle.GetL2Output(l1CallOpts, new(big.Int).Sub(latestOutputIndex, common.Big1))
// Retrieve finalized L1 block after finalized L2 block to ensure it is
l1Head, err := retryOp(ctx, func() (*types.Header, error) {
return l1Client.HeaderByNumber(ctx, big.NewInt(rpc.FinalizedBlockNumber.Int64()))
})
if err != nil {
return fmt.Errorf("fetch l2 output before %v: %w", latestOutputIndex, err)
return fmt.Errorf("failed to retrieve finalized L1 block: %w", err)
}
l2BlockAtOutput, err := retryOp(ctx, func() (*types.Block, error) { return l2Client.BlockByNumber(ctx, agreedOutput.L2BlockNumber) })
// Process the 100 blocks leading up to finalized
startBlockNum := uint64(0)
if l2Finalized.NumberU64() > 100 {
startBlockNum = l2Finalized.NumberU64() - 100
}
agreedBlockInfo, agreedOutputRoot, err := outputAtBlockNum(ctx, l2Client, startBlockNum)
if err != nil {
return fmt.Errorf("retrieve agreed block: %w", err)
return fmt.Errorf("failed to find starting block info: %w", err)
}
claimedBlockInfo, claimedOutputRoot, err := outputAtBlockNum(ctx, l2Client, l2Finalized.NumberU64())
if err != nil {
return fmt.Errorf("failed to find ending block info: %w", err)
}
l2Head := l2BlockAtOutput.Hash()
l2BlockNumber := output.L2BlockNumber
l2Claim := common.Hash(output.OutputRoot)
l1Head := l1HeadBlock.Hash()
return r.run(l1Head.Hash(), agreedBlockInfo, agreedOutputRoot, claimedOutputRoot, claimedBlockInfo)
}
if dataDir == "" {
dataDir, err = os.MkdirTemp("", "oracledata")
func (r *Runner) run(l1Head common.Hash, agreedBlockInfo eth.BlockInfo, agreedOutputRoot common.Hash, claimedOutputRoot common.Hash, claimedBlockInfo eth.BlockInfo) error {
var err error
if r.dataDir == "" {
r.dataDir, err = os.MkdirTemp("", "oracledata")
if err != nil {
return fmt.Errorf("create temp dir: %w", err)
}
defer func() {
err := os.RemoveAll(dataDir)
err := os.RemoveAll(r.dataDir)
if err != nil {
fmt.Println("Failed to remove temp dir:" + err.Error())
}
}()
} else {
if err := os.MkdirAll(dataDir, 0755); err != nil {
fmt.Printf("Could not create data directory %v: %v", dataDir, err)
if err := os.MkdirAll(r.dataDir, 0755); err != nil {
fmt.Printf("Could not create data directory %v: %v", r.dataDir, err)
os.Exit(1)
}
}
fmt.Printf("Using dir: %s\n", dataDir)
fmt.Printf("Using dir: %s\n", r.dataDir)
args := []string{
"--log.level", "DEBUG",
"--network", network,
"--network", r.network,
"--exec", "./bin/op-program-client",
"--datadir", dataDir,
"--datadir", r.dataDir,
"--l1.head", l1Head.Hex(),
"--l2.head", l2Head.Hex(),
"--l2.outputroot", common.Bytes2Hex(agreedOutput.OutputRoot[:]),
"--l2.claim", l2Claim.Hex(),
"--l2.blocknumber", l2BlockNumber.String(),
"--l2.head", agreedBlockInfo.Hash().Hex(),
"--l2.outputroot", agreedOutputRoot.Hex(),
"--l2.claim", claimedOutputRoot.Hex(),
"--l2.blocknumber", strconv.FormatUint(claimedBlockInfo.NumberU64(), 10),
}
argsStr := strings.Join(args, " ")
// args.txt is used by the verify job for offline verification in CI
if err := os.WriteFile(filepath.Join(dataDir, "args.txt"), []byte(argsStr), 0644); err != nil {
if err := os.WriteFile(filepath.Join(r.dataDir, "args.txt"), []byte(argsStr), 0644); err != nil {
fmt.Printf("Could not write args: %v", err)
os.Exit(1)
}
fmt.Printf("Configuration: %s\n", argsStr)
rollupCfg, err := rollup.LoadOPStackRollupConfig(chainCfg.ChainID.Uint64())
if err != nil {
return fmt.Errorf("failed to load rollup config: %w", err)
}
offlineCfg := config.NewConfig(
rollupCfg, chainCfg, l1Head, l2Head, agreedOutput.OutputRoot, l2Claim, l2BlockNumber.Uint64())
offlineCfg.DataDir = dataDir
r.rollupCfg, r.chainCfg, l1Head, agreedBlockInfo.Hash(), agreedOutputRoot, claimedOutputRoot, claimedBlockInfo.NumberU64())
offlineCfg.DataDir = r.dataDir
onlineCfg := *offlineCfg
onlineCfg.L1URL = l1RpcUrl
onlineCfg.L1BeaconURL = l1BeaconUrl
onlineCfg.L2URL = l2RpcUrl
if l1RpcKind != "" {
onlineCfg.L1RPCKind = sources.RPCProviderKind(l1RpcKind)
onlineCfg.L1URL = r.l1RpcUrl
onlineCfg.L1BeaconURL = r.l1BeaconUrl
onlineCfg.L2URL = r.l2RpcUrl
if r.l1RpcKind != "" {
onlineCfg.L1RPCKind = sources.RPCProviderKind(r.l1RpcKind)
}
fmt.Println("Running in online mode")
err = host.Main(oplog.NewLogger(os.Stderr, logger), &onlineCfg)
err = host.Main(oplog.NewLogger(os.Stderr, r.logCfg), &onlineCfg)
if err != nil {
return fmt.Errorf("online mode failed: %w", err)
}
fmt.Println("Running in offline mode")
err = host.Main(oplog.NewLogger(os.Stderr, logger), offlineCfg)
err = host.Main(oplog.NewLogger(os.Stderr, r.logCfg), offlineCfg)
if err != nil {
return fmt.Errorf("offline mode failed: %w", err)
}
return nil
}
func outputAtBlockNum(ctx context.Context, l2Client *sources.L2Client, blockNum uint64) (eth.BlockInfo, common.Hash, error) {
startBlockInfo, err := l2Client.InfoByNumber(ctx, blockNum)
if err != nil {
return nil, common.Hash{}, fmt.Errorf("failed to retrieve info for block %v: %w", startBlockInfo, err)
}
output, err := retryOp(ctx, func() (*eth.OutputV0, error) {
return l2Client.OutputV0AtBlock(ctx, startBlockInfo.Hash())
})
if err != nil {
return nil, common.Hash{}, fmt.Errorf("failed to retrieve agreed output root for block %v: %w", startBlockInfo.Hash(), err)
}
return startBlockInfo, common.Hash(eth.OutputRoot(output)), nil
}
func retryOp[T any](ctx context.Context, op func() (T, error)) (T, error) {
return retry.Do(ctx, 10, retry.Fixed(time.Second*2), op)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment