txmgr: Add custom publish error message handling (#13856)

* txmgr: Add custom publish error message handling

Fixes #13739

* add prefix TXMGR to env var
Co-authored-by: protolambda <proto@protolambda.com>

---------
Co-authored-by: protolambda <proto@protolambda.com>

op-service/txmgr/cli.go

@@ -27,17 +27,18 @@ const (
 	HDPathFlagName     = "hd-path"
 	PrivateKeyFlagName = "private-key"
 	// TxMgr Flags (new + legacy + some shared flags)
-	NumConfirmationsFlagName          = "num-confirmations"
-	SafeAbortNonceTooLowCountFlagName = "safe-abort-nonce-too-low-count"
-	FeeLimitMultiplierFlagName        = "fee-limit-multiplier"
-	FeeLimitThresholdFlagName         = "txmgr.fee-limit-threshold"
-	MinBaseFeeFlagName                = "txmgr.min-basefee"
-	MinTipCapFlagName                 = "txmgr.min-tip-cap"
-	ResubmissionTimeoutFlagName       = "resubmission-timeout"
-	NetworkTimeoutFlagName            = "network-timeout"
-	TxSendTimeoutFlagName             = "txmgr.send-timeout"
-	TxNotInMempoolTimeoutFlagName     = "txmgr.not-in-mempool-timeout"
-	ReceiptQueryIntervalFlagName      = "txmgr.receipt-query-interval"
+	NumConfirmationsFlagName           = "num-confirmations"
+	SafeAbortNonceTooLowCountFlagName  = "safe-abort-nonce-too-low-count"
+	FeeLimitMultiplierFlagName         = "fee-limit-multiplier"
+	FeeLimitThresholdFlagName          = "txmgr.fee-limit-threshold"
+	MinBaseFeeFlagName                 = "txmgr.min-basefee"
+	MinTipCapFlagName                  = "txmgr.min-tip-cap"
+	ResubmissionTimeoutFlagName        = "resubmission-timeout"
+	NetworkTimeoutFlagName             = "network-timeout"
+	TxSendTimeoutFlagName              = "txmgr.send-timeout"
+	TxNotInMempoolTimeoutFlagName      = "txmgr.not-in-mempool-timeout"
+	ReceiptQueryIntervalFlagName       = "txmgr.receipt-query-interval"
+	AlreadyPublishedCustomErrsFlagName = "txmgr.already-published-custom-errs"
 )
 
 var (
@@ -191,28 +192,34 @@ func CLIFlagsWithDefaults(envPrefix string, defaults DefaultFlagValues) []cli.Fl
 			Value:   defaults.ReceiptQueryInterval,
 			EnvVars: prefixEnvVars("TXMGR_RECEIPT_QUERY_INTERVAL"),
 		},
+		&cli.StringSliceFlag{
+			Name:    AlreadyPublishedCustomErrsFlagName,
+			Usage:   "List of custom RPC error messages that indicate that a transaction has already been published.",
+			EnvVars: prefixEnvVars("TXMGR_ALREADY_PUBLISHED_CUSTOM_ERRS"),
+		},
 	}, opsigner.CLIFlags(envPrefix, "")...)
 }
 
 type CLIConfig struct {
-	L1RPCURL                  string
-	Mnemonic                  string
-	HDPath                    string
-	SequencerHDPath           string
-	L2OutputHDPath            string
-	PrivateKey                string
-	SignerCLIConfig           opsigner.CLIConfig
-	NumConfirmations          uint64
-	SafeAbortNonceTooLowCount uint64
-	FeeLimitMultiplier        uint64
-	FeeLimitThresholdGwei     float64
-	MinBaseFeeGwei            float64
-	MinTipCapGwei             float64
-	ResubmissionTimeout       time.Duration
-	ReceiptQueryInterval      time.Duration
-	NetworkTimeout            time.Duration
-	TxSendTimeout             time.Duration
-	TxNotInMempoolTimeout     time.Duration
+	L1RPCURL                   string
+	Mnemonic                   string
+	HDPath                     string
+	SequencerHDPath            string
+	L2OutputHDPath             string
+	PrivateKey                 string
+	SignerCLIConfig            opsigner.CLIConfig
+	NumConfirmations           uint64
+	SafeAbortNonceTooLowCount  uint64
+	FeeLimitMultiplier         uint64
+	FeeLimitThresholdGwei      float64
+	MinBaseFeeGwei             float64
+	MinTipCapGwei              float64
+	ResubmissionTimeout        time.Duration
+	ReceiptQueryInterval       time.Duration
+	NetworkTimeout             time.Duration
+	TxSendTimeout              time.Duration
+	TxNotInMempoolTimeout      time.Duration
+	AlreadyPublishedCustomErrs []string
 }
 
 func NewCLIConfig(l1RPCURL string, defaults DefaultFlagValues) CLIConfig {
@@ -270,24 +277,25 @@ func (m CLIConfig) Check() error {
 
 func ReadCLIConfig(ctx *cli.Context) CLIConfig {
 	return CLIConfig{
-		L1RPCURL:                  ctx.String(L1RPCFlagName),
-		Mnemonic:                  ctx.String(MnemonicFlagName),
-		HDPath:                    ctx.String(HDPathFlagName),
-		SequencerHDPath:           ctx.String(SequencerHDPathFlag.Name),
-		L2OutputHDPath:            ctx.String(L2OutputHDPathFlag.Name),
-		PrivateKey:                ctx.String(PrivateKeyFlagName),
-		SignerCLIConfig:           opsigner.ReadCLIConfig(ctx),
-		NumConfirmations:          ctx.Uint64(NumConfirmationsFlagName),
-		SafeAbortNonceTooLowCount: ctx.Uint64(SafeAbortNonceTooLowCountFlagName),
-		FeeLimitMultiplier:        ctx.Uint64(FeeLimitMultiplierFlagName),
-		FeeLimitThresholdGwei:     ctx.Float64(FeeLimitThresholdFlagName),
-		MinBaseFeeGwei:            ctx.Float64(MinBaseFeeFlagName),
-		MinTipCapGwei:             ctx.Float64(MinTipCapFlagName),
-		ResubmissionTimeout:       ctx.Duration(ResubmissionTimeoutFlagName),
-		ReceiptQueryInterval:      ctx.Duration(ReceiptQueryIntervalFlagName),
-		NetworkTimeout:            ctx.Duration(NetworkTimeoutFlagName),
-		TxSendTimeout:             ctx.Duration(TxSendTimeoutFlagName),
-		TxNotInMempoolTimeout:     ctx.Duration(TxNotInMempoolTimeoutFlagName),
+		L1RPCURL:                   ctx.String(L1RPCFlagName),
+		Mnemonic:                   ctx.String(MnemonicFlagName),
+		HDPath:                     ctx.String(HDPathFlagName),
+		SequencerHDPath:            ctx.String(SequencerHDPathFlag.Name),
+		L2OutputHDPath:             ctx.String(L2OutputHDPathFlag.Name),
+		PrivateKey:                 ctx.String(PrivateKeyFlagName),
+		SignerCLIConfig:            opsigner.ReadCLIConfig(ctx),
+		NumConfirmations:           ctx.Uint64(NumConfirmationsFlagName),
+		SafeAbortNonceTooLowCount:  ctx.Uint64(SafeAbortNonceTooLowCountFlagName),
+		FeeLimitMultiplier:         ctx.Uint64(FeeLimitMultiplierFlagName),
+		FeeLimitThresholdGwei:      ctx.Float64(FeeLimitThresholdFlagName),
+		MinBaseFeeGwei:             ctx.Float64(MinBaseFeeFlagName),
+		MinTipCapGwei:              ctx.Float64(MinTipCapFlagName),
+		ResubmissionTimeout:        ctx.Duration(ResubmissionTimeoutFlagName),
+		ReceiptQueryInterval:       ctx.Duration(ReceiptQueryIntervalFlagName),
+		NetworkTimeout:             ctx.Duration(NetworkTimeoutFlagName),
+		TxSendTimeout:              ctx.Duration(TxSendTimeoutFlagName),
+		TxNotInMempoolTimeout:      ctx.Duration(TxNotInMempoolTimeoutFlagName),
+		AlreadyPublishedCustomErrs: ctx.StringSlice(AlreadyPublishedCustomErrsFlagName),
 	}
 }
 
@@ -339,16 +347,18 @@ func NewConfig(cfg CLIConfig, l log.Logger) (*Config, error) {
 	}
 
 	res := Config{
-		Backend:                   l1,
-		ChainID:                   chainID,
-		TxSendTimeout:             cfg.TxSendTimeout,
-		TxNotInMempoolTimeout:     cfg.TxNotInMempoolTimeout,
-		NetworkTimeout:            cfg.NetworkTimeout,
-		ReceiptQueryInterval:      cfg.ReceiptQueryInterval,
-		NumConfirmations:          cfg.NumConfirmations,
-		SafeAbortNonceTooLowCount: cfg.SafeAbortNonceTooLowCount,
-		Signer:                    signerFactory(chainID),
-		From:                      from,
+		Backend: l1,
+		ChainID: chainID,
+		Signer:  signerFactory(chainID),
+		From:    from,
+
+		TxSendTimeout:              cfg.TxSendTimeout,
+		TxNotInMempoolTimeout:      cfg.TxNotInMempoolTimeout,
+		NetworkTimeout:             cfg.NetworkTimeout,
+		ReceiptQueryInterval:       cfg.ReceiptQueryInterval,
+		NumConfirmations:           cfg.NumConfirmations,
+		SafeAbortNonceTooLowCount:  cfg.SafeAbortNonceTooLowCount,
+		AlreadyPublishedCustomErrs: cfg.AlreadyPublishedCustomErrs,
 	}
 
 	res.ResubmissionTimeout.Store(int64(cfg.ResubmissionTimeout))
@@ -422,6 +432,10 @@ type Config struct {
 	// GasPriceEstimatorFn is used to estimate the gas price for a transaction.
 	// If nil, DefaultGasPriceEstimatorFn is used.
 	GasPriceEstimatorFn GasPriceEstimatorFn
+
+	// List of custom RPC error messages that indicate that a transaction has
+	// already been published.
+	AlreadyPublishedCustomErrs []string
 }
 
 func (m *Config) Check() error {

op-service/txmgr/txmgr.go

@@ -638,9 +638,14 @@ func (m *SimpleTxManager) publishTx(ctx context.Context, tx *types.Transaction,
 		cancel()
 		sendState.ProcessSendError(err)
 
-		if err == nil {
+		if err == nil || errStringContainsAny(err, m.cfg.AlreadyPublishedCustomErrs) {
+			// only empty error strings are recorded as successful publishes
 			m.metr.TxPublished("")
-			l.Info("Transaction successfully published", "tx", tx.Hash())
+			if err == nil {
+				l.Info("Transaction successfully published", "tx", tx.Hash())
+			} else {
+				l.Info("Transaction successfully published (custom RPC error)", "tx", tx.Hash(), "err", err)
+			}
 			// Tx made it into the mempool, so we'll need a fee bump if we end up trying to replace
 			// it with another publish attempt.
 			sendState.bumpFees = true
@@ -1053,6 +1058,19 @@ func errStringMatch(err, target error) bool {
 	return strings.Contains(err.Error(), target.Error())
 }
 
+func errStringContainsAny(err error, targets []string) bool {
+	if err == nil || len(targets) == 0 {
+		return false
+	}
+
+	for _, target := range targets {
+		if strings.Contains(err.Error(), target) {
+			return true
+		}
+	}
+	return false
+}
+
 // finishBlobTx finishes creating a blob tx message by safely converting bigints to uint256
 func finishBlobTx(message *types.BlobTx, chainID, tip, fee, blobFee, value *big.Int) error {
 	var o bool

op-service/txmgr/txmgr_test.go

@@ -1581,6 +1581,29 @@ func TestCloseWaitingForConfirmation(t *testing.T) {
 	})
 }
 
+func TestTxMgrCustomPublishError(t *testing.T) {
+	customErr := errors.New("custom test error")
+
+	testSendVariants(t, func(t *testing.T, send testSendVariantsFn) {
+		cfg := configWithNumConfs(1)
+		cfg.AlreadyPublishedCustomErrs = []string{customErr.Error()}
+		h := newTestHarnessWithConfig(t, cfg)
+
+		sendTx := func(ctx context.Context, tx *types.Transaction) error {
+			txHash := tx.Hash()
+			h.backend.mine(&txHash, tx.GasFeeCap(), nil)
+			return customErr
+		}
+		h.backend.setTxSender(sendTx)
+
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		receipt, err := send(ctx, h, h.createTxCandidate())
+		require.Nil(t, err)
+		require.NotNil(t, receipt)
+	})
+}
+
 func TestMakeSidecar(t *testing.T) {
 	var blob eth.Blob
 	_, err := rand.Read(blob[:])