op-conductor: Increase error backoff time to 0-2000 ms, set to 0 in tests (#10710)

* op-conductor: Increase error backoff time to 0-2000 ms, set to 0 in tests

* Improve test perf

* Un-change max test time

op-conductor/conductor/service.go

@@ -72,6 +72,7 @@ func NewOpConductor(
 		ctrl:         ctrl,
 		cons:         cons,
 		hmon:         hmon,
+		retryBackoff: func() time.Duration { return time.Duration(rand.Intn(2000)) * time.Millisecond },
 	}
 	oc.loopActionFn = oc.loopAction
 
@@ -280,6 +281,8 @@ type OpConductor struct {
 
 	rpcServer     *oprpc.Server
 	metricsServer *httputil.HTTPServer
+
+	retryBackoff func() time.Duration
 }
 
 type state struct {
@@ -642,8 +645,7 @@ func (oc *OpConductor) action() {
 	oc.log.Debug("exiting action with status and error", "status", status, "err", err)
 	if err != nil {
 		oc.log.Error("failed to execute step, queueing another one to retry", "err", err, "status", status)
-		// randomly sleep for 0-200ms to avoid excessive retry
-		time.Sleep(time.Duration(rand.Intn(200)) * time.Millisecond)
+		time.Sleep(oc.retryBackoff())
 		oc.queueAction()
 		return
 	}

op-conductor/conductor/service_test.go

@@ -118,6 +118,7 @@ func (s *OpConductorTestSuite) SetupTest() {
 
 	conductor, err := NewOpConductor(s.ctx, &s.cfg, s.log, s.metrics, s.version, s.ctrl, s.cons, s.hmon)
 	s.NoError(err)
+	conductor.retryBackoff = func() time.Duration { return 0 } // disable retry backoff for tests
 	s.conductor = conductor
 
 	s.healthUpdateCh = make(chan error, 1)
@@ -751,7 +752,7 @@ func (s *OpConductorTestSuite) TestFailureAndRetry3() {
 			s.executeAction()
 		}
 		return res
-	}, 2*time.Second, 100*time.Millisecond)
+	}, 2*time.Second, time.Millisecond)
 }
 
 func (s *OpConductorTestSuite) TestHandleInitError() {