update

server/workerManager.go

@@ -280,8 +280,12 @@ func (wm *WorkerManager) manageWorker(worker *Worker) error {
 			gb.GoodbyeMessage = &omanager.GoodbyeMessage{}
 			msg.Message = gb
 
-		case <-worker.quit:
+		case reason, ok := <-worker.quit:
+			if ok {
+				log.WithField("reason", reason).WithField("worker-uuid", worker.uuid).Error("worker quit")
+			}
 			return nil
+
 		case <-workerCheckTicker.C:
 			if worker.info.nodeInfo != nil {
 				nodeinfoTicker.Reset(time.Hour * 24)
@@ -295,12 +299,6 @@ func (wm *WorkerManager) manageWorker(worker *Worker) error {
 				deviceUsageTicker.Reset(time.Second * time.Duration(tickerConf.DeviceUsageTicker))
 			}
 
-			if time.Now().Unix()-wm.GetHeartBeat(worker.uuid) > int64(workerCheckDuration.Seconds()) {
-				log.WithField("worker-uuid", worker.uuid).Error("worker heartbeat expired")
-				close(worker.quit)
-				return ErrHeartBeatExpired
-			}
-
 			if worker.registed && worker.addFirstSucceed == false && len(worker.deviceInfoHash) > 0 {
 				wm.AddWorkerToQueue(worker)
 			}
@@ -422,16 +420,29 @@ func (wm *WorkerManager) handleWorkerMsg(worker *Worker) {
 	l := log.WithField("worker-uuid", worker.uuid)
 	l.WithField("worker-addr", worker.workerAddr).Info("start handle worker message")
 	defer l.WithField("worker-addr", worker.workerAddr).Info("exit handle worker message")
+	defer close(worker.quit)
+
+	checkDuration := config.GetConfig().Tickers.HeartBeat * 3
+
+	workerCheckTicker := time.NewTicker(time.Second * time.Duration(checkDuration))
+	defer workerCheckTicker.Stop()
+	defer close(worker.quit)
+
 	for {
 		select {
 		case <-wm.quit:
 			return
-		case <-worker.quit:
-			return
+		case <-workerCheckTicker.C:
+			if time.Now().Unix()-wm.GetHeartBeat(worker.uuid) > int64(checkDuration) {
+				log.WithField("worker-uuid", worker.uuid).Error("worker heartbeat expired")
+				worker.quit <- ErrHeartBeatExpired
+				return
+			}
 		default:
 			wmsg, err := worker.stream.Recv()
 			if err != nil {
 				l.WithError(err).WithField("worker-addr", worker.workerAddr).Error("recv msg failed")
+				worker.quit <- "recv msg failed"
 				return
 			}
 			worker.online = true