update mutex

swarm/swarm.go

@@ -15,8 +15,10 @@ type ClientWithRateLimiter struct {
 }
 
 type Swarm struct {
-	clients map[string]*ClientWithRateLimiter
-	mu      sync.Mutex
+	clients    map[string]*ClientWithRateLimiter
+	mu         sync.Mutex
+	countmu    sync.Mutex
+	followermu sync.Mutex
 }
 
 var (
@@ -37,9 +39,20 @@ func InitSwarm(initialBees []string) (*Swarm, error) {
 	for _, bee := range initialBees {
 		s.AddClient(bee)
 	}
+	gSwarm = s
 	return s, nil
 }
 
+func (s *Swarm) copyedClients() map[string]*ClientWithRateLimiter {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	clients := make(map[string]*ClientWithRateLimiter)
+	for k, v := range s.clients {
+		clients[k] = v
+	}
+	return clients
+}
+
 func (s *Swarm) AddClient(url string) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -57,26 +70,32 @@ func (s *Swarm) RemoveClient(url string) {
 }
 
 func (s *Swarm) GetFollowerCount(userID string) (int, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
+	clients := s.copyedClients()
+	s.countmu.Lock()
+	defer s.countmu.Unlock()
 
-	for _, cli := range s.clients {
+	for _, cli := range clients {
 		res, err := cli.GetFollowerCount(userID)
 		if err == nil {
+			fmt.Printf("get %v follower count %d\n", userID, res.Count)
 			return res.Count, nil
+		} else {
+			fmt.Println("get follower count failed with err", err.Error())
 		}
 	}
 	return 0, fmt.Errorf("can not get the %v follower count", userID)
 }
 
 func (s *Swarm) GetFollowerList(user string, cursor string) ([]*twitter.UserObj, string, *twitter.RateLimit, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
+	clients := s.copyedClients()
 
-	for _, cli := range s.clients {
-		if cli.RateLimit.Allow() == false {
-			continue
-		}
+	s.followermu.Lock()
+	defer s.followermu.Unlock()
+
+	for _, cli := range clients {
+		//if cli.RateLimit.Allow() == false {
+		//	continue
+		//}
 		res, err := cli.GetFollowerList(user, cursor)
 		if err == nil {
 			list := make([]*twitter.UserObj, 0, len(res.List))
@@ -89,7 +108,7 @@ func (s *Swarm) GetFollowerList(user string, cursor string) ([]*twitter.UserObj,
 			}
 			return list, res.Next, nil, nil
 		} else {
-
+			fmt.Println("get follower list failed with err", err.Error())
 		}
 	}
 	return nil, "", nil, fmt.Errorf("can not get the %v follower list", user)