profile get followers

accounts.go

@@ -18,7 +18,7 @@ func generateTOTP(secret string) (string, error) {
 	return totp.GenerateCode(secret, time.Now())
 }
 
-func GetLoginAccount() ([]*twitterscraper.Scraper, error) {
+func GetLoginAccount() ([]ScraperWithTimer, error) {
 
 	accounts, err := GetAvailableAccounts()
 
@@ -28,7 +28,7 @@ func GetLoginAccount() ([]*twitterscraper.Scraper, error) {
 
 	fmt.Println("len(accounts)", len(accounts))
 
-	res := make([]*twitterscraper.Scraper, 0, len(accounts))
+	res := make([]ScraperWithTimer, 0, len(accounts))
 
 	for _, v := range accounts {
 
@@ -49,7 +49,10 @@ func GetLoginAccount() ([]*twitterscraper.Scraper, error) {
 			if !scraper.IsLoggedIn() {
 				needLogin = true
 			} else {
-				res = append(res, scraper)
+				res = append(res, ScraperWithTimer{
+					Scraper:     scraper,
+					AccountInfo: v,
+				})
 				continue
 			}
 		} else {
@@ -80,7 +83,10 @@ func GetLoginAccount() ([]*twitterscraper.Scraper, error) {
 				continue
 			}
 
-			res = append(res, scraper)
+			res = append(res, ScraperWithTimer{
+				Scraper:     scraper,
+				AccountInfo: v,
+			})
 		}
 	}
 
@@ -89,9 +95,8 @@ func GetLoginAccount() ([]*twitterscraper.Scraper, error) {
 
 type ScraperWithTimer struct {
 	*twitterscraper.Scraper
+	AccountInfo Account
 	Timer       *time.Timer
-	Email    string
-	UserName string
 }
 
 var accChan chan ScraperWithTimer = make(chan ScraperWithTimer, 20)
@@ -166,15 +171,33 @@ func (f *FollowerRateLimit) Follower(userName string, cursor string) ([]*twitter
 	if err := f.RateLimit.Wait(ctx); err != nil { // This is a blocking call.
 		return nil, "", nil, err
 	}
-
+	var (
+		history = make(map[string]bool)
+		users   []*twitterscraper.Profile
+		res     []*twitter.UserObj
+		next    string
+		err     error
+		success bool = false
+		try          = 0
+	)
+	for !success && try < 10 {
 		select {
 		case account := <-accChan:
-		users, next, err := account.FetchFollowers(userName, 1000, cursor)
-		if err != nil {
-			slog.Error(err.Error())
+			accChan <- account
+			history[account.AccountInfo.Username] = true
+
+			if _, exist := history[account.AccountInfo.Username]; exist {
+				// loop all account, exit.
+				try = 100
+				break
+			}
+			if users, next, err = account.FetchFollowers(userName, 1000, cursor); err != nil {
+				slog.Error("FetchFollowers", "failed", err.Error())
+				continue
 			}
 
-		res := make([]*twitter.UserObj, 0, len(users))
+			success = true
+			res = make([]*twitter.UserObj, 0, len(users))
 
 			for _, v := range users {
 				sDec, _ := base64.StdEncoding.DecodeString(v.UserID)
@@ -186,8 +209,7 @@ func (f *FollowerRateLimit) Follower(userName string, cursor string) ([]*twitter
 					UserName: v.Username,
 				})
 			}
-
-		accChan <- account
-		return res, next, nil, err
 		}
+	}
+	return res, next, nil, err
 }

db.go

 package main
 
 import (
+	_ "code.wuban.net.cn/odysseus/twitter_syncer/docs"
 	"encoding/json"
 	"fmt"
 	"log/slog"
-	"time"
-
-	_ "code.wuban.net.cn/odysseus/twitter_syncer/docs"
 	// docs are generated by Swag CLI, you have to import them.
 	// replace with your own docs folder, usually "github.com/username/reponame/docs"
 	//_ "github.com/gofiber/swagger/example/docs"
@@ -42,12 +40,9 @@ func init() {
 		slog.Error(err.Error())
 	}
 
-	for k, v := range accounts {
+	for _, v := range accounts {
 		//v.Timer = time.NewTimer(time.Duration(k) * time.Duration(5) * time.Minute)
-		accChan <- ScraperWithTimer{
-			Scraper: v,
-			Timer:   time.NewTimer(time.Duration(k) * time.Duration(5) * time.Minute),
-		}
+		accChan <- v
 	}
 }