follower API validate

accounts.go

 package main
 
 import (
+	"context"
 	"encoding/base64"
 	"fmt"
 	"log/slog"
@@ -9,6 +10,7 @@ import (
 
 	twitter "github.com/g8rswimmer/go-twitter/v2"
 	twitterscraper "github.com/imperatrona/twitter-scraper"
+	"golang.org/x/time/rate"
 )
 
 func GetLoginAccount() ([]*twitterscraper.Scraper, error) {
@@ -95,11 +97,79 @@ var accChan chan ScraperWithTimer = make(chan ScraperWithTimer, 20)
 // func init() {
 
 // }
+func NewFollowerOb() *FollowerRateLimit {
 
-func Follower(userName string, next string) ([]*twitter.UserObj, string, *twitter.RateLimit, error) {
+	return &FollowerRateLimit{
+		RateLimit: rate.NewLimiter(rate.Every(15*time.Minute), 40),
+	}
+}
+
+type FollowerRateLimit struct {
+	RateLimit *rate.Limiter
+	Scraper   *twitterscraper.Scraper
+}
+
+func (f *FollowerRateLimit) TryProfileFollowerCount(username string) (int, error) {
+
+	i := 0
+	for {
+
+		if i > 10 {
+			return 0, fmt.Errorf("can not get the %v follower count", username)
+		}
+
+		fc, err := f.ProfileFollowerCount(username)
+
+		if err != nil {
+
+			f.Scraper.GetGuestToken()
+			// twitterscraper.GetGuestToken()
+			// c.Scraper = twitterscraper.New()
+			slog.Error("ProfileFollowerCount", "err", err.Error())
+			time.Sleep(time.Second * time.Duration(i))
+
+			continue
+		}
+
+		return fc, nil
+	}
+
+}
+
+func (f *FollowerRateLimit) ProfileFollowerCount(username string) (int, error) {
+
+	if f.Scraper == nil {
+		f.Scraper = twitterscraper.New()
+	}
+
+	//c.Scraper.GetProfile("aon_aonet")
+
+	pro, err := f.Scraper.GetProfile(username)
+
+	if err != nil {
+		return 0, err
+	}
+
+	return pro.FollowersCount, nil
+
+}
+
+//= rate.NewLimiter(rate.Every(15*time.Minute), 40)
+
+func (f *FollowerRateLimit) Follower(userName string, next string) ([]*twitter.UserObj, string, *twitter.RateLimit, error) {
 
 	fmt.Println("Follower request-------------")
 
+	//FollowerRateLimit.
+	ctx := context.Background()
+
+	err := f.RateLimit.Wait(ctx) // This is a blocking call.
+
+	//err := c.RetweeterRatelimiter.Wait(ctx) // This is a blocking call.
+	if err != nil {
+		return nil, "", nil, err
+	}
+
 	select {
 	case account := <-accChan:
 

api_db.go

@@ -146,6 +146,45 @@ func CheckTaskExist(userId, taskId, taskType string) (bool, error) {
 
 }
 
+func CheckFollowerTaskAndAccountCount() (bool, error) {
+
+	fc, err := FollowerTaskCount()
+	if err != nil {
+		return false, err
+	}
+
+	ac, err := AvailableAccountCount()
+
+	if err != nil {
+		return false, err
+	}
+
+	return fc < ac, nil
+}
+
+func FollowerTaskCount() (int64, error) {
+
+	_, count, err := client.From("tasks").Select("*", "exact", false).Eq("stop", "false").Execute()
+
+	if err != nil {
+		return 0, err
+	}
+
+	return count, nil
+
+}
+
+func AvailableAccountCount() (int64, error) {
+
+	_, count, err := client.From("accounts").Select("*", "exact", false).Eq("available", "false").Execute()
+
+	if err != nil {
+		return 0, err
+	}
+
+	return count, nil
+}
+
 func QueryProjectByUserId(userId string) ([]ProjectInDb, bool, error) {
 
 	data, count, err := client.From("project").Select("*", "exact", false).Eq("user_id", userId).Execute()
@@ -164,6 +203,18 @@ func QueryProjectByUserId(userId string) ([]ProjectInDb, bool, error) {
 
 }
 
+func QueryProjectByUserIdAndName(userName, userId string) (bool, error) {
+
+	_, count, err := client.From("project").Select("*", "exact", false).Eq("user_id", userId).Eq("username", userName).Execute()
+
+	if err != nil {
+		return false, err
+	}
+
+	return count >= 1, nil
+
+}
+
 func QueryAvailableProject() ([]ProjectInDb, error) {
 
 	data, count, err := client.From("project").Select("*", "exact", false).Eq("available", "true").Execute()

api_service.go

@@ -205,15 +205,60 @@ func TaskAdd(c *fiber.Ctx) error {
 
 	if req.TaskType == FollowType {
 
-		var err error
-		fc, err = NewFollowClient().TryProfileFollowerCount(req.TaskId)
+		ok, err := CheckFollowerTaskAndAccountCount()
 
 		if err != nil {
 			return c.JSON(Res{
 				Code: 500,
-				Msg:  "NewFollowClient().TryProfileFollowerCount " + req.TaskId,
+				Msg:  "CheckFollowerTaskAndAccountCount " + req.User,
 			})
 		}
+
+		if !ok {
+
+			return c.JSON(Res{
+				Code: 500,
+				Msg:  "CheckFollowerTaskAndAccountCount !ok",
+			})
+		}
+
+		ok, err = QueryProjectByUserIdAndName(req.TaskId, req.User)
+
+		if err != nil {
+			return c.JSON(Res{
+				Code: 500,
+				Msg:  "QueryProjectByUserIdAndName " + req.User,
+			})
+		}
+
+		if !ok {
+
+			return c.JSON(Res{
+				Code: 500,
+				Msg:  "QueryProjectByUserIdAndName !ok",
+			})
+		}
+		//NewFollowerOb()
+		//var err error
+		//fc, err = NewFollowClient().TryProfileFollowerCount(req.TaskId)
+
+		fc, err = NewFollowerOb().TryProfileFollowerCount(req.TaskId)
+
+		if err != nil {
+			return c.JSON(Res{
+				Code: 500,
+				Msg:  "QCheckTask " + req.User,
+			})
+		}
+
+		if ok {
+
+			return c.JSON(Res{
+				Code: 500,
+				Msg:  "task already existed",
+			})
+		}
+
 	}
 
 	job := TaskJob{
@@ -260,6 +305,25 @@ func TaskAdd(c *fiber.Ctx) error {
 	})
 }
 
+// func CheckErrOk(ok bool, err error, c *fiber.Ctx) error {
+
+// 	if err != nil {
+// 		return c.JSON(Res{
+// 			Code: 500,
+// 			Msg:  "QCheckTask " + req.User,
+// 		})
+// 	}
+
+// 	if ok {
+
+// 		return c.JSON(Res{
+// 			Code: 500,
+// 			Msg:  "task already existed",
+// 		})
+// 	}
+
+// }
+
 func TaskStop(c *fiber.Ctx) error {
 
 	slog.Info(c.Route().Path, "body", string(c.Request().Body()))

client.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"log/slog"
 	"net/http"
 	"time"
 
@@ -69,11 +68,11 @@ type Config struct {
 	Token             string `json:"token"`
 }
 
-func NewFollowClient() *Client {
-	return &Client{
-		Scraper: twitterscraper.New(),
-	}
-}
+// func NewFollowClient() *Client {
+// 	return &Client{
+// 		Scraper: twitterscraper.New(),
+// 	}
+// }
 
 func NewLikeClient(cfg Config) *Client {
 	return NewClient(cfg, LikeRateLimit)
@@ -145,50 +144,6 @@ func NewOAuth2Client0817() *Client {
 		Client: twitterClient,
 	}
 }
-func (c *Client) TryProfileFollowerCount(username string) (int, error) {
-
-	i := 0
-	for {
-
-		if i > 10 {
-			return 0, fmt.Errorf("can not get the %v follower count", username)
-		}
-
-		fc, err := c.ProfileFollowerCount(username)
-
-		if err != nil {
-
-			c.Scraper.GetGuestToken()
-			// twitterscraper.GetGuestToken()
-			// c.Scraper = twitterscraper.New()
-			slog.Error("ProfileFollowerCount", "err", err.Error())
-			time.Sleep(time.Second * time.Duration(i))
-
-			continue
-		}
-
-		return fc, nil
-	}
-
-}
-
-func (c *Client) ProfileFollowerCount(username string) (int, error) {
-
-	if c.Scraper == nil {
-		c.Scraper = twitterscraper.New()
-	}
-
-	//c.Scraper.GetProfile("aon_aonet")
-
-	pro, err := c.Scraper.GetProfile(username)
-
-	if err != nil {
-		return 0, err
-	}
-
-	return pro.FollowersCount, nil
-
-}
 
 func (c *Client) Retweeters(tweetId string, next string) ([]*twitter.UserObj, string, *twitter.RateLimit, error) {
 

idx.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"container/list"
+	"fmt"
 	"log/slog"
 	"time"
 
@@ -65,9 +66,10 @@ func (s *Idx) Idx(page []UserTask) (bool, *list.List) {
 		match := false
 
 		for ik, iv := range s.idx {
-			slog.Debug("match", "idx", iv.UserId, "page user id", v.UserId, "page user name", v.UserName)
+			slog.Info("match", "idx", iv.UserId, "page user id", v.UserId, "page user name", v.UserName)
 			if v.UserId == iv.UserId {
 				match = true
+				break
 				_, _ = k, ik
 			}
 		}
@@ -80,6 +82,8 @@ func (s *Idx) Idx(page []UserTask) (bool, *list.List) {
 			s.idx = s.newIdx
 			s.newIdx = make([]UserTask, 0, 10)
 
+			fmt.Println("newList", "new users", newList.Len())
+
 			return true, newList
 
 		} else {

task.go

@@ -74,14 +74,14 @@ func (w *Work) RunJob(t TaskJob) chan<- interface{} {
 			cli = NewRetweeterClient(t.Config)
 		} else if t.TaskType == TweetLikingUsersType {
 			cli = NewLikeClient(t.Config)
-		} else if t.TaskType == FollowType {
-			cli = NewFollowClient()
 		}
 
 		page := NewPageUsers(NewIdx(t.Idx))
 
 		if t.TaskType == FollowType {
 
+			cli := NewFollowerOb()
+
 			secondTicker := time.NewTicker(time.Second * 3)
 			fiveMinutesTicker := time.NewTicker(time.Minute * 5)
 			halfHourTicker := time.NewTicker(time.Minute * 30)
@@ -116,7 +116,7 @@ func (w *Work) RunJob(t TaskJob) chan<- interface{} {
 
 					if maybeFound {
 						halfHourTicker.Reset(time.Minute * 30)
-						if err := Request(Follower, page, t); err != nil {
+						if err := Request(cli.Follower, page, t); err != nil {
 							slog.Error(" page.Request", "task id", t.TaskId, "t.TaskType", t.TaskType, "err", err.Error())
 							continue
 						}
@@ -163,7 +163,7 @@ func (w *Work) RunJob(t TaskJob) chan<- interface{} {
 
 					// if maybeFound {
 					//halfHourTicker = (*time.Ticker)(time.NewTimer())  //Reset(time.Minute * 30)
-					if err := Request(Follower, page, t); err != nil {
+					if err := Request(cli.Follower, page, t); err != nil {
 						slog.Error(" page.Request", "task id", t.TaskId, "t.TaskType", t.TaskType, "err", err.Error())
 						continue
 					}