Skip to content

T
twitter_syncer
Commit 6655d06a authored by vicotor's avatar vicotor
Browse files
Options

add twitter swarm instead of account.

parent 548943c1
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 152 additions and 1087 deletions
accounts-db.go deleted 100644 → 0
View file @ 548943c1
package main
import (
"encoding/json"
"net/http"
)
type Account struct {
Username string `json:"username"`
Password string `json:"password"`
Email string `json:"email"`
// Cookies []byte `json:"cookies"`
F2A string `json:"two_fa_pk"`
Cookies []*http.Cookie `json:"cookies"`
Available bool `json:"available"`
}
func GetAvailableAccounts() ([]Account, error) {
data, count, err := client.From("accounts").Select("*", "exact", false).Eq("available", "true").Execute()
if err != nil {
return nil, err
}
//return count == 1, nil
_ = count
res := make([]Account, 0, count)
if err := json.Unmarshal(data, &res); err != nil {
return nil, err
}
return res, nil
}
// []*http.Cookie
func UpdateCookies(username string, cookies []*http.Cookie) error {
res, _, err := client.From("accounts").Update(&struct {
Cookies []*http.Cookie `json:"cookies"`
}{
Cookies: cookies,
}, "", "exact").Eq("username", username).Execute()
_ = res
return err
}
func UpdateCookiesByBytes(username string, cookies []byte) error {
res, _, err := client.From("accounts").Update(&struct {
Cookies []byte `json:"cookies"`
}{
Cookies: cookies,
}, "", "exact").Eq("username", username).Execute()
_ = res
return err
}
func SetNotAvailable(username string, errStr string) error {
res, _, err := client.From("accounts").Update(&struct {
// Error string `json:"error"`
Error string `json:"error"`
Available bool `json:"available"`
}{
Error: errStr,
Available: false,
}, "", "exact").Eq("username", username).Execute()
_ = res
return err
}
func AddAccount(acc Account) error {
//fmt.Printf("add account %v\n", acc)
res, _, err := client.From("accounts").Insert(acc, true, "", "representation", "").Execute()
_ = res
return err
}
accounts.go deleted 100644 → 0
View file @ 548943c1
package main
import (
"context"
"encoding/base64"
"fmt"
"github.com/pquerna/otp/totp"
"log/slog"
"strings"
"time"
twitter "github.com/g8rswimmer/go-twitter/v2"
twitterscraper "github.com/imperatrona/twitter-scraper"
"golang.org/x/time/rate"
)
func generateTOTP(secret string) (string, error) {
return totp.GenerateCode(secret, time.Now())
}
func GetLoginAccount() ([]ScraperWithTimer, error) {
accounts, err := GetAvailableAccounts()
if err != nil {
return nil, err
}
fmt.Println("len(accounts)", len(accounts))
res := make([]ScraperWithTimer, 0, len(accounts))
for _, v := range accounts {
needLogin := false
scraper := twitterscraper.New()
fmt.Println("user cookies ", v.Username, v.Email)
if false { // cookie is not valid, need login.
if v.Cookies != nil {
//var cookies []*http.Cookie
// if err := json.Unmarshal(v.Cookies, cookies); err != nil {
// slog.Error("cookies json.Unmarshal(", "err", err.Error())
// needLogin = true
// }
scraper.SetCookies(v.Cookies)
if !scraper.IsLoggedIn() {
needLogin = true
} else {
res = append(res, ScraperWithTimer{
Scraper: scraper,
AccountInfo: v,
})
continue
}
}
} else {
needLogin = true
}
fmt.Println("needLogin", needLogin)
if needLogin {
if v.F2A != "" {
code, _ := generateTOTP(v.F2A)
if err := scraper.AutoLogin(v.Username, v.Password, v.Email, code); err != nil {
SetNotAvailable(v.Username, err.Error())
slog.Error("scraper.Login", "err", err.Error())
continue
}
} else {
if err := scraper.AutoLogin(v.Username, v.Password, v.Email); err != nil {
SetNotAvailable(v.Username, err.Error())
slog.Error("scraper.Login", "err", err.Error())
continue
}
}
cookies := scraper.GetCookies()
if err := UpdateCookies(v.Username, cookies); err != nil {
slog.Error("cookies UpdateCookies", "err", err.Error())
continue
}
res = append(res, ScraperWithTimer{
Scraper: scraper,
AccountInfo: v,
})
}
}
return res, nil
}
type ScraperWithTimer struct {
*twitterscraper.Scraper
AccountInfo Account
Timer *time.Timer
}
var accChan chan ScraperWithTimer = make(chan ScraperWithTimer, 20)
// func init() {
// }
func NewFollowerOb() *FollowerRateLimit {
return &FollowerRateLimit{
RateLimit: rate.NewLimiter(rate.Every(15*time.Minute), 40),
}
}
type FollowerRateLimit struct {
RateLimit *rate.Limiter
Scraper *twitterscraper.Scraper
}
func (f *FollowerRateLimit) TryProfileFollowerCount(username string) (int, error) {
tryCount := 0
for {
if tryCount > 10 {
return 0, fmt.Errorf("can not get the %v follower count", username)
}
fc, err := f.ProfileFollowerCount(username)
if err != nil {
f.Scraper.GetGuestToken()
// twitterscraper.GetGuestToken()
// c.Scraper = twitterscraper.New()
slog.Error("ProfileFollowerCount", "err", err.Error())
tryCount++
time.Sleep(time.Second * time.Duration(tryCount))
continue
}
return fc, nil
}
}
func (f *FollowerRateLimit) ProfileFollowerCount(username string) (int, error) {
if f.Scraper == nil {
f.Scraper = twitterscraper.New()
}
pro, err := f.Scraper.GetProfile(username)
if err != nil {
return 0, err
}
return pro.FollowersCount, nil
}
//= rate.NewLimiter(rate.Every(15*time.Minute), 40)
func (f *FollowerRateLimit) Follower(userName string, cursor string) ([]*twitter.UserObj, string, *twitter.RateLimit, error) {
slog.Info("Follower request", "userName", userName, "cursor", cursor)
ctx := context.Background()
if err := f.RateLimit.Wait(ctx); err != nil { // This is a blocking call.
return nil, "", nil, err
}
var (
history = make(map[string]bool)
users []*twitterscraper.Profile
res []*twitter.UserObj
next string
err error
success bool = false
try = 0
)
for !success && try < 10 {
select {
case account := <-accChan:
accChan <- account
if _, exist := history[account.AccountInfo.Username]; exist {
// loop all account, exit.
try = 100
break
}
history[account.AccountInfo.Username] = true
if users, next, err = account.FetchFollowers(userName, 1000, cursor); err != nil {
slog.Error("FetchFollowers", "failed", err.Error())
continue
}
success = true
res = make([]*twitter.UserObj, 0, len(users))
for _, v := range users {
sDec, _ := base64.StdEncoding.DecodeString(v.UserID)
userId, _ := strings.CutPrefix(string(sDec), "User:")
res = append(res, &twitter.UserObj{
ID: userId,
Name: v.Name,
UserName: v.Username,
})
}
}
}
return res, next, nil, err
}
accounts_test.go
View file @ 6655d06a
......@@ -6,7 +6,6 @@ import (
"golang.org/x/time/rate"
"net/http"
"os"
"strings"
"testing"
"time"
......@@ -51,83 +50,6 @@ func TestProfile(t *testing.T) {
}
func TestAccounts(t *testing.T) {
accounts, err := GetAvailableAccounts()
if err != nil {
t.Fatal(err)
}
for k, v := range accounts {
t.Log(k, v)
}
}
// func TestSetCookies(t *testing.T) {
// accounts, err := GetAvailableAccounts()
// if err != nil {
// t.Fatal(err)
// }
// // for k, v := range accounts {
// // t.Log(k, v)
// // }
// data, err := json.Marshal(accounts)
// if err != nil {
// t.Fatal(err)
// }
// for _, v := range accounts {
// //t.Log(k, v)
// if err := UpdateCookies(v.Username, data); err != nil {
// t.Fatal(err)
// }
// }
// }
func TestGetLoginAccount(t *testing.T) {
accounts, err := GetLoginAccount()
if err != nil {
t.Fatal(err)
}
t.Log("login account ok")
for _, v := range accounts {
//v.Timer = time.NewTimer(time.Duration(k) * time.Duration(5) * time.Minute)
accChan <- v
}
for {
select {
case account := <-accChan:
<-account.Timer.C
users, _, err := account.WithDelay(300).FetchFollowers("bitcoin", 20, "")
if err != nil {
t.Log(err.Error())
continue
}
t.Log("len(users)", len(users), time.Now())
account.Timer = time.NewTimer(time.Hour)
accChan <- account
}
}
}
func TestCookies(t *testing.T) {
scraper := twitterscraper.New()
......@@ -145,12 +67,6 @@ func TestCookies(t *testing.T) {
panic("Invalid cookies")
}
// users, _, err := scraper.FetchFollowers("bitcoin", 20, "")
// if err != nil {
// t.Log(err.Error())
// continue
// }
}
func TestRateLimiter(t *testing.T) {
......@@ -162,124 +78,3 @@ func TestRateLimiter(t *testing.T) {
time.Sleep(1 * time.Second)
}
}
func TestAddAccount(t *testing.T) {
acclist := `ADeirdre20860----PtmcQvWd1DoD----tlugmeqmyt@rambler.ru----1589408cnVMmCa----P7G5GRG7ULCYKBMY----808f3b93ac4f9b0ac5085a3b1ba608e1c1a8e675
EulaSusann56500----SuIZROXhN14D----szahcqejrt@rambler.ru----9911065aAPM8ga----XDC6WMJEH4ILTTXN----f44e136089679d62247f353570be9cc78c60ff6b
MShondra82485----B7P38LYzz----mvwadpmfxr@rambler.ru----5973556MwG4gqa----I6XQJHQMUUL5MDXX----a43cbb2fd4a891e0513169131fe7dc7a0bb54f1f
KylieBritt10079----up1kj10rgjkry----xttuuxbtdc@rambler.ru----9507993gP7QJCa----GDZ6QWDQ5Q2KDOE5----330f127059d8a686cc0bc82e12aac5380fce9b03
StevieJ95145----9XqzE8JBSd----kkkfqtgkym@rambler.ru----5141799R36rxaa----ZFRHKNVV3TAU6GJ6----55bbfe00bfc34c4e9e7cc6d46eef0cd79ff31284`
accs := strings.Split(acclist, "\n")
for _, v := range accs {
info := strings.Split(v, "----")
if len(info) != 6 {
t.Errorf("invalid account info %s", v)
} else {
record := Account{
Username: info[0],
Password: info[1],
Email: info[2],
F2A: info[4],
Available: true,
}
if err := AddAccount(record); err != nil {
t.Error(err)
} else {
t.Logf("add account %s success", record.Username)
}
}
}
}
func TestLogin(t *testing.T) {
//userCookies := make(map[string][]*http.Cookie)
info := []struct {
Username string `json:"username"`
Password string `json:"password"`
Email string `json:"email"`
F2A string `json:"f2a"`
}{
//{"EulaSusann56500", "SuIZROXhN14D", "szahcqejrt@rambler.ru", "XDC6WMJEH4ILTTXN"},
{"MShondra82485", "B7P38LYzz", "mvwadpmfxr@rambler.ru", "I6XQJHQMUUL5MDXX"},
//{"KylieBritt10079", "up1kj10rgjkry", "xttuuxbtdc@rambler.ru", "GDZ6QWDQ5Q2KDOE5"},
}
tasks := make(chan string, 1000)
type HistoryInfo struct {
Users []string
Next string
}
filter := func(users []string, history *HistoryInfo) []string {
if history == nil {
return users
}
m := make(map[string]bool)
for _, v := range history.Users {
m[v] = true
}
newusers := make([]string, 0, 1000)
for _, v := range users {
if _, ok := m[v]; !ok {
newusers = append(newusers, v)
}
}
return newusers
}
history := make(map[string]*HistoryInfo)
tasks <- "sager"
accounts := make(map[string]*twitterscraper.Scraper)
for _, v := range info {
scraper := twitterscraper.New()
code, err := generateTOTP(v.F2A)
if err != nil {
t.Fatal(err)
}
{
if err := scraper.AutoLogin(v.Username, v.Password, v.Email, code); err != nil {
t.Error(err)
continue
} else {
t.Log("login success")
accounts[v.Username] = scraper
}
}
}
for {
select {
case task := <-tasks:
tHistory := history[task]
if tHistory == nil {
tHistory = &HistoryInfo{}
}
for k, v := range accounts {
newusers := make([]string, 0, 1000)
users, next, err := v.FetchFollowers(task, 1000, tHistory.Next)
if err != nil {
t.Error("fetch followers error", err, "now", time.Now())
if strings.Contains(err.Error(), "429") {
time.Sleep(1 * time.Minute)
}
continue
}
for _, u := range users {
newusers = append(newusers, u.UserID)
}
tHistory.Users = append(tHistory.Users, filter(newusers, tHistory)...)
tHistory.Next = next
if len(users) == 0 {
t.Logf("query follower with username:%s, next:%s, newuser=0",
k, next)
} else {
t.Logf("query follower with username:%s, next:%s, newuser[0]=%s, newusercount=%d",
k, next, users[0].Username, len(users))
}
time.Sleep(1 * time.Second)
}
history[task] = tHistory
tasks <- task
}
}
}
api_service.go
View file @ 6655d06a
package main
import (
"code.wuban.net.cn/odysseus/twitter_syncer/swarm"
"encoding/json"
"fmt"
"log/slog"
......@@ -259,7 +260,7 @@ func TaskAdd(c *fiber.Ctx) error {
})
}
fc, err = NewFollowerOb().TryProfileFollowerCount(req.TaskId)
fc, err = swarm.GetSwarm().GetFollowerCount(req.TaskId)
if err != nil {
return c.JSON(Res{
Code: 500,
......
client.go
View file @ 6655d06a
......@@ -8,7 +8,6 @@ import (
"github.com/dghubble/oauth1"
twitter "github.com/g8rswimmer/go-twitter/v2"
twitterscraper "github.com/imperatrona/twitter-scraper"
"golang.org/x/time/rate"
)
......@@ -56,8 +55,6 @@ type Client struct {
RetweeterRatelimiter *rate.Limiter
LikingUserRatelimiter *rate.Limiter
Scraper *twitterscraper.Scraper
}
type Config struct {
......@@ -68,12 +65,6 @@ type Config struct {
Token string `json:"token"`
}
// func NewFollowClient() *Client {
// return &Client{
// Scraper: twitterscraper.New(),
// }
// }
func NewLikeClient(cfg Config) *Client {
return NewClient(cfg, LikeRateLimit)
}
......
cookies copy.json deleted 100644 → 0
View file @ 548943c1
[
{
"domain": ".twitter.com",
"expirationDate": 1756348784.685643,
"hostOnly": false,
"httpOnly": false,
"name": "_ga",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "GA1.2.1387953141.1721788785",
"id": 1
},
{
"domain": ".twitter.com",
"hostOnly": false,
"httpOnly": true,
"name": "_twitter_sess",
"path": "/",
"sameSite": "unspecified",
"secure": true,
"session": true,
"storeId": "0",
"value": "BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo%250ASGFzaHsABjoKQHVzZWR7ADoPY3JlYXRlZF9hdGwrCFW%252FuN6QAToMY3NyZl9p%250AZCIlMmY4NmMwNTcxYzhjY2RkYmE2MTQwNWI0NDBjMDJjZWY6B2lkIiUyMzZh%250AODljY2Q3MTY5MDE2NDQ0ZmUxNmFiNzgxOTljNg%253D%253D--14854efc1913bc8618de3f72817a90f681021de9",
"id": 2
},
{
"domain": ".twitter.com",
"expirationDate": 1756283662.960466,
"hostOnly": false,
"httpOnly": true,
"name": "auth_token",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "6694c415423126c4099fc819b7d4142b578ebf23",
"id": 3
},
{
"domain": ".twitter.com",
"expirationDate": 1756283663.360468,
"hostOnly": false,
"httpOnly": false,
"name": "ct0",
"path": "/",
"sameSite": "lax",
"secure": true,
"session": false,
"storeId": "0",
"value": "fef7fe7d5d33870bef38f705111f2d16b3ab6236e89312d7339eb792b2f3c7faf25345307fc1509264eb277368f5bdaa9392b85de1941f2423d0180debd9efc8c312b6f8a4b7ddcd348c677fb3cc3d56",
"id": 4
},
{
"domain": ".twitter.com",
"expirationDate": 1756348723.939432,
"hostOnly": false,
"httpOnly": false,
"name": "des_opt_in",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "N",
"id": 5
},
{
"domain": ".twitter.com",
"expirationDate": 1756451096.332794,
"hostOnly": false,
"httpOnly": false,
"name": "dnt",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "1",
"id": 6
},
{
"domain": ".twitter.com",
"expirationDate": 1751425532.227206,
"hostOnly": false,
"httpOnly": false,
"name": "guest_id",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "v1%3A170860225263973367",
"id": 7
},
{
"domain": ".twitter.com",
"expirationDate": 1757316314.85636,
"hostOnly": false,
"httpOnly": false,
"name": "guest_id_ads",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "v1%3A170860225263973367",
"id": 8
},
{
"domain": ".twitter.com",
"expirationDate": 1757316314.856412,
"hostOnly": false,
"httpOnly": false,
"name": "guest_id_marketing",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "v1%3A170860225263973367",
"id": 9
},
{
"domain": ".twitter.com",
"expirationDate": 1756283662.960332,
"hostOnly": false,
"httpOnly": true,
"name": "kdt",
"path": "/",
"sameSite": "unspecified",
"secure": true,
"session": false,
"storeId": "0",
"value": "LvWUyXTiPsV6xBShzLMHFKwP1lA2QtXqGG3BbgFj",
"id": 10
},
{
"domain": ".twitter.com",
"expirationDate": 1754292309.732566,
"hostOnly": false,
"httpOnly": false,
"name": "night_mode",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "2",
"id": 11
},
{
"domain": ".twitter.com",
"expirationDate": 1757316314.856452,
"hostOnly": false,
"httpOnly": false,
"name": "personalization_id",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "\"v1_wADam4N8E7iunHX/QZVB6g==\"",
"id": 12
},
{
"domain": ".twitter.com",
"expirationDate": 1754292323.952039,
"hostOnly": false,
"httpOnly": false,
"name": "twid",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "u%3D1815666691260702720",
"id": 13
},
{
"domain": "twitter.com",
"expirationDate": 1737275644,
"hostOnly": true,
"httpOnly": false,
"name": "g_state",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "{\"i_l\":0}",
"id": 14
},
{
"domain": "twitter.com",
"hostOnly": true,
"httpOnly": false,
"name": "lang",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "en",
"id": 15
}
]
\ No newline at end of file
cookies.json deleted 100644 → 0
View file @ 548943c1
[
{
"domain": ".twitter.com",
"expirationDate": 1756348784.685643,
"hostOnly": false,
"httpOnly": false,
"name": "_ga",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "GA1.2.1387953141.1721788785",
"id": 1
},
{
"domain": ".twitter.com",
"hostOnly": false,
"httpOnly": true,
"name": "_twitter_sess",
"path": "/",
"sameSite": "unspecified",
"secure": true,
"session": true,
"storeId": "0",
"value": "BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo%250ASGFzaHsABjoKQHVzZWR7ADoPY3JlYXRlZF9hdGwrCFW%252FuN6QAToMY3NyZl9p%250AZCIlMmY4NmMwNTcxYzhjY2RkYmE2MTQwNWI0NDBjMDJjZWY6B2lkIiUyMzZh%250AODljY2Q3MTY5MDE2NDQ0ZmUxNmFiNzgxOTljNg%253D%253D--14854efc1913bc8618de3f72817a90f681021de9",
"id": 2
},
{
"domain": ".twitter.com",
"expirationDate": 1756283662.960466,
"hostOnly": false,
"httpOnly": true,
"name": "auth_token",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "6694c415423126c4099fc819b7d4142b578ebf23",
"id": 3
},
{
"domain": ".twitter.com",
"expirationDate": 1756283663.360468,
"hostOnly": false,
"httpOnly": false,
"name": "ct0",
"path": "/",
"sameSite": "lax",
"secure": true,
"session": false,
"storeId": "0",
"value": "fef7fe7d5d33870bef38f705111f2d16b3ab6236e89312d7339eb792b2f3c7faf25345307fc1509264eb277368f5bdaa9392b85de1941f2423d0180debd9efc8c312b6f8a4b7ddcd348c677fb3cc3d56",
"id": 4
},
{
"domain": ".twitter.com",
"expirationDate": 1756348723.939432,
"hostOnly": false,
"httpOnly": false,
"name": "des_opt_in",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "N",
"id": 5
},
{
"domain": ".twitter.com",
"expirationDate": 1756451096.332794,
"hostOnly": false,
"httpOnly": false,
"name": "dnt",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "1",
"id": 6
},
{
"domain": ".twitter.com",
"expirationDate": 1751425532.227206,
"hostOnly": false,
"httpOnly": false,
"name": "guest_id",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "v1%3A170860225263973367",
"id": 7
},
{
"domain": ".twitter.com",
"expirationDate": 1757316314.85636,
"hostOnly": false,
"httpOnly": false,
"name": "guest_id_ads",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "v1%3A170860225263973367",
"id": 8
},
{
"domain": ".twitter.com",
"expirationDate": 1757316314.856412,
"hostOnly": false,
"httpOnly": false,
"name": "guest_id_marketing",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "v1%3A170860225263973367",
"id": 9
},
{
"domain": ".twitter.com",
"expirationDate": 1756283662.960332,
"hostOnly": false,
"httpOnly": true,
"name": "kdt",
"path": "/",
"sameSite": "unspecified",
"secure": true,
"session": false,
"storeId": "0",
"value": "LvWUyXTiPsV6xBShzLMHFKwP1lA2QtXqGG3BbgFj",
"id": 10
},
{
"domain": ".twitter.com",
"expirationDate": 1754292309.732566,
"hostOnly": false,
"httpOnly": false,
"name": "night_mode",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "2",
"id": 11
},
{
"domain": ".twitter.com",
"expirationDate": 1757316314.856452,
"hostOnly": false,
"httpOnly": false,
"name": "personalization_id",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "\"v1_wADam4N8E7iunHX/QZVB6g==\"",
"id": 12
},
{
"domain": ".twitter.com",
"expirationDate": 1754292323.952039,
"hostOnly": false,
"httpOnly": false,
"name": "twid",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "u%3D1815666691260702720",
"id": 13
},
{
"domain": "twitter.com",
"expirationDate": 1737275644,
"hostOnly": true,
"httpOnly": false,
"name": "g_state",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "{\"i_l\":0}",
"id": 14
},
{
"domain": "twitter.com",
"hostOnly": true,
"httpOnly": false,
"name": "lang",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "en",
"id": 15
}
]
\ No newline at end of file
db.go
View file @ 6655d06a
......@@ -34,16 +34,6 @@ func init() {
}
}
accounts, err := GetLoginAccount()
if err != nil {
slog.Error(err.Error())
}
for _, v := range accounts {
//v.Timer = time.NewTimer(time.Duration(k) * time.Duration(5) * time.Minute)
accChan <- v
}
}
const FollowType = "followers"
......@@ -73,7 +63,7 @@ type ApiConfig struct {
}
type TaskInDB struct {
// ID int `json:"id"`
ID int `json:"id"`
User string `json:"user_id"`
TaskType string `json:"task_type"`
TaskId string `json:"task_id"`
......
db_test.go
View file @ 6655d06a
......@@ -3,6 +3,7 @@ package main
import (
"encoding/json"
"fmt"
"strconv"
"testing"
)
......@@ -118,3 +119,40 @@ func TestGetTasks(t *testing.T) {
// // fmt.Println("found", ok, task)
// }
func TestUpdateTask(t *testing.T) {
aonConfig := Config{
ApiKey: "u5HOlaBhMFNqXbs7lznEuUQVx",
ApiKeySecrect: "VhpTsl4TJUQi9FSAymajDCWfpgyJoK2d18i4lX9sPGiWc462nR",
AccessToken: "1783145144700874752-jS6Ow8fuXIJl2M6Ao9IuNwfNMpkM2r",
AccessTokenSecret: "NDgp8oQ72ulhjdv2Xs4PzzKOozq36bBQBKjAquUwe00pT",
Token: "AAAAAAAAAAAAAAAAAAAAAIZTwQEAAAAA4PxqcVAC4Wl2JYGHm4i%2Fcalh93o%3DDhSWWmvsHjj2XGInA3ZvYtz9C91nQ9YD1jbFQFJUHjK3niU4X1",
}
appBaseConfig := Config{
ApiKey: "UsNE66ZLht4EZVyu8K1eTKiN9",
ApiKeySecrect: "dUJ5CBW8NfrNy6QiAV6BPRY0q860yA018fBL6djgkaOcDB2RGf",
AccessToken: "1657936763577581569-9iWT0MTlRyXk6fVWvmJEFAxirL0N9d",
AccessTokenSecret: "p0sEpaSN4ZETDrXrzpdn9155s5ZNrzKHovyIUbHiI9tZj",
Token: "AAAAAAAAAAAAAAAAAAAAAPI%2BwwEAAAAAiMNc1Uy0oN1KMP8jKH28DmZZRBc%3DcsfwVvWi6vgIVIgw2Auz4SUhayv84GJA7jsIVNAaLoTOarJB5x",
}
tasks, err := QueryAllTask()
if err != nil {
t.Fatal(err)
}
for _, task := range tasks {
switch task.TaskId {
case "aon_aonet", "1800805503066661056", "1843181642300674228":
task.ApiConfig = parseToApiConfig(aonConfig)
case "1853676550111306025", "AppBaseGlobal":
task.ApiConfig = parseToApiConfig(appBaseConfig)
}
idstr := strconv.Itoa(task.ID)
t.Log("id str ", idstr, "apiconfig", task.ApiConfig)
// update task.
_, _, err := client.From("tasks").Update(&task, "", "exact").Eq("id", idstr).Execute()
if err != nil {
t.Error("update task error:", err)
}
}
}
go.mod
View file @ 6655d06a
......@@ -6,35 +6,26 @@ require (
github.com/dghubble/oauth1 v0.7.3
github.com/g8rswimmer/go-twitter/v2 v2.1.5
github.com/gofiber/fiber/v2 v2.52.5
github.com/imperatrona/twitter-scraper v0.0.14
github.com/gofiber/swagger v1.1.0
github.com/imperatrona/twitter-scraper v0.0.15
github.com/pquerna/otp v1.4.0
github.com/supabase-community/postgrest-go v0.0.11
github.com/supabase-community/supabase-go v0.0.4
github.com/swaggo/swag v1.16.3
github.com/xueqianLu/twitter-bee v0.0.0-20241213092233-9a0472c44890
golang.org/x/time v0.6.0
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
)
require (
github.com/AlexEidt/Vidio v1.5.1 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/dghubble/go-twitter v0.0.0-20221104224141-912508c3888b // indirect
github.com/dghubble/sling v1.4.0 // indirect
github.com/go-openapi/analysis v0.21.4 // indirect
github.com/go-openapi/errors v0.20.4 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/loads v0.21.2 // indirect
github.com/go-openapi/runtime v0.26.2 // indirect
github.com/go-openapi/spec v0.21.0 // indirect
github.com/go-openapi/strfmt v0.21.8 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-openapi/validate v0.22.3 // indirect
github.com/gofiber/contrib/swagger v1.2.0 // indirect
github.com/gofiber/swagger v1.1.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/compress v1.17.11 // indirect
......@@ -42,9 +33,6 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/pquerna/otp v1.4.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/supabase-community/functions-go v0.0.0-20220927045802-22373e6cb51d // indirect
github.com/supabase-community/gotrue-go v1.2.0 // indirect
......@@ -54,13 +42,10 @@ require (
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.56.0 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect
go.mongodb.org/mongo-driver v1.13.1 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/sys v0.26.0 // indirect
golang.org/x/tools v0.26.0 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
......
go.sum
View file @ 6655d06a
This diff is collapsed.
swarm/swarm.go 0 → 100644
View file @ 6655d06a
package swarm
import (
"fmt"
"github.com/g8rswimmer/go-twitter/v2"
"github.com/xueqianLu/twitter-bee/client"
"golang.org/x/time/rate"
"sync"
"time"
)
type ClientWithRateLimiter struct {
*client.BeeClient
RateLimit *rate.Limiter
}
type Swarm struct {
clients map[string]*ClientWithRateLimiter
mu sync.Mutex
}
var (
gSwarm *Swarm
)
func GetSwarm() *Swarm {
if gSwarm == nil {
gSwarm, _ = InitSwarm(nil)
}
return gSwarm
}
func InitSwarm(initialBees []string) (*Swarm, error) {
s := &Swarm{
clients: make(map[string]*ClientWithRateLimiter),
}
for _, bee := range initialBees {
s.AddClient(bee)
}
return s, nil
}
func (s *Swarm) AddClient(url string) {
s.mu.Lock()
defer s.mu.Unlock()
cli := new(ClientWithRateLimiter)
cli.BeeClient = client.NewBeeClient(url)
cli.RateLimit = rate.NewLimiter(rate.Every(15*time.Minute), 40)
s.clients[url] = cli
}
func (s *Swarm) RemoveClient(url string) {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.clients, url)
}
func (s *Swarm) GetFollowerCount(userID string) (int, error) {
s.mu.Lock()
defer s.mu.Unlock()
for _, cli := range s.clients {
res, err := cli.GetFollowerCount(userID)
if err == nil {
return res.Count, nil
}
}
return 0, fmt.Errorf("can not get the %v follower count", userID)
}
func (s *Swarm) GetFollowerList(user string, cursor string) ([]*twitter.UserObj, string, *twitter.RateLimit, error) {
s.mu.Lock()
defer s.mu.Unlock()
for _, cli := range s.clients {
if cli.RateLimit.Allow() == false {
continue
}
res, err := cli.GetFollowerList(user, cursor)
if err == nil {
list := make([]*twitter.UserObj, 0, len(res.List))
for _, u := range res.List {
list = append(list, &twitter.UserObj{
ID: u.ID,
Name: u.Name,
UserName: u.UserName,
})
}
return list, res.Next, nil, nil
} else {
}
}
return nil, "", nil, fmt.Errorf("can not get the %v follower list", user)
}
task.go
View file @ 6655d06a
package main
import (
"code.wuban.net.cn/odysseus/twitter_syncer/swarm"
"fmt"
"log/slog"
"sync"
......@@ -80,7 +81,7 @@ func (w *Work) RunJob(t TaskJob) chan<- interface{} {
if t.TaskType == FollowType {
cli := NewFollowerOb()
cli := swarm.GetSwarm()
secondTicker := time.NewTicker(time.Second * 3)
fiveMinutesTicker := time.NewTicker(time.Minute * 1)
......@@ -116,7 +117,7 @@ func (w *Work) RunJob(t TaskJob) chan<- interface{} {
if maybeFound {
fiveMinutesTicker.Reset(time.Minute * 3)
halfHourTicker.Reset(time.Minute * 30)
if err := Request(cli.Follower, page, t); err != nil {
if err := Request(cli.GetFollowerList, page, t); err != nil {
slog.Error(" page.Request", "task id", t.TaskId, "t.TaskType", t.TaskType, "err", err.Error())
continue
}
......@@ -131,7 +132,7 @@ func (w *Work) RunJob(t TaskJob) chan<- interface{} {
//recordFc = make(map[string]int)
case <-secondTicker.C:
fc, err := cli.TryProfileFollowerCount(t.TaskId)
fc, err := cli.GetFollowerCount(t.TaskId)
if err != nil {
slog.Error("TryProfileFollowerCount", "err", err.Error())
......@@ -157,7 +158,7 @@ func (w *Work) RunJob(t TaskJob) chan<- interface{} {
fmt.Println(" t.FollowerCount", t.FollowerCount)
if err := Request(cli.Follower, page, t); err != nil {
if err := Request(cli.GetFollowerList, page, t); err != nil {
slog.Error(" page.Request", "task id", t.TaskId, "t.TaskType", t.TaskType, "err", err.Error())
continue
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment