Skip to content

T
twitter_syncer
Commit 8d89b8ce authored by Ubuntu's avatar Ubuntu
Browse files
Options

add followers

parent 2d92293a
Show whitespace changes
Inline Side-by-side
Showing with 1380 additions and 72 deletions
11.json 0 → 100644
View file @ 8d89b8ce
[
{
"domain": ".twitter.com",
"expirationDate": 1758441362.275071,
"hostOnly": false,
"httpOnly": true,
"name": "auth_token",
"path": "/",
"sameSite": 0,
"secure": true,
"session": false,
"storeId": "0",
"value": "e73961c8546088afaa948c90e0e039e4e7a505ba",
"id": 3
},
{
"domain": ".twitter.com",
"expirationDate": 1758441363.239801,
"hostOnly": false,
"httpOnly": false,
"name": "ct0",
"path": "/",
"sameSite": 0,
"secure": true,
"session": false,
"storeId": "0",
"value": "d5ab30650093fc7ccdbdc1955903fd98f159db1e76742beb45ce73cdcaa81dc4bf71aea5564cc37bfc517e0946f246437a8cea47506df2593360a762bc36fe3af98c299057b2527406bb6cf7cc21766a",
"id": 4
},
{
"domain": ".twitter.com",
"expirationDate": 1755762974.437168,
"hostOnly": false,
"httpOnly": false,
"name": "twid",
"path": "/",
"sameSite": 0,
"secure": true,
"session": false,
"storeId": "0",
"value": "u%3D1825771992370540544",
"id": 15
},
{
"domain": ".twitter.com",
"expirationDate": 1758441362.27461,
"hostOnly": false,
"httpOnly": true,
"name": "kdt",
"path": "/",
"sameSite": 0,
"secure": true,
"session": false,
"storeId": "0",
"value": "mCPT0sbrzr60LYyoeGPWe71Y1OcHiWf8swMwJkK1",
"id": 12
}
]
\ No newline at end of file
12.json 0 → 100644
View file @ 8d89b8ce
[
{
"domain": ".twitter.com",
"expirationDate": 1758441520.606224,
"hostOnly": false,
"httpOnly": true,
"name": "auth_token",
"path": "/",
"sameSite": 0,
"secure": true,
"session": false,
"storeId": "0",
"value": "52ebe4da3acfea88582d4933e5eb33afcc063889",
"id": 3
},
{
"domain": ".twitter.com",
"expirationDate": 1758441521.589833,
"hostOnly": false,
"httpOnly": false,
"name": "ct0",
"path": "/",
"secure": true,
"session": false,
"storeId": "0",
"value": "d66d331170289f0b3ea6901967fb6d62495828cd1a78ad2858499bac2494d61bfce506178c4a0d09caa4d1debdaea9a81ea3bd1d4e1b830aec0ec28caf655be64ef114740758f40794aeb0fbf2f504dd",
"id": 4
},
{
"domain": ".twitter.com",
"expirationDate": 1755763122.949893,
"hostOnly": false,
"httpOnly": false,
"name": "twid",
"path": "/",
"secure": true,
"session": false,
"storeId": "0",
"value": "u%3D1825773249311158275",
"id": 15
},
{
"domain": ".twitter.com",
"expirationDate": 1758441520.603441,
"hostOnly": false,
"httpOnly": true,
"name": "kdt",
"path": "/",
"secure": true,
"session": false,
"storeId": "0",
"value": "mCPT0sbrzr60LYyoeGPWe71Y1OcHiWf8swMwJkK1",
"id": 12
}
]
13.json 0 → 100644
View file @ 8d89b8ce
[
{
"domain": ".twitter.com",
"expirationDate": 1758592308.9478,
"hostOnly": false,
"httpOnly": false,
"name": "_ga",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "GA1.2.1560808148.1724032309",
"id": 1
},
{
"domain": ".twitter.com",
"expirationDate": 1724311671.753654,
"hostOnly": false,
"httpOnly": true,
"name": "att",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "1-IenzjEs9m1KIPEytX1OlS3g4KRavlNXZX2PoOqEx",
"id": 2
},
{
"domain": ".twitter.com",
"expirationDate": 1758439670.573073,
"hostOnly": false,
"httpOnly": true,
"name": "auth_token",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "e1f42945b992c0c0a8bee5d7c698e13fe4df6e63",
"id": 3
},
{
"domain": ".twitter.com",
"expirationDate": 1758439670.93836,
"hostOnly": false,
"httpOnly": false,
"name": "ct0",
"path": "/",
"sameSite": "lax",
"secure": true,
"session": false,
"storeId": "0",
"value": "d19c60117ec160dcbddf22428774bab71c55a748318552009a64b9dde8acc45333ebcad8b7c05f748631a6b372842543ebad9d2792c64806b4ef5c3df8582de1b213e2c3ff7af8c07c1f80008e4e5b7c",
"id": 4
},
{
"domain": ".twitter.com",
"expirationDate": 1758590770.765236,
"hostOnly": false,
"httpOnly": false,
"name": "des_opt_in",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "Y",
"id": 5
},
{
"domain": ".twitter.com",
"expirationDate": 1758439645.782011,
"hostOnly": false,
"httpOnly": false,
"name": "dnt",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "1",
"id": 6
},
{
"domain": ".twitter.com",
"expirationDate": 1724829536.661659,
"hostOnly": false,
"httpOnly": false,
"name": "external_referer",
"path": "/",
"sameSite": "unspecified",
"secure": true,
"session": false,
"storeId": "0",
"value": "padhuUp37zjgzgv1mFWxJ5Xq0CLV%2BbpWuS41v6lN3QU%3D|0|8e8t2xd8A2w%3D",
"id": 7
},
{
"domain": ".twitter.com",
"expirationDate": 1724234167.785527,
"hostOnly": false,
"httpOnly": false,
"name": "gt",
"path": "/",
"sameSite": "unspecified",
"secure": true,
"session": false,
"storeId": "0",
"value": "1826158827626180842",
"id": 8
},
{
"domain": ".twitter.com",
"expirationDate": 1758439649.230565,
"hostOnly": false,
"httpOnly": false,
"name": "guest_id",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "v1%3A172422524915073785",
"id": 9
},
{
"domain": ".twitter.com",
"expirationDate": 1758534122.272825,
"hostOnly": false,
"httpOnly": false,
"name": "guest_id_ads",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "v1%3A172397411571272058",
"id": 10
},
{
"domain": ".twitter.com",
"expirationDate": 1758534122.272775,
"hostOnly": false,
"httpOnly": false,
"name": "guest_id_marketing",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "v1%3A172397411571272058",
"id": 11
},
{
"domain": ".twitter.com",
"expirationDate": 1758439670.572575,
"hostOnly": false,
"httpOnly": true,
"name": "kdt",
"path": "/",
"sameSite": "unspecified",
"secure": true,
"session": false,
"storeId": "0",
"value": "mCPT0sbrzr60LYyoeGPWe71Y1OcHiWf8swMwJkK1",
"id": 12
},
{
"domain": ".twitter.com",
"expirationDate": 1755761273.98706,
"hostOnly": false,
"httpOnly": false,
"name": "night_mode",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "2",
"id": 13
},
{
"domain": ".twitter.com",
"expirationDate": 1758534122.272848,
"hostOnly": false,
"httpOnly": false,
"name": "personalization_id",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "\"v1_3E9bvxHKMrYGUJQEBNa1xQ==\"",
"id": 14
},
{
"domain": ".twitter.com",
"expirationDate": 1755761275.700381,
"hostOnly": false,
"httpOnly": false,
"name": "twid",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "u%3D1825774185513426944",
"id": 15
},
{
"domain": ".twitter.com",
"expirationDate": 1755568288,
"hostOnly": false,
"httpOnly": false,
"name": "twtr_pixel_opt_in",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "Y",
"id": 16
},
{
"domain": "twitter.com",
"hostOnly": true,
"httpOnly": false,
"name": "lang",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "en",
"id": 17
}
]
\ No newline at end of file
14.json 0 → 100644
View file @ 8d89b8ce
[
{
"domain": ".twitter.com",
"expirationDate": 1758441717.077623,
"hostOnly": false,
"httpOnly": true,
"name": "kdt",
"path": "/",
"secure": true,
"session": false,
"storeId": "0",
"value": "mCPT0sbrzr60LYyoeGPWe71Y1OcHiWf8swMwJkK1",
"id": 12
},
{
"domain": ".twitter.com",
"expirationDate": 1755763319.2501,
"hostOnly": false,
"httpOnly": false,
"name": "twid",
"path": "/",
"secure": true,
"session": false,
"storeId": "0",
"value": "u%3D1825775397222617088",
"id": 15
},
{
"domain": ".twitter.com",
"expirationDate": 1758441717.449801,
"hostOnly": false,
"httpOnly": false,
"name": "ct0",
"path": "/",
"secure": true,
"session": false,
"storeId": "0",
"value": "d11694eb2cc9e7488a59e5c276904e2fbcbfc165e78614ef0cd521849c7cf8febe8baa2a9d12825331e0641ee2d1b7ef691b4552fdf36b3b0e99749a1520bde4479908b7a20fa9835ea5f1a0aec03b8e",
"id": 4
},
{
"domain": ".twitter.com",
"expirationDate": 1758441717.078207,
"hostOnly": false,
"httpOnly": true,
"name": "auth_token",
"path": "/",
"secure": true,
"session": false,
"storeId": "0",
"value": "2692e1420589800c115c1a49df562f3fbc7ef4e6",
"id": 3
}
]
\ No newline at end of file
15.json 0 → 100644
View file @ 8d89b8ce
[
{
"domain": ".twitter.com",
"expirationDate": 1758439840.582876,
"hostOnly": false,
"httpOnly": true,
"name": "kdt",
"path": "/",
"sameSite": 0,
"secure": true,
"session": false,
"storeId": "0",
"value": "mCPT0sbrzr60LYyoeGPWe71Y1OcHiWf8swMwJkK1",
"id": 12
},
{
"domain": ".twitter.com",
"expirationDate": 1755761444.486664,
"hostOnly": false,
"httpOnly": false,
"name": "twid",
"path": "/",
"sameSite": 0,
"secure": true,
"session": false,
"storeId": "0",
"value": "u%3D1825776446427791360",
"id": 15
},
{
"domain": ".twitter.com",
"expirationDate": 1758439840.944653,
"hostOnly": false,
"httpOnly": false,
"name": "ct0",
"path": "/",
"sameSite": 0,
"secure": true,
"session": false,
"storeId": "0",
"value": "a401af784dd1ac5fbf6cb1dcec6ad82cce11616d3a057dd727cae2bf2f47c40490710079f1cad2ef6e47faa5e4316c5dd1ed5617b280bd77787d918733e194f88a8a0a24607e6769e761dcdffa309772",
"id": 4
},
{
"domain": ".twitter.com",
"expirationDate": 1758439840.583027,
"hostOnly": false,
"httpOnly": true,
"name": "auth_token",
"path": "/",
"sameSite": 0,
"secure": true,
"session": false,
"storeId": "0",
"value": "480728ab03b0dfe07e8e34eb5231b07958846c51",
"id": 3
}
]
\ No newline at end of file
16.json 0 → 100644
View file @ 8d89b8ce
[
{
"domain": ".twitter.com",
"expirationDate": 1758441877.11485,
"hostOnly": false,
"httpOnly": true,
"name": "kdt",
"path": "/",
"secure": true,
"session": false,
"storeId": "0",
"value": "mCPT0sbrzr60LYyoeGPWe71Y1OcHiWf8swMwJkK1",
"id": 12
},
{
"domain": ".twitter.com",
"expirationDate": 1755763479.156539,
"hostOnly": false,
"httpOnly": false,
"name": "twid",
"path": "/",
"secure": true,
"session": false,
"storeId": "0",
"value": "u%3D1825777453769650176",
"id": 15
},
{
"domain": ".twitter.com",
"expirationDate": 1758441877.714491,
"hostOnly": false,
"httpOnly": false,
"name": "ct0",
"path": "/",
"secure": true,
"session": false,
"storeId": "0",
"value": "533f3852926f49769cc6ed6f449ee1e351b79adb6f837ef1f96e93e4c73af10d07b8968cf82b76bf77e60e333f6a4530d883f56559b09716d1c0ac01f333e37e247b3f64b3547ce38d325da90f41afa9",
"id": 4
},
{
"domain": ".twitter.com",
"expirationDate": 1758441877.115212,
"hostOnly": false,
"httpOnly": true,
"name": "auth_token",
"path": "/",
"secure": true,
"session": false,
"storeId": "0",
"value": "4f74ba9c0ca532fe643ee055ee96cfa01dce9c66",
"id": 3
}
]
accounts-db.go 0 → 100644
View file @ 8d89b8ce
package main
import (
"encoding/json"
"net/http"
)
type Account struct {
Username string `json:"username"`
Password string `json:"password"`
Email string `json:"email"`
// Cookies []byte `json:"cookies"`
Cookies []*http.Cookie `json:"cookies"`
Available bool `json:"available"`
}
func GetAvailableAccounts() ([]Account, error) {
data, count, err := client.From("accounts").Select("*", "exact", false).Eq("available", "true").Execute()
if err != nil {
return nil, err
}
//return count == 1, nil
_ = count
res := make([]Account, 0, count)
if err := json.Unmarshal(data, &res); err != nil {
return nil, err
}
return res, nil
}
// []*http.Cookie
func UpdateCookies(username string, cookies []*http.Cookie) error {
res, _, err := client.From("accounts").Update(&struct {
Cookies []*http.Cookie `json:"cookies"`
}{
Cookies: cookies,
}, "", "exact").Eq("username", username).Execute()
_ = res
return err
}
func UpdateCookiesByBytes(username string, cookies []byte) error {
res, _, err := client.From("accounts").Update(&struct {
Cookies []byte `json:"cookies"`
}{
Cookies: cookies,
}, "", "exact").Eq("username", username).Execute()
_ = res
return err
}
func SetNotAvailable(username string, errStr string) error {
res, _, err := client.From("accounts").Update(&struct {
// Error string `json:"error"`
Error string `json:"error"`
Available bool `json:"available"`
}{
Error: errStr,
Available: false,
}, "", "exact").Eq("username", username).Execute()
_ = res
return err
}
accounts.go
View file @ 8d89b8ce
package main
import "encoding/json"
type Account struct {
Username string `json:"username"`
Password string `json:"password"`
Email string `json:"email"`
Cookies []byte `json:"cookies"`
Available bool `json:"available"`
}
import (
"encoding/base64"
"fmt"
"log/slog"
"strings"
"time"
twitter "github.com/g8rswimmer/go-twitter/v2"
twitterscraper "github.com/imperatrona/twitter-scraper"
)
func GetAvailableAccounts() ([]Account, error) {
func GetLoginAccount() ([]*twitterscraper.Scraper, error) {
data, count, err := client.From("accounts").Select("*", "exact", false).Eq("available", "true").Execute()
accounts, err := GetAvailableAccounts()
if err != nil {
return nil, err
}
//return count == 1, nil
fmt.Println("len(accounts)", len(accounts))
_ = count
res := make([]*twitterscraper.Scraper, 0, len(accounts))
res := make([]Account, 0, count)
if err := json.Unmarshal(data, &res); err != nil {
return nil, err
for _, v := range accounts {
needLogin := false
scraper := twitterscraper.New()
fmt.Println("user cookies ", v.Username, v.Email)
if v.Cookies != nil {
//var cookies []*http.Cookie
// if err := json.Unmarshal(v.Cookies, cookies); err != nil {
// slog.Error("cookies json.Unmarshal(", "err", err.Error())
// needLogin = true
// }
scraper.SetCookies(v.Cookies)
if !scraper.IsLoggedIn() {
needLogin = true
} else {
res = append(res, scraper)
continue
}
} else {
needLogin = true
}
fmt.Println("needLogin", needLogin)
if needLogin {
//if err := scraper.Login("wuljKNFZmI87376", "Twitter24081907", "000007@agidev.io")
if err := scraper.Login(v.Username, v.Password, v.Email); err != nil {
SetNotAvailable(v.Username, err.Error())
slog.Error("scraper.Login", "err", err.Error())
continue
}
cookies := scraper.GetCookies()
// data, err := json.Marshal(cookies)
// if err != nil {
// slog.Error("cookies json.Marshal", "err", err.Error())
// continue
// }
//fmt.Println("user cookies", v.Username, v.Email, string(data))
if err := UpdateCookies(v.Username, cookies); err != nil {
slog.Error("cookies UpdateCookies", "err", err.Error())
continue
}
res = append(res, scraper)
}
}
return res, nil
}
type ScraperWithTimer struct {
*twitterscraper.Scraper
Timer *time.Timer
Email string
UserName string
}
func UpdateCookies(username string, cookies []byte) error {
var accChan chan ScraperWithTimer = make(chan ScraperWithTimer, 20)
// func init() {
// }
return nil
func Follower(userName string, next string) ([]*twitter.UserObj, string, *twitter.RateLimit, error) {
fmt.Println("Follower request-------------")
select {
case account := <-accChan:
//users, next, err := account.WithDelay(300).FetchFollowers("bitcoin", 20, "")
users, next, err := account.FetchFollowers(userName, 20, "")
if err != nil {
slog.Error(err.Error())
}
res := make([]*twitter.UserObj, 0, len(users))
for _, v := range users {
sDec, _ := base64.StdEncoding.DecodeString(v.UserID)
userId, _ := strings.CutPrefix(string(sDec), "User:")
res = append(res, &twitter.UserObj{
ID: userId,
Name: v.Name,
UserName: v.Username,
})
}
accChan <- account
return res, next, nil, nil
}
}
accounts_test.go
View file @ 8d89b8ce
package main
import "testing"
import (
"encoding/json"
"net/http"
"os"
"testing"
"time"
twitterscraper "github.com/imperatrona/twitter-scraper"
)
/*
{"Avatar":"https://pbs.twimg.com/profile_images/1812708680594296832/TkKsEyvW_normal.jpg","Banner":"https://pbs.twimg.com/profile_banners/1783145144700874752/1721029871","Biography":"#AON: The best way to quickly build and share Web3 #Al Apps\n\nJoin our community: https://t.co/ECV29IujSK","Birthday":"","FollowersCount":19697,"FollowingCount":72,"FriendsCount":37,"IsPrivate":false,"IsVerified":false,"Joined":"2024-04-24T14:45:19Z","LikesCount":72,"ListedCount":3,"Location":"Hong Kong ","Name":"AON","PinnedTweetIDs":["1800805503066661056"],"TweetsCount":129,"URL":"https://twitter.com/aon_aonet","UserID":"1783145144700874752","Username":"aon_aonet","Website":"https://aonnet.io/","Sensitive":false,"Following":false,"FollowedBy":false}
*/
func TestProfile(t *testing.T) {
scraper := twitterscraper.New()
i := 0
for {
i = i + 1
t.Log(i, time.Now())
pro, err := scraper.GetProfile("aon_aonet")
if err != nil {
t.Fatal(err)
}
proAsJson, err := json.Marshal(pro)
if err != nil {
t.Fatal(err)
}
t.Log(string(proAsJson))
t.Log(pro.FollowersCount)
time.Sleep(time.Second)
}
}
func TestAccounts(t *testing.T) {
......@@ -15,3 +61,94 @@ func TestAccounts(t *testing.T) {
}
}
// func TestSetCookies(t *testing.T) {
// accounts, err := GetAvailableAccounts()
// if err != nil {
// t.Fatal(err)
// }
// // for k, v := range accounts {
// // t.Log(k, v)
// // }
// data, err := json.Marshal(accounts)
// if err != nil {
// t.Fatal(err)
// }
// for _, v := range accounts {
// //t.Log(k, v)
// if err := UpdateCookies(v.Username, data); err != nil {
// t.Fatal(err)
// }
// }
// }
func TestGetLoginAccount(t *testing.T) {
accounts, err := GetLoginAccount()
if err != nil {
t.Fatal(err)
}
t.Log("login account ok")
for k, v := range accounts {
//v.Timer = time.NewTimer(time.Duration(k) * time.Duration(5) * time.Minute)
accChan <- ScraperWithTimer{
Scraper: v,
Timer: time.NewTimer(time.Duration(k) * time.Duration(5) * time.Minute),
}
}
for {
select {
case account := <-accChan:
<-account.Timer.C
users, _, err := account.WithDelay(300).FetchFollowers("bitcoin", 20, "")
if err != nil {
t.Log(err.Error())
continue
}
t.Log("len(users)", len(users), time.Now())
account.Timer = time.NewTimer(time.Hour)
accChan <- account
}
}
}
func TestCookies(t *testing.T) {
scraper := twitterscraper.New()
var cookies []*http.Cookie
f, _ := os.Open("16.json")
if err := json.NewDecoder(f).Decode(&cookies); err != nil {
t.Fatal(err)
}
t.Log("len(cookies)", len(cookies))
scraper.SetCookies(cookies)
if !scraper.IsLoggedIn() {
panic("Invalid cookies")
}
// users, _, err := scraper.FetchFollowers("bitcoin", 20, "")
// if err != nil {
// t.Log(err.Error())
// continue
// }
}
api_db.go
View file @ 8d89b8ce
......@@ -4,7 +4,7 @@ import "encoding/json"
func VerifyFollowerInDb(userId, followerId string) (bool, error) {
_, count, err := client.From("followers").Select("*", "exact", false).Eq("user_id", userId).Eq("follower_id", followerId).Execute()
_, count, err := client.From("followers").Select("*", "exact", false).Eq("task_id", userId).Eq("user_id", followerId).Execute()
if err != nil {
return false, err
......@@ -35,7 +35,7 @@ func VerifyLikeInDb(tweetId, userId string) (bool, error) {
return count == 1, nil
}
func AddTaskInsert(req AddTaskReq) error {
func AddTaskInsert(req AddTaskReq, followerCount int) error {
task := TaskInDB{
User: req.User,
......@@ -43,6 +43,7 @@ func AddTaskInsert(req AddTaskReq) error {
TaskId: req.TaskId,
Start: true,
Stop: false,
FollowerCount: followerCount,
}
res, _, err := client.From("tasks").Insert(task, true, "", "representation", "").Execute()
......@@ -50,7 +51,19 @@ func AddTaskInsert(req AddTaskReq) error {
_ = res
return err
}
func UpdateFollowerTaskCount(taskId, taskType, userId string, fc int) error {
res, _, err := client.From("tasks").Update(&struct {
FC int `json:"follower_count"`
}{
FC: fc,
}, "", "exact").Eq("task_type", taskType).Eq("task_id", taskId).Eq("task_type", taskType).Execute()
_ = res
return err
}
func StopTaskUpdate(req AddTaskReq) error {
......@@ -96,16 +109,29 @@ func AddOrUpdateProject(cfg ProjectReq, user UserInfo) error {
return err
}
func QueryProjectByKeysAndToken(cfg ProjectReq) (bool, error) {
func QueryProjectByKeysAndToken(cfg ProjectReq) (UserInfo, bool, error) {
_, count, err := client.From("project").Select("*", "exact", false).Eq("api_key", cfg.ApiKey).Eq("api_key_secret", cfg.ApiKeySecrect).Eq("token", cfg.Token).Eq("access_token", cfg.AccessToken).Eq("access_token_secret", cfg.AccessTokenSecret).Execute()
data, count, err := client.From("project").Select("*", "exact", false).Eq("api_key", cfg.ApiKey).Eq("api_key_secret", cfg.ApiKeySecrect).Eq("token", cfg.Token).Eq("access_token", cfg.AccessToken).Eq("access_token_secret", cfg.AccessTokenSecret).Execute()
if err != nil {
return false, err
return UserInfo{}, false, err
}
return count == 1, nil
if count == 0 {
return UserInfo{}, false, nil
}
res := make([]ProjectInDb, 0, count)
if err := json.Unmarshal(data, &res); err != nil {
return UserInfo{}, false, err
}
if len(res) > 0 {
return res[0].UserInfo, count >= 1, nil
}
return UserInfo{}, count >= 1, nil
}
func CheckTaskExist(userId, taskId, taskType string) (bool, error) {
......
api_service.go
View file @ 8d89b8ce
......@@ -70,7 +70,7 @@ func Project(c *fiber.Ctx) error {
slog.Info("cfg", "project", req.Project, "ApiKey", req.ApiKey, "req.ApiKeySecrect", req.ApiKeySecrect, "req.AccessToken", req.AccessToken, "req.AccessTokenSecret", req.AccessTokenSecret, "req.Token", req.Token)
ok, err := QueryProjectByKeysAndToken(req)
data, ok, err := QueryProjectByKeysAndToken(req)
if err != nil {
slog.Error("QueryProject", "err", err.Error())
......@@ -84,6 +84,7 @@ func Project(c *fiber.Ctx) error {
return c.JSON(ProjectRes{
Code: 500,
Msg: "already existed",
Data: data,
})
}
......@@ -137,6 +138,7 @@ type AddTaskReq struct {
User string `json:"user_id"`
TaskType string `json:"task_type"`
TaskId string `json:"task_id"`
//FollowCount int
}
func TaskAdd(c *fiber.Ctx) error {
......@@ -199,6 +201,20 @@ func TaskAdd(c *fiber.Ctx) error {
Msg: "task already existed",
})
}
fc := 0
if req.TaskType == FollowType {
var err error
fc, err = NewFollowClient().TryProfileFollowerCount(req.TaskId)
if err != nil {
return c.JSON(Res{
Code: 500,
Msg: "NewFollowClient().TryProfileFollowerCount " + req.TaskId,
})
}
}
job := TaskJob{
Config: projects[0].Config,
......@@ -207,6 +223,7 @@ func TaskAdd(c *fiber.Ctx) error {
UserId: req.User,
TaskId: req.TaskId,
TaskType: req.TaskType,
FollowerCount: fc,
}
fmt.Println(job.String())
......@@ -225,7 +242,7 @@ func TaskAdd(c *fiber.Ctx) error {
slog.Info("add task into db", "user", req.User, "TaskType", req.TaskType, "TaskId", req.TaskId)
// req.AddOrStop = true
err = AddTaskInsert(req)
err = AddTaskInsert(req, fc)
if err != nil {
......@@ -336,8 +353,8 @@ func VerifyRetweeter(c *fiber.Ctx) error {
func VerifyFollower(c *fiber.Ctx) error {
userId := c.Query("user_id")
followerId := c.Query("user_id")
userId := c.Query("task_id")
followerId := c.Query("follower_id")
if len(userId) == 0 || len(followerId) == 0 {
slog.Error("VerifyFollower", "userId", userId, "followerId", followerId)
......
client.go
View file @ 8d89b8ce
......@@ -3,11 +3,13 @@ package main
import (
"context"
"fmt"
"log/slog"
"net/http"
"time"
"github.com/dghubble/oauth1"
twitter "github.com/g8rswimmer/go-twitter/v2"
twitterscraper "github.com/imperatrona/twitter-scraper"
"golang.org/x/time/rate"
)
......@@ -55,6 +57,8 @@ type Client struct {
RetweeterRatelimiter *rate.Limiter
LikingUserRatelimiter *rate.Limiter
Scraper *twitterscraper.Scraper
}
type Config struct {
......@@ -65,6 +69,12 @@ type Config struct {
Token string `json:"token"`
}
func NewFollowClient() *Client {
return &Client{
Scraper: twitterscraper.New(),
}
}
func NewLikeClient(cfg Config) *Client {
return NewClient(cfg, LikeRateLimit)
}
......@@ -135,6 +145,50 @@ func NewOAuth2Client0817() *Client {
Client: twitterClient,
}
}
func (c *Client) TryProfileFollowerCount(username string) (int, error) {
i := 0
for {
if i > 10 {
return 0, fmt.Errorf("can not get the %v follower count", username)
}
fc, err := c.ProfileFollowerCount(username)
if err != nil {
c.Scraper.GetGuestToken()
// twitterscraper.GetGuestToken()
// c.Scraper = twitterscraper.New()
slog.Error("ProfileFollowerCount", "err", err.Error())
time.Sleep(time.Second * time.Duration(i))
continue
}
return fc, nil
}
}
func (c *Client) ProfileFollowerCount(username string) (int, error) {
if c.Scraper == nil {
c.Scraper = twitterscraper.New()
}
//c.Scraper.GetProfile("aon_aonet")
pro, err := c.Scraper.GetProfile(username)
if err != nil {
return 0, err
}
return pro.FollowersCount, nil
}
func (c *Client) Retweeters(tweetId string, next string) ([]*twitter.UserObj, string, *twitter.RateLimit, error) {
......
cookies copy.json 0 → 100644
View file @ 8d89b8ce
[
{
"domain": ".twitter.com",
"expirationDate": 1756348784.685643,
"hostOnly": false,
"httpOnly": false,
"name": "_ga",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "GA1.2.1387953141.1721788785",
"id": 1
},
{
"domain": ".twitter.com",
"hostOnly": false,
"httpOnly": true,
"name": "_twitter_sess",
"path": "/",
"sameSite": "unspecified",
"secure": true,
"session": true,
"storeId": "0",
"value": "BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo%250ASGFzaHsABjoKQHVzZWR7ADoPY3JlYXRlZF9hdGwrCFW%252FuN6QAToMY3NyZl9p%250AZCIlMmY4NmMwNTcxYzhjY2RkYmE2MTQwNWI0NDBjMDJjZWY6B2lkIiUyMzZh%250AODljY2Q3MTY5MDE2NDQ0ZmUxNmFiNzgxOTljNg%253D%253D--14854efc1913bc8618de3f72817a90f681021de9",
"id": 2
},
{
"domain": ".twitter.com",
"expirationDate": 1756283662.960466,
"hostOnly": false,
"httpOnly": true,
"name": "auth_token",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "6694c415423126c4099fc819b7d4142b578ebf23",
"id": 3
},
{
"domain": ".twitter.com",
"expirationDate": 1756283663.360468,
"hostOnly": false,
"httpOnly": false,
"name": "ct0",
"path": "/",
"sameSite": "lax",
"secure": true,
"session": false,
"storeId": "0",
"value": "fef7fe7d5d33870bef38f705111f2d16b3ab6236e89312d7339eb792b2f3c7faf25345307fc1509264eb277368f5bdaa9392b85de1941f2423d0180debd9efc8c312b6f8a4b7ddcd348c677fb3cc3d56",
"id": 4
},
{
"domain": ".twitter.com",
"expirationDate": 1756348723.939432,
"hostOnly": false,
"httpOnly": false,
"name": "des_opt_in",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "N",
"id": 5
},
{
"domain": ".twitter.com",
"expirationDate": 1756451096.332794,
"hostOnly": false,
"httpOnly": false,
"name": "dnt",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "1",
"id": 6
},
{
"domain": ".twitter.com",
"expirationDate": 1751425532.227206,
"hostOnly": false,
"httpOnly": false,
"name": "guest_id",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "v1%3A170860225263973367",
"id": 7
},
{
"domain": ".twitter.com",
"expirationDate": 1757316314.85636,
"hostOnly": false,
"httpOnly": false,
"name": "guest_id_ads",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "v1%3A170860225263973367",
"id": 8
},
{
"domain": ".twitter.com",
"expirationDate": 1757316314.856412,
"hostOnly": false,
"httpOnly": false,
"name": "guest_id_marketing",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "v1%3A170860225263973367",
"id": 9
},
{
"domain": ".twitter.com",
"expirationDate": 1756283662.960332,
"hostOnly": false,
"httpOnly": true,
"name": "kdt",
"path": "/",
"sameSite": "unspecified",
"secure": true,
"session": false,
"storeId": "0",
"value": "LvWUyXTiPsV6xBShzLMHFKwP1lA2QtXqGG3BbgFj",
"id": 10
},
{
"domain": ".twitter.com",
"expirationDate": 1754292309.732566,
"hostOnly": false,
"httpOnly": false,
"name": "night_mode",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "2",
"id": 11
},
{
"domain": ".twitter.com",
"expirationDate": 1757316314.856452,
"hostOnly": false,
"httpOnly": false,
"name": "personalization_id",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "\"v1_wADam4N8E7iunHX/QZVB6g==\"",
"id": 12
},
{
"domain": ".twitter.com",
"expirationDate": 1754292323.952039,
"hostOnly": false,
"httpOnly": false,
"name": "twid",
"path": "/",
"sameSite": "no_restriction",
"secure": true,
"session": false,
"storeId": "0",
"value": "u%3D1815666691260702720",
"id": 13
},
{
"domain": "twitter.com",
"expirationDate": 1737275644,
"hostOnly": true,
"httpOnly": false,
"name": "g_state",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": false,
"storeId": "0",
"value": "{\"i_l\":0}",
"id": 14
},
{
"domain": "twitter.com",
"hostOnly": true,
"httpOnly": false,
"name": "lang",
"path": "/",
"sameSite": "unspecified",
"secure": false,
"session": true,
"storeId": "0",
"value": "en",
"id": 15
}
]
\ No newline at end of file
db.go
View file @ 8d89b8ce
......@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"log/slog"
"time"
_ "code.wuban.net.cn/odysseus/twitter_syncer/docs"
// docs are generated by Swag CLI, you have to import them.
......@@ -33,6 +34,20 @@ func init() {
break
}
}
accounts, err := GetLoginAccount()
if err != nil {
slog.Error(err.Error())
}
for k, v := range accounts {
//v.Timer = time.NewTimer(time.Duration(k) * time.Duration(5) * time.Minute)
accChan <- ScraperWithTimer{
Scraper: v,
Timer: time.NewTimer(time.Duration(k) * time.Duration(5) * time.Minute),
}
}
}
// func TwitterAccountFromDB() ([]TwitterAccount, error) {
......@@ -215,6 +230,7 @@ type TaskInDB struct {
TaskId string `json:"task_id"`
Start bool `json:"start"`
Stop bool `json:"stop"`
FollowerCount int `json:"follower_count"`
}
func QueryAllTask() ([]TaskInDB, error) {
......@@ -247,6 +263,7 @@ type TaskJob struct {
UserId string
TaskType string
Config
FollowerCount int
}
func (job *TaskJob) String() string {
......@@ -296,6 +313,7 @@ func GetTasks() ([]TaskJob, error) {
UserId: task.User,
TaskType: task.TaskType,
TaskId: task.TaskId,
FollowerCount: task.FollowerCount,
}
// taskJob := NewTaskJob(task.TaskId, task.User, task.TaskType)
......
docs/swagger.yaml
View file @ 8d89b8ce
......@@ -146,24 +146,18 @@ paths:
- Verify
summary: Verify the provide user
parameters:
- name: user_id
- name: task_id
in: query
required: true
explode: true
type: string
example: "1570057485914087429"
example: "aon_aonet"
- name: follower_id
in: query
required: true
explode: true
type: string
example: "1577163456993558529"
- name: follower_username
in: query
required: false
explode: false
type: string
example: available
example: "1825777453769650176"
responses:
"200":
description: successful operation
......@@ -351,11 +345,11 @@ components:
- retweeters
- tweet_liking_users
- followers
example: retweeters
example: followers
task_id:
description: tweet id;
type: string
example: "1800805503066661056"
example: "aon_aonet"
StopTaskReq:
required:
- user_id
......
idx.go
View file @ 8d89b8ce
......@@ -117,7 +117,7 @@ func (p *PageUsers) Request(tweetId string, next string, f req) ([]UserTask, err
return nil, err
}
if rt.Remaining == 0 {
if rt != nil && rt.Remaining == 0 {
time.Sleep(time.Until(rt.Reset.Time().Add(500 * time.Millisecond)))
}
......
syncer
View file @ 8d89b8ce
No preview for this file type
syncer.test.ok 0 → 100755
View file @ 8d89b8ce
File added
task.go
View file @ 8d89b8ce
......@@ -4,6 +4,7 @@ import (
"fmt"
"log/slog"
"sync"
"time"
)
type Work struct {
......@@ -56,20 +57,132 @@ func (w *Work) AddJob(t TaskJob) error {
return nil
}
type TimeAndFollowCount struct {
Date time.Time
FollowCount int
}
func (w *Work) RunJob(t TaskJob) chan<- interface{} {
slog.Info("exect job", "userid", t.UserId, "task type", t.TaskType, "task id", t.TaskId, "t.FollowerCount", t.FollowerCount)
done := make(chan interface{})
go func() {
var cli *Client
if t.TaskType == RetweetType {
cli = NewRetweeterClient(t.Config)
} else {
} else if t.TaskType == TweetLikingUsersType {
cli = NewLikeClient(t.Config)
} else if t.TaskType == FollowType {
cli = NewFollowClient()
}
page := NewPageUsers(NewIdx(t.Idx))
if t.TaskType == FollowType {
secondTicker := time.NewTicker(time.Second * 3)
fiveMinutesTicker := time.NewTicker(time.Minute * 5)
halfHourTicker := time.NewTicker(time.Minute * 30)
//recordFc := make(map[string]int)
recordFc := make([]TimeAndFollowCount, 0, 100)
for {
select {
case _, ok := <-done:
if !ok {
return
}
case <-fiveMinutesTicker.C:
slog.Info("case <-fiveMinutesTicker.C:")
maybeFound := false
for k, v := range recordFc {
fmt.Println(k, v)
if v.FollowCount != t.FollowerCount {
maybeFound = true
}
if k == len(recordFc)-1 {
t.FollowerCount = v.FollowCount
}
}
fmt.Println(" t.FollowerCount", t.FollowerCount, "maybeFound", maybeFound)
if maybeFound {
halfHourTicker.Reset(time.Minute * 30)
if err := Request(Follower, page, t); err != nil {
slog.Error(" page.Request", "task id", t.TaskId, "t.TaskType", t.TaskType, "err", err.Error())
continue
}
if err := UpdateFollowerTaskCount(t.TaskId, t.TaskType, t.UserId, t.FollowerCount); err != nil {
slog.Error("UpdateFollowerTaskCount", "err", err.Error(), "t.TaskId,", t.TaskId, "t.TaskType", t.TaskType, "t.UserId", t.UserId)
}
}
recordFc = make([]TimeAndFollowCount, 0, 100)
//recordFc = make(map[string]int)
case <-secondTicker.C:
fc, err := cli.TryProfileFollowerCount(t.TaskId)
if err != nil {
slog.Error("TryProfileFollowerCount", "err", err.Error())
continue
}
recordFc = append(recordFc, TimeAndFollowCount{
Date: time.Now(),
FollowCount: fc,
})
// 先用和like retweet一样的周期模式；
case <-halfHourTicker.C:
slog.Info("case <-halfHourTicker.C:")
//recordFc = make(map[string]int)
//maybeFound := false
for k, v := range recordFc {
fmt.Println(k, v)
if k == len(recordFc)-1 {
t.FollowerCount = v.FollowCount
}
}
fmt.Println(" t.FollowerCount", t.FollowerCount)
// if maybeFound {
//halfHourTicker = (*time.Ticker)(time.NewTimer()) //Reset(time.Minute * 30)
if err := Request(Follower, page, t); err != nil {
slog.Error(" page.Request", "task id", t.TaskId, "t.TaskType", t.TaskType, "err", err.Error())
continue
}
if err := UpdateFollowerTaskCount(t.TaskId, t.TaskType, t.UserId, t.FollowerCount); err != nil {
slog.Error("UpdateFollowerTaskCount", "err", err.Error(), "t.TaskId,", t.TaskId, "t.TaskType", t.TaskType, "t.UserId", t.UserId)
}
//}
recordFc = make([]TimeAndFollowCount, 0, 100)
// if err := Request(Follower, page, t); err != nil {
// slog.Error(" page.Request", "task id", t.TaskId, "t.TaskType", t.TaskType, "err", err.Error())
// continue
// }
}
}
}
for {
select {
......@@ -87,11 +200,26 @@ func (w *Work) RunJob(t TaskJob) chan<- interface{} {
f = cli.TweetLikingUsers
}
if err := Request(f, page, t); err != nil {
slog.Error(" page.Request", "task id", t.TaskId, "t.TaskType", t.TaskType, "err", err.Error())
continue
}
}
}
}()
return done
}
func Request(f req, page *PageUsers, t TaskJob) error {
users, err := page.Request(t.TaskId, "", f)
if err != nil {
slog.Error(" page.Request", "task id", t.TaskId, "t.TaskType", t.TaskType, "err", err.Error())
continue
return err
}
if err := InsertTaskRes(users, t.TaskType, t.TaskId); err != nil {
......@@ -104,11 +232,25 @@ func (w *Work) RunJob(t TaskJob) chan<- interface{} {
}
slog.Info("InsertTaskRes", "task id", t.TaskId, "t.TaskType", t.TaskType, "len(users)", len(users))
}
}
}()
return done
return nil
}
// users, err := page.Request(t.TaskId, "", f)
// if err != nil {
// slog.Error(" page.Request", "task id", t.TaskId, "t.TaskType", t.TaskType, "err", err.Error())
// continue
// }
// if err := InsertTaskRes(users, t.TaskType, t.TaskId); err != nil {
// for k, v := range users {
// fmt.Println(k, v.UserId, v.UserName)
// }
// slog.Error("InsertTaskRes", "task id", t.TaskId, "t.TaskType", t.TaskType, "len(users)", len(users), "err", err.Error())
// }
// slog.Info("InsertTaskRes", "task id", t.TaskId, "t.TaskType", t.TaskType, "len(users)", len(users))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment