Skip to content

T
twitter_syncer
Commit a6620d2c authored by Ubuntu's avatar Ubuntu
Browse files
Options

fix api doc

parent 45719791
Show whitespace changes
Inline Side-by-side
Showing with 1053 additions and 264 deletions
api.go 0 → 100644
View file @ a6620d2c
package main
import (
"encoding/json"
"fmt"
"log/slog"
// "github.com/gofiber/fiber/v2"
// "github.com/gofiber/fiber/v2/middleware/cors"
// "github.com/gofiber/swagger"
"github.com/gofiber/fiber/v2"
//"github.com/gofiber/swagger"
_ "code.wuban.net.cn/odysseus/twitter_syncer/docs"
// docs are generated by Swag CLI, you have to import them.
// replace with your own docs folder, usually "github.com/username/reponame/docs"
//_ "github.com/gofiber/swagger/example/docs"
)
type Res struct {
Code int64 `json:"code"`
Msg string `json:"msg"`
}
type VerifyRes struct {
Code int64 `json:"code"`
Msg string `json:"msg"`
Data struct {
Ok bool `json:"ok"`
} `json:"data"`
}
type Req struct {
// AddOrStop bool
Project string `json:"project"`
TweetId string `json:"tweet_id"`
// Like bool `json:"like"`
Follow bool `json:"follow"`
Retweet bool `json:"retweet"`
}
func TaskAdd(c *fiber.Ctx) error {
//fmt.Println(string(c.Request().Body()))
slog.Info(c.Route().Path, "body", string(c.Request().Body()))
req := Req{}
if err := json.Unmarshal(c.Request().Body(), &req); err != nil {
slog.Error("json.Unmarshal(c.Request().Body(), &req)", "err", err.Error())
return c.JSON(Res{
Code: 500,
Msg: err.Error(),
})
}
slog.Info(c.Route().Path, "Project", req.Project, "TweetId", req.TweetId, "Retweet", req.Retweet, "Follow", req.Follow)
if req.Project == "" || req.TweetId == "" {
return c.JSON(Res{
Code: 500,
Msg: "must provide project and tweet_id",
})
}
if !req.Retweet && !req.Follow {
return c.JSON(Res{
Code: 500,
Msg: "like or retweet must be true",
})
}
// req.AddOrStop = true
err := AddTaskInsertOrUpdate(req)
//res, _, err := client.From("twitter_task").Insert(req, true, "", "representation", "").Execute()
if err != nil {
slog.Error("twitter_syncer insert", "err", err.Error())
return c.JSON(Res{
Code: 500,
Msg: err.Error(),
})
}
//slog.Info("twitter_syncer insert", "res", string(res))
return c.JSON(Res{
Code: 200,
})
}
func TaskStop(c *fiber.Ctx) error {
slog.Info(c.Route().Path, "body", string(c.Request().Body()))
// fmt.Println(string(c.Request().Body()))
req := Req{}
if err := json.Unmarshal(c.Request().Body(), &req); err != nil {
return c.JSON(Res{
Code: 500,
Msg: err.Error(),
})
}
slog.Info(c.Route().Path, "Project", req.Project, "TweetId", req.TweetId, "Retweet", req.Retweet, "Follow", req.Follow)
if req.Project == "" || req.TweetId == "" {
return c.JSON(Res{
Code: 500,
Msg: "must provide project and tweet_id",
})
}
if !req.Retweet && !req.Follow {
return c.JSON(Res{
Code: 200,
})
}
err := StopTaskInsertOrUpdate(req)
if err != nil {
slog.Error("twitter_syncer stop", "err", err.Error())
return c.JSON(Res{
Code: 500,
Msg: err.Error(),
})
}
return c.JSON(Res{
Code: 200,
})
}
func VerifyFollower(c *fiber.Ctx) error {
userId := c.Query("user_id")
followerId := c.Query("follower_id")
if len(userId) == 0 || len(followerId) == 0 {
slog.Error("VerifyFollower", "userId", userId, "followerId", followerId)
return c.JSON(Res{
Code: 500,
Msg: fmt.Sprintf("must provide userId [%v] and followerId [%v]", userId, followerId),
})
}
followerUserName := c.Query("follower_username")
slog.Info(c.Route().Path, "user_id", userId, "followerId", followerId, "followerUserName", followerUserName)
ok, err := VerifyFollowerInDb(userId, followerId)
if err != nil {
slog.Error("VerifyFollowerInDb", "userId", userId, "followerId", followerId, "err", err.Error())
return c.JSON(Res{
Code: 500,
Msg: fmt.Sprintf("VerifyFollowerInDb userId %v followerId %v err %v", userId, followerId, err.Error()),
})
}
return c.JSON(VerifyRes{
Code: 200,
Data: struct {
Ok bool "json:\"ok\""
}{
Ok: ok,
},
})
}
func VerifyRetweeter(c *fiber.Ctx) error {
tweetId := c.Query("tweet_id")
retweeterId := c.Query("retweeter_id")
if len(tweetId) == 0 || len(retweeterId) == 0 {
slog.Error("VerifyFollower", "tweetId", tweetId, "retweeterId", retweeterId)
return c.JSON(Res{
Code: 500,
Msg: fmt.Sprintf("must provide tweetId [%v] and retweeterId [%v]", tweetId, retweeterId),
})
}
slog.Info(c.Route().Path, "tweetId", tweetId, "retweeterId", retweeterId)
ok, err := VerifyRetweeterInDb(tweetId, retweeterId)
if err != nil {
slog.Error("VerifyRetweeter", "tweetId", tweetId, "retweeterId", retweeterId, "err", err.Error())
return c.JSON(Res{
Code: 500,
//Msg: fmt.Sprint("VerifyFollowerInDb", "tweetId", tweetId, "retweeterId", retweeterId, "err", err.Error()),
Msg: fmt.Sprintf("VerifyRetweeter tweetId %v retweeterId %v err %v", tweetId, retweeterId, err.Error()),
})
}
return c.JSON(VerifyRes{
Code: 200,
Data: struct {
Ok bool "json:\"ok\""
}{
Ok: ok,
},
})
}
api_db.go 0 → 100644
View file @ a6620d2c
package main
import (
"encoding/json"
"fmt"
_ "code.wuban.net.cn/odysseus/twitter_syncer/docs"
// docs are generated by Swag CLI, you have to import them.
// replace with your own docs folder, usually "github.com/username/reponame/docs"
//_ "github.com/gofiber/swagger/example/docs"
)
// var client *supabase.Client
// func init() {
// var API_URL = "http://43.198.54.207:8000"
// var API_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q"
// for {
// cli, err := supabase.NewClient(API_URL, API_KEY, &supabase.ClientOptions{
// Schema: "twitter",
// })
// if err != nil {
// slog.Error("supabase.NewClient", "err", err.Error())
// return
// } else {
// client = cli
// break
// }
// }
// }
func AddTaskInsertOrUpdate(req Req) error {
task, ok, err := Query(req)
if err != nil {
return err
}
_ = task
if ok {
//update
return AddTaskUpdate(req, task)
} else {
//insert
return Insert(req)
}
return nil
}
func AddTaskUpdate(req Req, task Task) error {
if !task.Follow && req.Follow {
task.Follow = true
}
if !task.Retweet && req.Retweet {
task.Retweet = true
}
res, _, err := client.From("tasks").Update(&struct {
Retweet bool `json:"retweet"`
Follow bool `json:"follow"`
}{
Retweet: task.Retweet,
Follow: task.Follow,
}, "", "exact").Eq("project", req.Project).Eq("tweet_id", req.TweetId).Execute()
_ = res
return err
}
func StopTaskInsertOrUpdate(req Req) error {
task, ok, err := Query(req)
if err != nil {
return err
}
if !ok {
return fmt.Errorf("can not stop the task,no record")
}
return StopTaskUpdate(req, task)
}
func StopTaskUpdate(req Req, task Task) error {
if !task.FollowStop && req.Follow {
task.FollowStop = true
}
if !task.RetweetStop && req.Retweet {
task.RetweetStop = true
}
res, _, err := client.From("tasks").Update(&struct {
Retweet bool `json:"retweet_stop"`
Follow bool `json:"follow_stop"`
}{
Retweet: task.RetweetStop,
Follow: task.FollowStop,
}, "", "exact").Eq("project", req.Project).Eq("tweet_id", req.TweetId).Execute()
_ = res
return err
//return nil
}
func Insert(req Req) error {
res, _, err := client.From("tasks").Insert(req, true, "", "representation", "").Execute()
_ = res
return err
}
func Query(req Req) (Task, bool, error) {
data, count, err := client.From("tasks").Select("*", "exact", false).Eq("project", req.Project).Eq("tweet_id", req.TweetId).Execute()
if err != nil {
return Task{}, false, err
}
if count == 0 {
return Task{}, false, nil
}
tasks := make([]Task, 0)
if err := json.Unmarshal(data, &tasks); err != nil {
return Task{}, false, err
}
var task Task
if len(tasks) == 1 {
task = tasks[0]
} else {
return Task{}, false, fmt.Errorf("expected raws len ==1, but actually %d", len(tasks))
}
return task, true, nil
}
/*
[{"id":3,"created_at":"2024-07-30T09:18:22.110223+00:00","project":"1","tweet_id":"1","like":true,"retweet":true,"like_stop":null,"retweet_stop":null}]
*/
// type Tasks []Task
type Task struct {
Project string `json:"project"`
TweetId string `json:"tweet_id"`
Like bool `json:"like"`
Follow bool `json:"follow"`
Retweet bool `json:"retweet"`
LikeStop bool `json:"like_stop"`
FollowStop bool `json:"follow_stop"`
RetweetStop bool `json:"retweet_stop"`
}
func VerifyFollowerInDb(userId, followerId string) (bool, error) {
_, count, err := client.From("followers").Select("*", "exact", false).Eq("user_id", userId).Eq("follower_id", followerId).Execute()
if err != nil {
return false, err
}
return count == 1, nil
}
func VerifyRetweeterInDb(tweetId, retweeter string) (bool, error) {
_, count, err := client.From("retweeters").Select("*", "exact", false).Eq("tweet_id", tweetId).Eq("retweeter_id", retweeter).Execute()
if err != nil {
return false, err
}
return count == 1, nil
}
db.go
View file @ a6620d2c
package main package main
import ( import (
"encoding/json"
"fmt"
"log/slog" "log/slog"
_ "code.wuban.net.cn/odysseus/twitter_syncer/docs" _ "code.wuban.net.cn/odysseus/twitter_syncer/docs"
...@@ -21,7 +19,9 @@ func init() { ...@@ -21,7 +19,9 @@ func init() {
var API_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q" var API_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q"
for { for {
cli, err := supabase.NewClient(API_URL, API_KEY, nil) cli, err := supabase.NewClient(API_URL, API_KEY, &supabase.ClientOptions{
Schema: "twitter",
})
if err != nil { if err != nil {
slog.Error("supabase.NewClient", "err", err.Error()) slog.Error("supabase.NewClient", "err", err.Error())
return return
...@@ -31,145 +31,3 @@ func init() { ...@@ -31,145 +31,3 @@ func init() {
} }
} }
} }
func AddTaskInsertOrUpdate(req Req) error {
task, ok, err := Query(req)
if err != nil {
return err
}
_ = task
if ok {
//update
return AddTaskUpdate(req, task)
} else {
//insert
return Insert(req)
}
return nil
}
func AddTaskUpdate(req Req, task Task) error {
if !task.Follow && req.Follow {
task.Follow = true
}
if !task.Retweet && req.Retweet {
task.Retweet = true
}
res, _, err := client.From("twitter_task").Update(&struct {
Retweet bool `json:"retweet"`
Follow bool `json:"follow"`
}{
Retweet: task.Retweet,
Follow: task.Follow,
}, "", "exact").Eq("project", req.Project).Eq("tweet_id", req.TweetId).Execute()
_ = res
return err
}
func StopTaskInsertOrUpdate(req Req) error {
task, ok, err := Query(req)
if err != nil {
return err
}
if !ok {
return fmt.Errorf("can not stop the task,no record")
}
return StopTaskUpdate(req, task)
}
func StopTaskUpdate(req Req, task Task) error {
if !task.FollowStop && req.Follow {
task.FollowStop = true
}
if !task.RetweetStop && req.Retweet {
task.RetweetStop = true
}
res, _, err := client.From("twitter_task").Update(&struct {
Retweet bool `json:"retweet_stop"`
Follow bool `json:"follow_stop"`
}{
Retweet: task.RetweetStop,
Follow: task.FollowStop,
}, "", "exact").Eq("project", req.Project).Eq("tweet_id", req.TweetId).Execute()
_ = res
return err
//return nil
}
func Insert(req Req) error {
res, _, err := client.From("twitter_task").Insert(req, true, "", "representation", "").Execute()
_ = res
return err
}
func Query(req Req) (Task, bool, error) {
data, count, err := client.From("twitter_task").Select("*", "exact", false).Eq("project", req.Project).Eq("tweet_id", req.TweetId).Execute()
if err != nil {
return Task{}, false, err
}
if count == 0 {
return Task{}, false, nil
}
tasks := make([]Task, 0)
if err := json.Unmarshal(data, &tasks); err != nil {
return Task{}, false, err
}
var task Task
if len(tasks) == 1 {
task = tasks[0]
} else {
return Task{}, false, fmt.Errorf("expected raws len ==1, but actually %d", len(tasks))
}
return task, true, nil
}
/*
[{"id":3,"created_at":"2024-07-30T09:18:22.110223+00:00","project":"1","tweet_id":"1","like":true,"retweet":true,"like_stop":null,"retweet_stop":null}]
*/
// type Tasks []Task
type Task struct {
Project string `json:"project"`
TweetId string `json:"tweet_id"`
Like bool `json:"like"`
Follow bool `json:"follow"`
Retweet bool `json:"retweet"`
LikeStop bool `json:"like_stop"`
FollowStop bool `json:"follow_stop"`
RetweetStop bool `json:"retweet_stop"`
}
docs/swagger.yaml
View file @ a6620d2c
openapi: 3.0.0 openapi: "3.0.3"
servers: servers:
- url: http://43.198.54.207:8001 - url: http://43.198.54.207:8001
description: Default server description: Default server
...@@ -28,6 +28,8 @@ externalDocs: ...@@ -28,6 +28,8 @@ externalDocs:
tags: tags:
- name: Task - name: Task
description: add or stop task description: add or stop task
- name: Verify
description: Verify the user's job
# - name: User # - name: User
# description: Everything about user # description: Everything about user
...@@ -50,7 +52,7 @@ x-tagGroups: ...@@ -50,7 +52,7 @@ x-tagGroups:
- name: Task Controler - name: Task Controler
tags: tags:
- task - task
- name: User Management - name: daily task
tags: tags:
- user - user
- name: Models - name: Models
...@@ -112,6 +114,97 @@ paths: ...@@ -112,6 +114,97 @@ paths:
enum: [200, 500] enum: [200, 500]
msg: msg:
type: string type: string
/verify/follower:
get:
tags:
- Verify
summary: Verify the provide user
parameters:
- name: user_id
in: query
required: true
explode: true
type: string
example: available
- name: follower_id
in: query
required: true
explode: true
type: string
example: available
- name: follower_username
in: query
required: false
explode: false
type: string
example: available
responses:
"200":
description: successful operation
content:
application/json:
schema:
type: object
properties:
code:
type: integer
format: int64
enum: [200, 500]
msg:
type: string
data:
$ref: '#/components/schemas/verify_res'
/verify/retweeter:
get:
tags:
- Verify
summary: Verify the provide user
parameters:
- name: tweet_id
in: query
required: true
explode: true
type: string
# default: available
example: available
- name: retweeter_id
in: query
required: true
explode: true
type: string
# default: available
example: available
- name: begin_time
in: query
type: string
format: date-time
example: "2024-01-01T00:00:00Z"
# default: "2024-10-01T00:00:00Z"
- name: end_time
in: query
type: string
format: date-time
example: "2024-12-01T00:00:00Z"
# default: "2024-10-01T00:00:00Z"
responses:
"200":
description: successful operation
content:
application/json:
schema:
type: object
properties:
code:
type: integer
format: int64
enum: [200, 500]
msg:
type: string
data:
$ref: '#/components/schemas/verify_res'
components: components:
securitySchemes: securitySchemes:
api_key: api_key:
...@@ -148,3 +241,10 @@ components: ...@@ -148,3 +241,10 @@ components:
type: boolean type: boolean
retweet: retweet:
type: boolean type: boolean
verify_res:
required:
- ok
type: object
properties:
ok:
type: boolean
go.mod
View file @ a6620d2c
...@@ -47,3 +47,6 @@ require ( ...@@ -47,3 +47,6 @@ require (
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect
) )
replace github.com/imperatrona/twitter-scraper => ../twitter-scraper
idx.go 0 → 100644
View file @ a6620d2c
package main
import (
"container/list"
"encoding/json"
"fmt"
"log/slog"
"time"
twitterscraper "github.com/imperatrona/twitter-scraper"
"github.com/supabase-community/postgrest-go"
)
func GetTasks() ([]Task, error) {
data, count, err := client.From("tasks").Select("*", "exact", false).
Eq("retweet", "true").Neq("retweet_stop", "true").
Eq("follow", "true").Neq("follow_stop", "true").
Execute()
if err != nil {
return nil, err
}
res := make([]Task, 0, count)
if err := json.Unmarshal(data, &res); err != nil {
return nil, err
}
return res, nil
}
func GetTasksFollowIdx() ([][]FollowerId, error) {
tasks, err := GetFollowTasks()
if err != nil {
return nil, err
}
res := make([][]FollowerId, 0, 10)
for _, task := range tasks {
data, count, err := client.From("followers").Select("", "user_id", false).
Eq("user_id", task.Project).
Order("id", &postgrest.OrderOpts{
Ascending: false,
// NullsFirst bool
// ForeignTable string
}).Range(0, 10, "").Execute()
if err != nil {
slog.Error("select * from followers error", err)
return nil, err
}
_ = count
fmt.Println("data", string(data))
userRes := make([]FollowerId, 0, 10)
if err := json.Unmarshal(data, &userRes); err != nil {
return nil, err
}
res = append(res, userRes)
}
return res, nil
}
func GetFollowTasks() ([]Task, error) {
data, count, err := client.From("tasks").Select("*", "exact", false).
Eq("follow", "true").Neq("follow_stop", "true").
Execute()
if err != nil {
return nil, err
}
res := make([]Task, 0, count)
if err := json.Unmarshal(data, &res); err != nil {
return nil, err
}
return res, nil
}
func FollowersToBackList(done <-chan interface{}, idxss [][]FollowerId) (<-chan *list.List, error) {
scraper := twitterscraper.New()
//err := scraper.Login("Wade_Leeeee", "923881393time")
err := scraper.Login("wuban358369", "123456789T")
if err != nil {
fmt.Println("scraper", err.Error())
return nil, err
}
/*
wuban01@gmail.com
wuban01
1234567890Wuban
1234567890Wuban
// 2346
980b
641a
774d
39fb
9425
c6c5
3140
b469
0b93
d1c6
6101
ef6f
9fb6
ae94
ee6f
//wuban001
wuban01@tutamail.com
1234567890Wuban
*/
scraper3 := twitterscraper.New()
//err = scraper3.Login("tifawe2861@stikezz.com", "123456789T")
//err = scraper3.Login("wuban358369", "123456789T")
err = scraper3.Login("Wade_Leeeee", "923881393time")
if err != nil {
fmt.Println("scraper3", err.Error())
return nil, err
}
fmt.Println("scraper.IsLoggedIn()", scraper.IsLoggedIn())
fmt.Println("scraper3.IsLoggedIn()", scraper3.IsLoggedIn())
userId := "Bitcoin"
next := ""
var backPushPop = list.New()
outStream := make(chan *list.List, 1)
go func() {
//newBegin := true
for _, idxs := range idxss {
newIdxs := make([]FollowerId, 0, len(idxs))
c := true
for {
c = !c
var err error
var users Users
if c {
users, err = FetchFollowers(scraper, userId, next)
if err != nil {
slog.Error("FetchFollowers", err)
time.Sleep(time.Second)
continue
}
} else {
users, err = FetchFollowers(scraper3, userId, next)
if err != nil {
slog.Error("FetchFollowers", err)
time.Sleep(time.Second)
continue
}
}
// users, err := FetchFollowers(scraper, userId, next)
// if err != nil {
// slog.Error("FetchFollowers", err)
// time.Sleep(time.Second)
// continue
// }
fmt.Println("len(users)-----------------", len(users.Profiles))
for k, v := range users.Profiles {
fmt.Println("k", k, "v", v.UserIdAsNumber, v.Username)
}
if len(newIdxs) == 0 {
for k, v := range users.Profiles {
newIdxs = append(newIdxs, FollowerId{
Follower: Follower{
Follower: v.UserIdAsNumber,
UserName: v.Username,
},
})
if k > 5 {
break
}
}
}
profiles, ok := MatchIdx(users, idxs)
fmt.Println("len(profiles)-----------------", len(profiles))
for _, v := range profiles {
backPushPop.PushFront(v)
}
fmt.Printf("!ok %v users.Next %s %d len(outStream) %d backPushPop len %d\n", !ok, users.Next, len(users.Next), len(outStream), backPushPop.Len())
time.Sleep(time.Second * 2)
if !ok && len(users.Next) != 0 {
next = users.Next
continue
}
/*
Api has a global limit on how many requests per second are allowed, don’t make requests more than once per 1.5 seconds from one account.
Also each endpoint has its own limits, most of them are 150 requests per 15 minutes.
*/
if backPushPop.Len() == 0 {
next = ""
//time.Sleep(time.Second * 20)
continue
}
idxs = newIdxs
newIdxs = make([]FollowerId, 0, len(idxs))
select {
case <-done:
return
case outStream <- backPushPop:
fmt.Println("case outStream <- backPushPop: ------------")
backPushPop = list.New()
//time.Sleep(time.Second * 20)
}
next = ""
//time.Sleep(time.Second * 20)
}
}
}()
return outStream, nil
}
func MatchIdx(data Users, idxs []FollowerId) ([]Profile, bool) {
if len(idxs) == 0 {
return data.Profiles, false
}
for k, v := range data.Profiles {
for _, idx := range idxs {
if v.UserIdAsNumber == idx.Follower.Follower {
return data.Profiles[:k], true
}
}
}
return data.Profiles, false
}
idx_test.go 0 → 100644
View file @ a6620d2c
package main
import (
"testing"
)
func TestTwitterTasks(t *testing.T) {
tasks, err := GetTasksIdx()
if err != nil {
t.Fatal(err)
}
for k, v := range tasks {
t.Log(k, "v.Project", v.Project, "v.TweetId", v.TweetId, "v.Follow", v.Follow)
}
}
main.go
View file @ a6620d2c
package main package main
import ( import (
"encoding/json"
"log/slog" "log/slog"
// "github.com/gofiber/fiber/v2" // "github.com/gofiber/fiber/v2"
...@@ -41,125 +40,16 @@ func main() { ...@@ -41,125 +40,16 @@ func main() {
//OAuth2RedirectUrl: "http://localhost:8080/swagger/oauth2-redirect.html", //OAuth2RedirectUrl: "http://localhost:8080/swagger/oauth2-redirect.html",
})) }))
app.Post("/task/add", func(c *fiber.Ctx) error { app.Post("/task/add", TaskAdd)
app.Post("/task/stop", TaskStop)
//fmt.Println(string(c.Request().Body())) app.Get("/verify/follower", VerifyFollower)
app.Get("/verify/retweeter", VerifyRetweeter)
slog.Info(c.Route().Path, "body", string(c.Request().Body()))
req := Req{}
if err := json.Unmarshal(c.Request().Body(), &req); err != nil {
slog.Error("json.Unmarshal(c.Request().Body(), &req)", "err", err.Error())
return c.JSON(Res{
Code: 500,
Msg: err.Error(),
})
}
slog.Info(c.Route().Path, "Project", req.Project, "TweetId", req.TweetId, "Retweet", req.Retweet, "Follow", req.Follow)
if req.Project == "" || req.TweetId == "" {
return c.JSON(Res{
Code: 500,
Msg: "must provide project and tweet_id",
})
}
if !req.Retweet && !req.Follow {
return c.JSON(Res{
Code: 500,
Msg: "like or retweet must be true",
})
}
// req.AddOrStop = true
err := AddTaskInsertOrUpdate(req)
//res, _, err := client.From("twitter_task").Insert(req, true, "", "representation", "").Execute()
if err != nil {
slog.Error("twitter_syncer insert", "err", err.Error())
return c.JSON(Res{
Code: 500,
Msg: err.Error(),
})
}
//slog.Info("twitter_syncer insert", "res", string(res))
return c.JSON(Res{
Code: 200,
})
})
app.Post("/task/stop", func(c *fiber.Ctx) error {
slog.Info(c.Route().Path, "body", string(c.Request().Body()))
// fmt.Println(string(c.Request().Body()))
req := Req{}
if err := json.Unmarshal(c.Request().Body(), &req); err != nil {
return c.JSON(Res{
Code: 500,
Msg: err.Error(),
})
}
slog.Info(c.Route().Path, "Project", req.Project, "TweetId", req.TweetId, "Retweet", req.Retweet, "Follow", req.Follow)
if req.Project == "" || req.TweetId == "" {
return c.JSON(Res{
Code: 500,
Msg: "must provide project and tweet_id",
})
}
if !req.Retweet && !req.Follow {
return c.JSON(Res{
Code: 200,
})
}
err := StopTaskInsertOrUpdate(req)
if err != nil {
slog.Error("twitter_syncer stop", "err", err.Error())
return c.JSON(Res{
Code: 500,
Msg: err.Error(),
})
}
return c.JSON(Res{
Code: 200,
})
})
if err := app.Listen(":8001"); err != nil { if err := app.Listen(":8001"); err != nil {
slog.Error(err.Error()) slog.Error(err.Error())
} }
} }
type Res struct {
Code int64 `json:"code"`
Msg string `json:"msg"`
}
type Req struct {
// AddOrStop bool
Project string `json:"project"`
TweetId string `json:"tweet_id"`
// Like bool `json:"like"`
Follow bool `json:"follow"`
Retweet bool `json:"retweet"`
}
// type ReqDB struct { // type ReqDB struct {
// AddOrStop bool // AddOrStop bool
// Project string `json:"project"` // Project string `json:"project"`
......
scraper.go.bak 0 → 100644
View file @ a6620d2c
package main
// import (
// // "strings"
// // "time"
// // b64 "encoding/base64"
// twitterscraper "github.com/imperatrona/twitter-scraper"
// )
// type NewScraper struct{
// twitterscraper.Scraper
// }
// func (s *NewScraper) RetweetsUsers(tweetId string) error {
// //https://x.com/i/api/graphql/_D6CD5GX3VTL4ZzoAjJjGA/Retweeters
// //req, err := s.newRequest("Get", "https://twitter.com/i/api/graphql/95Bp6m3-FTFVeZ_rPD3mZQ/Favoriters")
// req, err := s.newRequest("Get", "https://twitter.com/i/api/graphql/_D6CD5GX3VTL4ZzoAjJjGA/Retweeters")
// if err != nil {
// return err
// }
// req.Header.Set("content-type", "application/json")
// /*
// {"tweetId":"1814231701658444272","count":20,"includePromotedContent":true}
// */
// variables := map[string]interface{}{
// "tweetId": tweetId,
// "count": 20,
// "includePromotedContent": true,
// }
// features := map[string]interface{}{
// "rweb_tipjar_consumption_enabled": true,
// "responsive_web_graphql_exclude_directive_enabled": true,
// "verified_phone_label_enabled": false,
// "creator_subscriptions_tweet_preview_api_enabled": true,
// "responsive_web_graphql_timeline_navigation_enabled": true,
// "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
// "communities_web_enable_tweet_community_results_fetch": true,
// "c9s_tweet_anatomy_moderator_badge_enabled": true,
// "articles_preview_enabled": true,
// "tweetypie_unmention_optimization_enabled": true,
// "responsive_web_edit_tweet_api_enabled": true,
// "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
// "view_counts_everywhere_api_enabled": true,
// "longform_notetweets_consumption_enabled": true,
// "responsive_web_twitter_article_tweet_consumption_enabled": true,
// "tweet_awards_web_tipping_enabled": false,
// "creator_subscriptions_quote_tweet_preview_enabled": false,
// "freedom_of_speech_not_reach_fetch_enabled": true,
// "standardized_nudges_misinfo": true,
// "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
// "rweb_video_timestamps_enabled": true,
// "longform_notetweets_rich_text_read_enabled": true,
// "longform_notetweets_inline_media_enabled": true,
// "responsive_web_enhance_cards_enabled": false,
// }
// // query := url.Values{}
// // query.Set("variables", mapToJSONString(variables))
// // query.Set("features", mapToJSONString(features))
// // req.URL.RawQuery = query.Encode()
// query := url.Values{}
// query.Set("variables", mapToJSONString(variables))
// query.Set("features", mapToJSONString(features))
// req.URL.RawQuery = query.Encode()
// // body := map[string]interface{}{
// // "variables": variables,
// // "queryId": "ZYKSe-w7KEslx3JhSIk5LA",
// // }
// // b, _ := json.Marshal(body)
// // req.Body = io.NopCloser(bytes.NewReader(b))
// // var response struct {
// // Data struct {
// // UnfavoriteTweet string `json:"unfavorite_tweet"`
// // } `json:"data"`
// // Errors []struct {
// // Message string `json:"message"`
// // Code int `json:"code"`
// // } `json:"errors"`
// // }
// err = s.RequestAPI(req, nil)
// if err != nil {
// return err
// }
// // if len(response.Errors) > 0 && response.Errors[0].Code == 144 {
// // return errors.New("tweet already not liked")
// // }
// // if response.Data.UnfavoriteTweet != "Done" {
// // return errors.New("unknown error")
// // }
// return nil
// }
task.go
View file @ a6620d2c
...@@ -9,14 +9,110 @@ import ( ...@@ -9,14 +9,110 @@ import (
twitterscraper "github.com/imperatrona/twitter-scraper" twitterscraper "github.com/imperatrona/twitter-scraper"
) )
func AddTask() { // task
// func sync() error {
// idxs, err := GetIndexList()
// if err != nil {
// return err
// }
// done := make(chan interface{})
// defer close(done)
// backListStream, err := FollowersToBackList(done, idxs)
// if err != nil {
// return err
// }
// usersStream := BackListToQueue(done, backListStream)
// if err := InsertOrUpdateUsers(done, usersStream); err != nil {
// return err
// }
// select {}
// }
func sync() error {
idxs, err := GetTasksFollowIdx()
//idxs, err := GetIndexList()
if err != nil {
return err
}
done := make(chan interface{})
defer close(done)
backListStream, err := FollowersToBackList(done, idxs)
if err != nil {
return err
}
_ = backListStream
return nil
} }
func StopTask() { func tasks(done <-chan interface{}, inStream <-chan TaskParam) (<-chan TaskParam, <-chan TaskParam) {
outResource := make(chan TaskParam)
outDb := make(chan TaskParam)
go func() {
}()
return outResource, outDb
} }
type TwitterUserPasswd struct {
User string
PassWd string
}
var twitterUsers []TwitterUserPasswd
func Resource() ([]*twitterscraper.Scraper, error) {
users := make([]struct {
User string
PassWd string
}, 0)
users = append(users, struct {
User string
PassWd string
}{
User: "Wade_Leeeee",
PassWd: "923881393time",
})
res := make([]*twitterscraper.Scraper, 0, len(users))
for _, v := range users {
scraper, err := InitScraper(v.User, v.PassWd)
if err != nil {
return nil, err
}
res = append(res, scraper)
}
return res, nil
}
func InitScraper(user, password string) (*twitterscraper.Scraper, error) { func InitScraper(user, password string) (*twitterscraper.Scraper, error) {
scraper3 := twitterscraper.New() scraper3 := twitterscraper.New()
...@@ -35,6 +131,13 @@ type TaskParam struct { ...@@ -35,6 +131,13 @@ type TaskParam struct {
Follow Follow Follow Follow
} }
type Retweet struct {
Enable bool
User string
Next string
Res chan interface{}
}
type Follow struct { type Follow struct {
Enable bool Enable bool
User string User string
...@@ -85,8 +188,10 @@ type FollowerId struct { ...@@ -85,8 +188,10 @@ type FollowerId struct {
} }
type Follower struct { type Follower struct {
//user_id
UserId string `json:"user_id"`
Follower string `json:"follower"` Follower string `json:"follower"`
UserName string `json:"user_name"` UserName string `json:"follower_username"`
} }
func FetchFollowers(scraper *twitterscraper.Scraper, user, next string) (Users, error) { func FetchFollowers(scraper *twitterscraper.Scraper, user, next string) (Users, error) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment