Skip to content

T
twitter_syncer
Commit 45719791 authored by Ubuntu's avatar Ubuntu
Browse files
Options

add stop task ok

parent b40f8f9c
Expand all Show whitespace changes
Inline Side-by-side
Showing with 468 additions and 8 deletions
db.go 0 → 100644
View file @ 45719791
package main
import (
"encoding/json"
"fmt"
"log/slog"
_ "code.wuban.net.cn/odysseus/twitter_syncer/docs"
// docs are generated by Swag CLI, you have to import them.
// replace with your own docs folder, usually "github.com/username/reponame/docs"
//_ "github.com/gofiber/swagger/example/docs"
"github.com/supabase-community/supabase-go"
)
var client *supabase.Client
func init() {
var API_URL = "http://43.198.54.207:8000"
var API_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q"
for {
cli, err := supabase.NewClient(API_URL, API_KEY, nil)
if err != nil {
slog.Error("supabase.NewClient", "err", err.Error())
return
} else {
client = cli
break
}
}
}
func AddTaskInsertOrUpdate(req Req) error {
task, ok, err := Query(req)
if err != nil {
return err
}
_ = task
if ok {
//update
return AddTaskUpdate(req, task)
} else {
//insert
return Insert(req)
}
return nil
}
func AddTaskUpdate(req Req, task Task) error {
if !task.Follow && req.Follow {
task.Follow = true
}
if !task.Retweet && req.Retweet {
task.Retweet = true
}
res, _, err := client.From("twitter_task").Update(&struct {
Retweet bool `json:"retweet"`
Follow bool `json:"follow"`
}{
Retweet: task.Retweet,
Follow: task.Follow,
}, "", "exact").Eq("project", req.Project).Eq("tweet_id", req.TweetId).Execute()
_ = res
return err
}
func StopTaskInsertOrUpdate(req Req) error {
task, ok, err := Query(req)
if err != nil {
return err
}
if !ok {
return fmt.Errorf("can not stop the task,no record")
}
return StopTaskUpdate(req, task)
}
func StopTaskUpdate(req Req, task Task) error {
if !task.FollowStop && req.Follow {
task.FollowStop = true
}
if !task.RetweetStop && req.Retweet {
task.RetweetStop = true
}
res, _, err := client.From("twitter_task").Update(&struct {
Retweet bool `json:"retweet_stop"`
Follow bool `json:"follow_stop"`
}{
Retweet: task.RetweetStop,
Follow: task.FollowStop,
}, "", "exact").Eq("project", req.Project).Eq("tweet_id", req.TweetId).Execute()
_ = res
return err
//return nil
}
func Insert(req Req) error {
res, _, err := client.From("twitter_task").Insert(req, true, "", "representation", "").Execute()
_ = res
return err
}
func Query(req Req) (Task, bool, error) {
data, count, err := client.From("twitter_task").Select("*", "exact", false).Eq("project", req.Project).Eq("tweet_id", req.TweetId).Execute()
if err != nil {
return Task{}, false, err
}
if count == 0 {
return Task{}, false, nil
}
tasks := make([]Task, 0)
if err := json.Unmarshal(data, &tasks); err != nil {
return Task{}, false, err
}
var task Task
if len(tasks) == 1 {
task = tasks[0]
} else {
return Task{}, false, fmt.Errorf("expected raws len ==1, but actually %d", len(tasks))
}
return task, true, nil
}
/*
[{"id":3,"created_at":"2024-07-30T09:18:22.110223+00:00","project":"1","tweet_id":"1","like":true,"retweet":true,"like_stop":null,"retweet_stop":null}]
*/
// type Tasks []Task
type Task struct {
Project string `json:"project"`
TweetId string `json:"tweet_id"`
Like bool `json:"like"`
Follow bool `json:"follow"`
Retweet bool `json:"retweet"`
LikeStop bool `json:"like_stop"`
FollowStop bool `json:"follow_stop"`
RetweetStop bool `json:"retweet_stop"`
}
db_test.go 0 → 100644
View file @ 45719791
package main
import (
"fmt"
"testing"
)
func TestQueryTask(t *testing.T) {
req := Req{
Project: "1",
TweetId: "1",
}
task, ok, err := Query(req)
if err != nil {
t.Fatal(err)
}
fmt.Println("found", ok, task)
}
func TestInsertTask(t *testing.T) {
req := Req{
Project: "1",
TweetId: "11",
}
err := Insert(req)
if err != nil {
t.Fatal(err)
}
// fmt.Println("found", ok, task)
}
docs/swagger.yaml
View file @ 45719791
......@@ -142,7 +142,9 @@ components:
type: string
tweet_id:
type: string
like:
like_cancel:
type: boolean
follow:
type: boolean
retweet:
type: boolean
go.mod
View file @ 45719791
......@@ -3,6 +3,7 @@ module code.wuban.net.cn/odysseus/twitter_syncer
go 1.22.4
require (
github.com/AlexEidt/Vidio v1.5.1 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/PuerkitoBio/purell v1.2.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
......@@ -15,6 +16,7 @@ require (
github.com/gofiber/fiber/v2 v2.52.5 // indirect
github.com/gofiber/swagger v1.1.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/imperatrona/twitter-scraper v0.0.9 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
......@@ -24,8 +26,14 @@ require (
github.com/rivo/uniseg v0.4.7 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/supabase-community/functions-go v0.0.0-20220927045802-22373e6cb51d // indirect
github.com/supabase-community/gotrue-go v1.2.0 // indirect
github.com/supabase-community/postgrest-go v0.0.11 // indirect
github.com/supabase-community/storage-go v0.7.0 // indirect
github.com/supabase-community/supabase-go v0.0.4 // indirect
github.com/swaggo/files/v2 v2.0.1 // indirect
github.com/swaggo/swag v1.16.3 // indirect
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
github.com/urfave/cli/v2 v2.27.3 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.55.0 // indirect
......
go.sum
View file @ 45719791
This diff is collapsed.
main.go
View file @ 45719791
......@@ -50,13 +50,14 @@ func main() {
req := Req{}
if err := json.Unmarshal(c.Request().Body(), &req); err != nil {
slog.Error("json.Unmarshal(c.Request().Body(), &req)", "err", err.Error())
return c.JSON(Res{
Code: 500,
Msg: err.Error(),
})
}
slog.Info(c.Route().Path, "Project", req.Project, "TweetId", req.TweetId, "Retweet", req.Retweet, "Like", req.Like)
slog.Info(c.Route().Path, "Project", req.Project, "TweetId", req.TweetId, "Retweet", req.Retweet, "Follow", req.Follow)
if req.Project == "" || req.TweetId == "" {
return c.JSON(Res{
......@@ -65,12 +66,30 @@ func main() {
})
}
if !req.Like && !req.Retweet {
if !req.Retweet && !req.Follow {
return c.JSON(Res{
Code: 200,
Code: 500,
Msg: "like or retweet must be true",
})
}
// req.AddOrStop = true
err := AddTaskInsertOrUpdate(req)
//res, _, err := client.From("twitter_task").Insert(req, true, "", "representation", "").Execute()
if err != nil {
slog.Error("twitter_syncer insert", "err", err.Error())
return c.JSON(Res{
Code: 500,
Msg: err.Error(),
})
}
//slog.Info("twitter_syncer insert", "res", string(res))
return c.JSON(Res{
Code: 200,
})
......@@ -90,7 +109,7 @@ func main() {
})
}
slog.Info(c.Route().Path, "Project", req.Project, "TweetId", req.TweetId, "Retweet", req.Retweet, "Like", req.Like)
slog.Info(c.Route().Path, "Project", req.Project, "TweetId", req.TweetId, "Retweet", req.Retweet, "Follow", req.Follow)
if req.Project == "" || req.TweetId == "" {
return c.JSON(Res{
......@@ -99,12 +118,23 @@ func main() {
})
}
if !req.Like && !req.Retweet {
if !req.Retweet && !req.Follow {
return c.JSON(Res{
Code: 200,
})
}
err := StopTaskInsertOrUpdate(req)
if err != nil {
slog.Error("twitter_syncer stop", "err", err.Error())
return c.JSON(Res{
Code: 500,
Msg: err.Error(),
})
}
return c.JSON(Res{
Code: 200,
})
......@@ -122,8 +152,18 @@ type Res struct {
}
type Req struct {
// AddOrStop bool
Project string `json:"project"`
TweetId string `json:"tweet_id"`
Like bool `json:"like"`
// Like bool `json:"like"`
Follow bool `json:"follow"`
Retweet bool `json:"retweet"`
}
// type ReqDB struct {
// AddOrStop bool
// Project string `json:"project"`
// TweetId string `json:"tweet_id"`
// Like bool `json:"like"`
// Retweet bool `json:"retweet"`
// }
task.go 0 → 100644
View file @ 45719791
package main
import (
"strings"
"time"
b64 "encoding/base64"
twitterscraper "github.com/imperatrona/twitter-scraper"
)
func AddTask() {
}
func StopTask() {
}
func InitScraper(user, password string) (*twitterscraper.Scraper, error) {
scraper3 := twitterscraper.New()
//err := scraper3.Login("Wade_Leeeee", "923881393time")
if err := scraper3.Login(user, password); err != nil {
return nil, err
}
return scraper3, nil
}
type TaskParam struct {
Follow Follow
}
type Follow struct {
Enable bool
User string
Next string
Res chan Users
}
func ImplementTask(done <-chan interface{}, scraper3 *twitterscraper.Scraper, tickerD time.Duration, inStream <-chan TaskParam) {
go func() {
ticker := time.NewTicker(tickerD)
for {
select {
case <-done:
return
case <-ticker.C:
select {
case <-done:
return
case req := <-inStream:
if req.Follow.Enable {
followers, err := FetchFollowers(scraper3, req.Follow.User, req.Follow.Next)
followers.Error = err
select {
case <-done:
return
case req.Follow.Res <- followers:
}
}
}
}
}
}()
}
type FollowerId struct {
Follower
Id int `json:"id"`
CreatedAt string `json:"created_at"`
}
type Follower struct {
Follower string `json:"follower"`
UserName string `json:"user_name"`
}
func FetchFollowers(scraper *twitterscraper.Scraper, user, next string) (Users, error) {
users, newNext, err := scraper.FetchFollowers(user, 20, next)
if err != nil {
return Users{}, err
}
usersWithUserNumber := make([]Profile, 0, len(users))
for _, v := range users {
sDec, _ := b64.StdEncoding.DecodeString(v.UserID)
userId, _ := strings.CutPrefix(string(sDec), "User:")
item := Profile{
UserIdAsNumber: userId,
Profile: v,
}
usersWithUserNumber = append(usersWithUserNumber, item)
}
res := Users{
Profiles: usersWithUserNumber,
Current: next,
Next: newNext,
}
return res, nil
}
type.go 0 → 100644
View file @ 45719791
package main
import (
twitterscraper "github.com/imperatrona/twitter-scraper"
)
/*
users {"Avatar":"https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png","Banner":"","Biography":"","Birthday":"","FollowersCount":61,"FollowingCount":7,"FriendsCount":77,"IsPrivate":false,"IsVerified":false,"Joined":"2022-06-11T15:16:41Z","LikesCount":7,"ListedCount":0,"Location":"","Name":"Wade","PinnedTweetIDs":[],"TweetsCount":55,"URL":"https://twitter.com/Wade_Leeeee","UserID":"VXNlcjoxNTM1NjQyMTUyNTY2MjU5NzEy","Username":"Wade_Leeeee","Website":"","Sensitive":false,"Following":false,"FollowedBy":false}
k 0 Wade_Leeeee VXNlcjoxNTM1NjQyMTUyNTY2MjU5NzEy
User:1535642152566259712
users {"Avatar":"https://pbs.twimg.com/profile_images/1787913259074691072/XioiWSMF_normal.jpg","Banner":"https://pbs.twimg.com/profile_banners/1568295462486016008/1715106782","Biography":"Positive mind, positive vibes, positive life ✨","Birthday":"","FollowersCount":47,"FollowingCount":0,"FriendsCount":548,"IsPrivate":false,"IsVerified":false,"Joined":"2022-09-09T17:49:17Z","LikesCount":0,"ListedCount":0,"Location":"South Kensington, UK","Name":"Aurora","PinnedTweetIDs":[],"TweetsCount":21,"URL":"https://twitter.com/Shae85083106","UserID":"VXNlcjoxNTY4Mjk1NDYyNDg2MDE2MDA4","Username":"Shae85083106","Website":"","Sensitive":false,"Following":false,"FollowedBy":false}
k 1 Shae85083106 VXNlcjoxNTY4Mjk1NDYyNDg2MDE2MDA4
User:1568295462486016008
*/
type Users struct {
Profiles []Profile
Current string
Next string
Error error
}
type Profile struct {
UserIdAsNumber string
*twitterscraper.Profile
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment