Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
T
twitter_syncer
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Odysseus
twitter_syncer
Commits
45719791
Commit
45719791
authored
Aug 01, 2024
by
Ubuntu
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
add stop task ok
parent
b40f8f9c
Changes
9
Show whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
468 additions
and
8 deletions
+468
-8
db.go
db.go
+175
-0
db_test.go
db_test.go
+36
-0
swagger.yaml
docs/swagger.yaml
+3
-1
go.mod
go.mod
+8
-0
go.sum
go.sum
+54
-0
main.go
main.go
+47
-7
scraper.go
scraper.go
+0
-0
task.go
task.go
+119
-0
type.go
type.go
+26
-0
No files found.
db.go
0 → 100644
View file @
45719791
package
main
import
(
"encoding/json"
"fmt"
"log/slog"
_
"code.wuban.net.cn/odysseus/twitter_syncer/docs"
// docs are generated by Swag CLI, you have to import them.
// replace with your own docs folder, usually "github.com/username/reponame/docs"
//_ "github.com/gofiber/swagger/example/docs"
"github.com/supabase-community/supabase-go"
)
var
client
*
supabase
.
Client
func
init
()
{
var
API_URL
=
"http://43.198.54.207:8000"
var
API_KEY
=
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q"
for
{
cli
,
err
:=
supabase
.
NewClient
(
API_URL
,
API_KEY
,
nil
)
if
err
!=
nil
{
slog
.
Error
(
"supabase.NewClient"
,
"err"
,
err
.
Error
())
return
}
else
{
client
=
cli
break
}
}
}
func
AddTaskInsertOrUpdate
(
req
Req
)
error
{
task
,
ok
,
err
:=
Query
(
req
)
if
err
!=
nil
{
return
err
}
_
=
task
if
ok
{
//update
return
AddTaskUpdate
(
req
,
task
)
}
else
{
//insert
return
Insert
(
req
)
}
return
nil
}
func
AddTaskUpdate
(
req
Req
,
task
Task
)
error
{
if
!
task
.
Follow
&&
req
.
Follow
{
task
.
Follow
=
true
}
if
!
task
.
Retweet
&&
req
.
Retweet
{
task
.
Retweet
=
true
}
res
,
_
,
err
:=
client
.
From
(
"twitter_task"
)
.
Update
(
&
struct
{
Retweet
bool
`json:"retweet"`
Follow
bool
`json:"follow"`
}{
Retweet
:
task
.
Retweet
,
Follow
:
task
.
Follow
,
},
""
,
"exact"
)
.
Eq
(
"project"
,
req
.
Project
)
.
Eq
(
"tweet_id"
,
req
.
TweetId
)
.
Execute
()
_
=
res
return
err
}
func
StopTaskInsertOrUpdate
(
req
Req
)
error
{
task
,
ok
,
err
:=
Query
(
req
)
if
err
!=
nil
{
return
err
}
if
!
ok
{
return
fmt
.
Errorf
(
"can not stop the task,no record"
)
}
return
StopTaskUpdate
(
req
,
task
)
}
func
StopTaskUpdate
(
req
Req
,
task
Task
)
error
{
if
!
task
.
FollowStop
&&
req
.
Follow
{
task
.
FollowStop
=
true
}
if
!
task
.
RetweetStop
&&
req
.
Retweet
{
task
.
RetweetStop
=
true
}
res
,
_
,
err
:=
client
.
From
(
"twitter_task"
)
.
Update
(
&
struct
{
Retweet
bool
`json:"retweet_stop"`
Follow
bool
`json:"follow_stop"`
}{
Retweet
:
task
.
RetweetStop
,
Follow
:
task
.
FollowStop
,
},
""
,
"exact"
)
.
Eq
(
"project"
,
req
.
Project
)
.
Eq
(
"tweet_id"
,
req
.
TweetId
)
.
Execute
()
_
=
res
return
err
//return nil
}
func
Insert
(
req
Req
)
error
{
res
,
_
,
err
:=
client
.
From
(
"twitter_task"
)
.
Insert
(
req
,
true
,
""
,
"representation"
,
""
)
.
Execute
()
_
=
res
return
err
}
func
Query
(
req
Req
)
(
Task
,
bool
,
error
)
{
data
,
count
,
err
:=
client
.
From
(
"twitter_task"
)
.
Select
(
"*"
,
"exact"
,
false
)
.
Eq
(
"project"
,
req
.
Project
)
.
Eq
(
"tweet_id"
,
req
.
TweetId
)
.
Execute
()
if
err
!=
nil
{
return
Task
{},
false
,
err
}
if
count
==
0
{
return
Task
{},
false
,
nil
}
tasks
:=
make
([]
Task
,
0
)
if
err
:=
json
.
Unmarshal
(
data
,
&
tasks
);
err
!=
nil
{
return
Task
{},
false
,
err
}
var
task
Task
if
len
(
tasks
)
==
1
{
task
=
tasks
[
0
]
}
else
{
return
Task
{},
false
,
fmt
.
Errorf
(
"expected raws len ==1, but actually %d"
,
len
(
tasks
))
}
return
task
,
true
,
nil
}
/*
[{"id":3,"created_at":"2024-07-30T09:18:22.110223+00:00","project":"1","tweet_id":"1","like":true,"retweet":true,"like_stop":null,"retweet_stop":null}]
*/
// type Tasks []Task
type
Task
struct
{
Project
string
`json:"project"`
TweetId
string
`json:"tweet_id"`
Like
bool
`json:"like"`
Follow
bool
`json:"follow"`
Retweet
bool
`json:"retweet"`
LikeStop
bool
`json:"like_stop"`
FollowStop
bool
`json:"follow_stop"`
RetweetStop
bool
`json:"retweet_stop"`
}
db_test.go
0 → 100644
View file @
45719791
package
main
import
(
"fmt"
"testing"
)
func
TestQueryTask
(
t
*
testing
.
T
)
{
req
:=
Req
{
Project
:
"1"
,
TweetId
:
"1"
,
}
task
,
ok
,
err
:=
Query
(
req
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
fmt
.
Println
(
"found"
,
ok
,
task
)
}
func
TestInsertTask
(
t
*
testing
.
T
)
{
req
:=
Req
{
Project
:
"1"
,
TweetId
:
"11"
,
}
err
:=
Insert
(
req
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
// fmt.Println("found", ok, task)
}
docs/swagger.yaml
View file @
45719791
...
...
@@ -142,7 +142,9 @@ components:
type
:
string
tweet_id
:
type
:
string
like
:
like_cancel
:
type
:
boolean
follow
:
type
:
boolean
retweet
:
type
:
boolean
go.mod
View file @
45719791
...
...
@@ -3,6 +3,7 @@ module code.wuban.net.cn/odysseus/twitter_syncer
go 1.22.4
require (
github.com/AlexEidt/Vidio v1.5.1 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/PuerkitoBio/purell v1.2.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
...
...
@@ -15,6 +16,7 @@ require (
github.com/gofiber/fiber/v2 v2.52.5 // indirect
github.com/gofiber/swagger v1.1.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/imperatrona/twitter-scraper v0.0.9 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
...
...
@@ -24,8 +26,14 @@ require (
github.com/rivo/uniseg v0.4.7 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/supabase-community/functions-go v0.0.0-20220927045802-22373e6cb51d // indirect
github.com/supabase-community/gotrue-go v1.2.0 // indirect
github.com/supabase-community/postgrest-go v0.0.11 // indirect
github.com/supabase-community/storage-go v0.7.0 // indirect
github.com/supabase-community/supabase-go v0.0.4 // indirect
github.com/swaggo/files/v2 v2.0.1 // indirect
github.com/swaggo/swag v1.16.3 // indirect
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
github.com/urfave/cli/v2 v2.27.3 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.55.0 // indirect
...
...
go.sum
View file @
45719791
github.com/AlexEidt/Vidio v1.5.1 h1:tovwvtgQagUz1vifiL9OeWkg1fP/XUzFazFKh7tFtaE=
github.com/AlexEidt/Vidio v1.5.1/go.mod h1:djhIMnWMqPrC3X6nB6ymGX6uWWlgw+VayYGKE1bNwmI=
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
...
...
@@ -38,10 +40,13 @@ github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtg
github.com/gofiber/swagger v1.1.0 h1:ff3rg1fB+Rp5JN/N8jfxTiZtMKe/9tB9QDc79fPiJKQ=
github.com/gofiber/swagger v1.1.0/go.mod h1:pRZL0Np35sd+lTODTE5The0G+TMHfNY+oC4hM2/i5m8=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/imperatrona/twitter-scraper v0.0.9 h1:iKvspUkJev7OxAevAXZJJbxheoyMSS1NOJiJHW79+BU=
github.com/imperatrona/twitter-scraper v0.0.9/go.mod h1:+Z1pca7Faf2tzpHVRnRcratFxy/PuDKj2iaygdgZRLM=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
...
...
@@ -80,12 +85,24 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/supabase-community/functions-go v0.0.0-20220927045802-22373e6cb51d h1:LOrsumaZy615ai37h9RjUIygpSubX+F+6rDct1LIag0=
github.com/supabase-community/functions-go v0.0.0-20220927045802-22373e6cb51d/go.mod h1:nnIju6x3+OZSojtGQCQzu0h3kv4HdIZk+UWCnNxtSak=
github.com/supabase-community/gotrue-go v1.2.0 h1:Zm7T5q3qbuwPgC6xyomOBKrSb7X5dvmjDZEmNST7MoE=
github.com/supabase-community/gotrue-go v1.2.0/go.mod h1:86DXBiAUNcbCfgbeOPEh0PQxScLfowUbYgakETSFQOw=
github.com/supabase-community/postgrest-go v0.0.11 h1:717GTUMfLJxSBuAeEQG2MuW5Q62Id+YrDjvjprTSErg=
github.com/supabase-community/postgrest-go v0.0.11/go.mod h1:cw6LfzMyK42AOSBA1bQ/HZ381trIJyuui2GWhraW7Cc=
github.com/supabase-community/storage-go v0.7.0 h1:cJ8HLbbnL54H5rHPtHfiwtpRwcbDfA3in9HL/ucHnqA=
github.com/supabase-community/storage-go v0.7.0/go.mod h1:oBKcJf5rcUXy3Uj9eS5wR6mvpwbmvkjOtAA+4tGcdvQ=
github.com/supabase-community/supabase-go v0.0.4 h1:sxMenbq6N8a3z9ihNpN3lC2FL3E1YuTQsjX09VPRp+U=
github.com/supabase-community/supabase-go v0.0.4/go.mod h1:SSHsXoOlc+sq8XeXaf0D3gE2pwrq5bcUfzm0+08u/o8=
github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw=
github.com/swaggo/files/v2 v2.0.0/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM=
github.com/swaggo/files/v2 v2.0.1 h1:XCVJO/i/VosCDsJu1YLpdejGsGnBE9deRMpjN4pJLHk=
github.com/swaggo/files/v2 v2.0.1/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM=
github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg=
github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk=
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
github.com/urfave/cli/v2 v2.27.3 h1:/POWahRmdh7uztQ3CYnaDddk0Rm90PyOgIxgW2rr41M=
github.com/urfave/cli/v2 v2.27.3/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
...
...
@@ -98,33 +115,70 @@ github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVS
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
...
...
main.go
View file @
45719791
...
...
@@ -50,13 +50,14 @@ func main() {
req
:=
Req
{}
if
err
:=
json
.
Unmarshal
(
c
.
Request
()
.
Body
(),
&
req
);
err
!=
nil
{
slog
.
Error
(
"json.Unmarshal(c.Request().Body(), &req)"
,
"err"
,
err
.
Error
())
return
c
.
JSON
(
Res
{
Code
:
500
,
Msg
:
err
.
Error
(),
})
}
slog
.
Info
(
c
.
Route
()
.
Path
,
"Project"
,
req
.
Project
,
"TweetId"
,
req
.
TweetId
,
"Retweet"
,
req
.
Retweet
,
"
Like"
,
req
.
Like
)
slog
.
Info
(
c
.
Route
()
.
Path
,
"Project"
,
req
.
Project
,
"TweetId"
,
req
.
TweetId
,
"Retweet"
,
req
.
Retweet
,
"
Follow"
,
req
.
Follow
)
if
req
.
Project
==
""
||
req
.
TweetId
==
""
{
return
c
.
JSON
(
Res
{
...
...
@@ -65,12 +66,30 @@ func main() {
})
}
if
!
req
.
Like
&&
!
req
.
Retweet
{
if
!
req
.
Retweet
&&
!
req
.
Follow
{
return
c
.
JSON
(
Res
{
Code
:
200
,
Code
:
500
,
Msg
:
"like or retweet must be true"
,
})
}
// req.AddOrStop = true
err
:=
AddTaskInsertOrUpdate
(
req
)
//res, _, err := client.From("twitter_task").Insert(req, true, "", "representation", "").Execute()
if
err
!=
nil
{
slog
.
Error
(
"twitter_syncer insert"
,
"err"
,
err
.
Error
())
return
c
.
JSON
(
Res
{
Code
:
500
,
Msg
:
err
.
Error
(),
})
}
//slog.Info("twitter_syncer insert", "res", string(res))
return
c
.
JSON
(
Res
{
Code
:
200
,
})
...
...
@@ -90,7 +109,7 @@ func main() {
})
}
slog
.
Info
(
c
.
Route
()
.
Path
,
"Project"
,
req
.
Project
,
"TweetId"
,
req
.
TweetId
,
"Retweet"
,
req
.
Retweet
,
"
Like"
,
req
.
Like
)
slog
.
Info
(
c
.
Route
()
.
Path
,
"Project"
,
req
.
Project
,
"TweetId"
,
req
.
TweetId
,
"Retweet"
,
req
.
Retweet
,
"
Follow"
,
req
.
Follow
)
if
req
.
Project
==
""
||
req
.
TweetId
==
""
{
return
c
.
JSON
(
Res
{
...
...
@@ -99,12 +118,23 @@ func main() {
})
}
if
!
req
.
Like
&&
!
req
.
Retweet
{
if
!
req
.
Retweet
&&
!
req
.
Follow
{
return
c
.
JSON
(
Res
{
Code
:
200
,
})
}
err
:=
StopTaskInsertOrUpdate
(
req
)
if
err
!=
nil
{
slog
.
Error
(
"twitter_syncer stop"
,
"err"
,
err
.
Error
())
return
c
.
JSON
(
Res
{
Code
:
500
,
Msg
:
err
.
Error
(),
})
}
return
c
.
JSON
(
Res
{
Code
:
200
,
})
...
...
@@ -122,8 +152,18 @@ type Res struct {
}
type
Req
struct
{
// AddOrStop bool
Project
string
`json:"project"`
TweetId
string
`json:"tweet_id"`
Like
bool
`json:"like"`
// Like bool `json:"like"`
Follow
bool
`json:"follow"`
Retweet
bool
`json:"retweet"`
}
// type ReqDB struct {
// AddOrStop bool
// Project string `json:"project"`
// TweetId string `json:"tweet_id"`
// Like bool `json:"like"`
// Retweet bool `json:"retweet"`
// }
scraper.go
0 → 100644
View file @
45719791
task.go
0 → 100644
View file @
45719791
package
main
import
(
"strings"
"time"
b64
"encoding/base64"
twitterscraper
"github.com/imperatrona/twitter-scraper"
)
func
AddTask
()
{
}
func
StopTask
()
{
}
func
InitScraper
(
user
,
password
string
)
(
*
twitterscraper
.
Scraper
,
error
)
{
scraper3
:=
twitterscraper
.
New
()
//err := scraper3.Login("Wade_Leeeee", "923881393time")
if
err
:=
scraper3
.
Login
(
user
,
password
);
err
!=
nil
{
return
nil
,
err
}
return
scraper3
,
nil
}
type
TaskParam
struct
{
Follow
Follow
}
type
Follow
struct
{
Enable
bool
User
string
Next
string
Res
chan
Users
}
func
ImplementTask
(
done
<-
chan
interface
{},
scraper3
*
twitterscraper
.
Scraper
,
tickerD
time
.
Duration
,
inStream
<-
chan
TaskParam
)
{
go
func
()
{
ticker
:=
time
.
NewTicker
(
tickerD
)
for
{
select
{
case
<-
done
:
return
case
<-
ticker
.
C
:
select
{
case
<-
done
:
return
case
req
:=
<-
inStream
:
if
req
.
Follow
.
Enable
{
followers
,
err
:=
FetchFollowers
(
scraper3
,
req
.
Follow
.
User
,
req
.
Follow
.
Next
)
followers
.
Error
=
err
select
{
case
<-
done
:
return
case
req
.
Follow
.
Res
<-
followers
:
}
}
}
}
}
}()
}
type
FollowerId
struct
{
Follower
Id
int
`json:"id"`
CreatedAt
string
`json:"created_at"`
}
type
Follower
struct
{
Follower
string
`json:"follower"`
UserName
string
`json:"user_name"`
}
func
FetchFollowers
(
scraper
*
twitterscraper
.
Scraper
,
user
,
next
string
)
(
Users
,
error
)
{
users
,
newNext
,
err
:=
scraper
.
FetchFollowers
(
user
,
20
,
next
)
if
err
!=
nil
{
return
Users
{},
err
}
usersWithUserNumber
:=
make
([]
Profile
,
0
,
len
(
users
))
for
_
,
v
:=
range
users
{
sDec
,
_
:=
b64
.
StdEncoding
.
DecodeString
(
v
.
UserID
)
userId
,
_
:=
strings
.
CutPrefix
(
string
(
sDec
),
"User:"
)
item
:=
Profile
{
UserIdAsNumber
:
userId
,
Profile
:
v
,
}
usersWithUserNumber
=
append
(
usersWithUserNumber
,
item
)
}
res
:=
Users
{
Profiles
:
usersWithUserNumber
,
Current
:
next
,
Next
:
newNext
,
}
return
res
,
nil
}
type.go
0 → 100644
View file @
45719791
package
main
import
(
twitterscraper
"github.com/imperatrona/twitter-scraper"
)
/*
users {"Avatar":"https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png","Banner":"","Biography":"","Birthday":"","FollowersCount":61,"FollowingCount":7,"FriendsCount":77,"IsPrivate":false,"IsVerified":false,"Joined":"2022-06-11T15:16:41Z","LikesCount":7,"ListedCount":0,"Location":"","Name":"Wade","PinnedTweetIDs":[],"TweetsCount":55,"URL":"https://twitter.com/Wade_Leeeee","UserID":"VXNlcjoxNTM1NjQyMTUyNTY2MjU5NzEy","Username":"Wade_Leeeee","Website":"","Sensitive":false,"Following":false,"FollowedBy":false}
k 0 Wade_Leeeee VXNlcjoxNTM1NjQyMTUyNTY2MjU5NzEy
User:1535642152566259712
users {"Avatar":"https://pbs.twimg.com/profile_images/1787913259074691072/XioiWSMF_normal.jpg","Banner":"https://pbs.twimg.com/profile_banners/1568295462486016008/1715106782","Biography":"Positive mind, positive vibes, positive life ✨","Birthday":"","FollowersCount":47,"FollowingCount":0,"FriendsCount":548,"IsPrivate":false,"IsVerified":false,"Joined":"2022-09-09T17:49:17Z","LikesCount":0,"ListedCount":0,"Location":"South Kensington, UK","Name":"Aurora","PinnedTweetIDs":[],"TweetsCount":21,"URL":"https://twitter.com/Shae85083106","UserID":"VXNlcjoxNTY4Mjk1NDYyNDg2MDE2MDA4","Username":"Shae85083106","Website":"","Sensitive":false,"Following":false,"FollowedBy":false}
k 1 Shae85083106 VXNlcjoxNTY4Mjk1NDYyNDg2MDE2MDA4
User:1568295462486016008
*/
type
Users
struct
{
Profiles
[]
Profile
Current
string
Next
string
Error
error
}
type
Profile
struct
{
UserIdAsNumber
string
*
twitterscraper
.
Profile
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment