Skip to content

T
twitter_syncer
Commit 2b2c3b48 authored by vicotor's avatar vicotor
Browse files
Options

fix bug

parent db94164e
Show whitespace changes
Inline Side-by-side
Showing with 64 additions and 18 deletions
cmd/manual/main.go
View file @ 2b2c3b48
...@@ -58,48 +58,91 @@ func manual(name string, bee string, sync_all bool) { ...@@ -58,48 +58,91 @@ func manual(name string, bee string, sync_all bool) {
} }
taskUser := core.UserObjectToUserTask(users) taskUser := core.UserObjectToUserTask(users)
slog.Info("GetFollowerList", "task id", todoTask.TaskId, "cursor", cursor, "len(users)", len(users), "newCursor", newCursor) slog.Info("GetFollowerList", "task id", todoTask.TaskId, "cursor", cursor, "len(users)", len(users), "newCursor", newCursor)
if (cursor != "" && newCursor == "") || (cursor != "" && newCursor == "0") { ok, l := page.GetIdx().Idx(taskUser)
slog.Info("Get all followers finished") if ok {
l := page.GetIdx().List
res := make([]core.UserTask, 0, l.Len()) res := make([]core.UserTask, 0, l.Len())
for e := l.Front(); e != nil; e = e.Next() { for e := l.Front(); e != nil; e = e.Next() {
if user, ok := e.Value.(core.UserTask); ok { if user, ok := e.Value.(core.UserTask); ok {
res = append(res, user) res = append(res, user)
} }
} }
// save it to json file.
d, _ := json.MarshalIndent(res, "", " ")
filename := fmt.Sprintf("follower-%s.json", todoTask.TaskId)
err = os.WriteFile(filename, d, 0644)
if err != nil {
slog.Error("write followers", "err", err.Error())
} else {
slog.Info("write followers", "task id", todoTask.TaskId, "len(users)", len(res))
}
if err := core.InsertTaskRes(res, todoTask.TaskType, todoTask.TaskId); err != nil { if err := core.InsertTaskRes(res, todoTask.TaskType, todoTask.TaskId); err != nil {
slog.Error("InsertTaskRes", "task id", todoTask.TaskId, slog.Error("InsertTaskRes", "task id", todoTask.TaskId,
"t.TaskType", todoTask.TaskType, "len(users)", len(users), "err", err.Error()) "t.TaskType", todoTask.TaskType, "len(users)", len(users), "err", err.Error())
} else {
slog.Info("InsertTaskRes", "task id", todoTask.TaskId, "len(users)", len(res))
} }
return return
} else { } else {
ok, l := page.GetIdx().Idx(taskUser) if newCursor == "" || newCursor == "0" {
if ok { // all followers are found.
slog.Info("Get all followers finished")
l := page.GetIdx().List
res := make([]core.UserTask, 0, l.Len()) res := make([]core.UserTask, 0, l.Len())
for e := l.Front(); e != nil; e = e.Next() { for e := l.Front(); e != nil; e = e.Next() {
if user, ok := e.Value.(core.UserTask); ok { if user, ok := e.Value.(core.UserTask); ok {
res = append(res, user) res = append(res, user)
} }
} }
// save it to json file.
d, _ := json.MarshalIndent(res, "", " ")
filename := fmt.Sprintf("follower-%s.json", todoTask.TaskId)
err = os.WriteFile(filename, d, 0644)
if err != nil {
slog.Error("write followers", "err", err.Error())
} else {
slog.Info("write followers", "task id", todoTask.TaskId, "len(users)", len(res))
}
if err := core.InsertTaskRes(res, todoTask.TaskType, todoTask.TaskId); err != nil { if err := core.InsertTaskRes(res, todoTask.TaskType, todoTask.TaskId); err != nil {
slog.Error("InsertTaskRes", "task id", todoTask.TaskId, slog.Error("InsertTaskRes", "task id", todoTask.TaskId,
"t.TaskType", todoTask.TaskType, "len(users)", len(users), "err", err.Error()) "t.TaskType", todoTask.TaskType, "len(users)", len(users), "err", err.Error())
} else {
slog.Info("InsertTaskRes", "task id", todoTask.TaskId, "len(users)", len(res))
} }
return return
} }
} }
//if (cursor != "" && newCursor == "") || (cursor != "" && newCursor == "0") {
// slog.Info("Get all followers finished")
// l := page.GetIdx().List
// res := make([]core.UserTask, 0, l.Len())
// for e := l.Front(); e != nil; e = e.Next() {
// if user, ok := e.Value.(core.UserTask); ok {
// res = append(res, user)
// }
// }
// // save it to json file.
// d, _ := json.MarshalIndent(res, "", " ")
// filename := fmt.Sprintf("follower-%s.json", todoTask.TaskId)
// err = os.WriteFile(filename, d, 0644)
// if err != nil {
// slog.Error("write followers", "err", err.Error())
// } else {
// slog.Info("write followers", "task id", todoTask.TaskId, "len(users)", len(res))
// }
//
// if err := core.InsertTaskRes(res, todoTask.TaskType, todoTask.TaskId); err != nil {
// slog.Error("InsertTaskRes", "task id", todoTask.TaskId,
// "t.TaskType", todoTask.TaskType, "len(users)", len(users), "err", err.Error())
// }
// return
//} else {
// ok, l := page.GetIdx().Idx(taskUser)
// if ok {
// res := make([]core.UserTask, 0, l.Len())
// for e := l.Front(); e != nil; e = e.Next() {
// if user, ok := e.Value.(core.UserTask); ok {
// res = append(res, user)
// }
// }
// if err := core.InsertTaskRes(res, todoTask.TaskType, todoTask.TaskId); err != nil {
// slog.Error("InsertTaskRes", "task id", todoTask.TaskId,
// "t.TaskType", todoTask.TaskType, "len(users)", len(users), "err", err.Error())
// } else {
// slog.Info("InsertTaskRes", "task id", todoTask.TaskId, "len(users)", len(res))
// }
// return
// }
//}
if newCursor == "" || newCursor == "0" { if newCursor == "" || newCursor == "0" {
break break
......
core/idx.go
View file @ 2b2c3b48
...@@ -35,6 +35,9 @@ func NewIdx(i []UserTask) *Idx { ...@@ -35,6 +35,9 @@ func NewIdx(i []UserTask) *Idx {
} }
func (s *Idx) Idx(page []UserTask) (bool, *list.List) { func (s *Idx) Idx(page []UserTask) (bool, *list.List) {
if len(page) == 0 {
return false, nil
}
if s.idx != nil && len(s.idx) == 0 { if s.idx != nil && len(s.idx) == 0 {
...@@ -61,7 +64,7 @@ func (s *Idx) Idx(page []UserTask) (bool, *list.List) { ...@@ -61,7 +64,7 @@ func (s *Idx) Idx(page []UserTask) (bool, *list.List) {
} }
} }
if !match { if !match {
newItems.PushFront(v) newItems.PushBack(v)
} else { } else {
// stop, all new items has added to newItems. // stop, all new items has added to newItems.
break break
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment