Skip to content

M
mybee
Commit cfb17e33 authored by acud's avatar acud Committed by GitHub
Browse files
Options

pullsync: add explicit termination (#371) 

* pullsync: add explicit termination of a request for intervals
parent 23f47d4d
Hide whitespace changes
Inline Side-by-side
Showing with 471 additions and 54 deletions
pkg/puller/puller.go
View file @ cfb17e33
......@@ -365,11 +365,17 @@ func (p *Puller) histSyncWorker(ctx context.Context, peer swarm.Address, bin uin
}
return
}
top, err := p.syncer.SyncInterval(ctx, peer, bin, s, cur)
top, ruid, err := p.syncer.SyncInterval(ctx, peer, bin, s, cur)
if err != nil {
if logMore {
p.logger.Debugf("histSyncWorker error syncing interval. peer %s, bin %d, cursor %d, err %v", peer.String(), bin, cur, err)
}
if ruid == 0 {
return
}
if err := p.syncer.CancelRuid(peer, ruid); err != nil && logMore {
p.logger.Debugf("histSyncWorker cancel ruid: %v", err)
}
return
}
err = p.addPeerInterval(peer, bin, s, top)
......@@ -399,11 +405,17 @@ func (p *Puller) liveSyncWorker(ctx context.Context, peer swarm.Address, bin uin
return
default:
}
top, err := p.syncer.SyncInterval(ctx, peer, bin, from, math.MaxUint64)
top, ruid, err := p.syncer.SyncInterval(ctx, peer, bin, from, math.MaxUint64)
if err != nil {
if logMore {
p.logger.Debugf("liveSyncWorker exit on sync error. peer %s bin %d from %d err %v", peer, bin, from, err)
}
if ruid == 0 {
return
}
if err := p.syncer.CancelRuid(peer, ruid); err != nil && logMore {
p.logger.Debugf("histSyncWorker cancel ruid: %v", err)
}
return
}
if top == 0 {
......
pkg/pullsync/mock/pullsync.go
View file @ cfb17e33
......@@ -106,7 +106,7 @@ func NewPullSync(opts ...Option) *PullSyncMock {
return s
}
func (p *PullSyncMock) SyncInterval(ctx context.Context, peer swarm.Address, bin uint8, from, to uint64) (topmost uint64, err error) {
func (p *PullSyncMock) SyncInterval(ctx context.Context, peer swarm.Address, bin uint8, from, to uint64) (topmost uint64, ruid uint32, err error) {
isLive := to == math.MaxUint64
call := SyncCall{
......@@ -129,9 +129,9 @@ func (p *PullSyncMock) SyncInterval(ctx context.Context, peer swarm.Address, bin
select {
case <-p.quit:
return 0, context.Canceled
return 0, 1, context.Canceled
case <-ctx.Done():
return 0, ctx.Err()
return 0, 1, ctx.Err()
default:
}
......@@ -150,12 +150,12 @@ func (p *PullSyncMock) SyncInterval(ctx context.Context, peer swarm.Address, bin
if sr.block {
select {
case <-p.quit:
return 0, context.Canceled
return 0, 1, context.Canceled
case <-ctx.Done():
return 0, ctx.Err()
return 0, 1, ctx.Err()
}
}
return sr.topmost, nil
return sr.topmost, 0, nil
}
panic("not found")
}
......@@ -163,7 +163,7 @@ func (p *PullSyncMock) SyncInterval(ctx context.Context, peer swarm.Address, bin
if isLive && p.blockLiveSync {
// don't respond, wait for quit
<-p.quit
return 0, io.EOF
return 0, 1, io.EOF
}
if isLive && len(p.liveSyncReplies) > 0 {
......@@ -175,7 +175,7 @@ func (p *PullSyncMock) SyncInterval(ctx context.Context, peer swarm.Address, bin
v := p.liveSyncReplies[p.liveSyncCalls]
p.liveSyncCalls++
p.mtx.Unlock()
return v, nil
return v, 1, nil
}
if p.autoReply {
......@@ -184,9 +184,9 @@ func (p *PullSyncMock) SyncInterval(ctx context.Context, peer swarm.Address, bin
if t > to {
t = to
}
return t, nil
return t, 1, nil
}
return to, nil
return to, 1, nil
}
func (p *PullSyncMock) GetCursors(_ context.Context, peer swarm.Address) ([]uint64, error) {
......@@ -208,6 +208,10 @@ func (p *PullSyncMock) SyncCalls(peer swarm.Address) (res []SyncCall) {
return res
}
func (p *PullSyncMock) CancelRuid(peer swarm.Address, ruid uint32) error {
return nil
}
func (p *PullSyncMock) LiveSyncCalls(peer swarm.Address) (res []SyncCall) {
p.mtx.Lock()
defer p.mtx.Unlock()
......
pkg/pullsync/pb/pullsync.pb.go
View file @ cfb17e33
......@@ -102,6 +102,94 @@ func (m *Ack) GetCursors() []uint64 {
return nil
}
type Ruid struct {
Ruid uint32 `protobuf:"varint,1,opt,name=Ruid,proto3" json:"Ruid,omitempty"`
}
func (m *Ruid) Reset() { *m = Ruid{} }
func (m *Ruid) String() string { return proto.CompactTextString(m) }
func (*Ruid) ProtoMessage() {}
func (*Ruid) Descriptor() ([]byte, []int) {
return fileDescriptor_d1dee042cf9c065c, []int{2}
}
func (m *Ruid) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *Ruid) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_Ruid.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *Ruid) XXX_Merge(src proto.Message) {
xxx_messageInfo_Ruid.Merge(m, src)
}
func (m *Ruid) XXX_Size() int {
return m.Size()
}
func (m *Ruid) XXX_DiscardUnknown() {
xxx_messageInfo_Ruid.DiscardUnknown(m)
}
var xxx_messageInfo_Ruid proto.InternalMessageInfo
func (m *Ruid) GetRuid() uint32 {
if m != nil {
return m.Ruid
}
return 0
}
type Cancel struct {
Ruid uint32 `protobuf:"varint,1,opt,name=Ruid,proto3" json:"Ruid,omitempty"`
}
func (m *Cancel) Reset() { *m = Cancel{} }
func (m *Cancel) String() string { return proto.CompactTextString(m) }
func (*Cancel) ProtoMessage() {}
func (*Cancel) Descriptor() ([]byte, []int) {
return fileDescriptor_d1dee042cf9c065c, []int{3}
}
func (m *Cancel) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *Cancel) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_Cancel.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *Cancel) XXX_Merge(src proto.Message) {
xxx_messageInfo_Cancel.Merge(m, src)
}
func (m *Cancel) XXX_Size() int {
return m.Size()
}
func (m *Cancel) XXX_DiscardUnknown() {
xxx_messageInfo_Cancel.DiscardUnknown(m)
}
var xxx_messageInfo_Cancel proto.InternalMessageInfo
func (m *Cancel) GetRuid() uint32 {
if m != nil {
return m.Ruid
}
return 0
}
type GetRange struct {
Bin int32 `protobuf:"varint,1,opt,name=Bin,proto3" json:"Bin,omitempty"`
From uint64 `protobuf:"varint,2,opt,name=From,proto3" json:"From,omitempty"`
......@@ -112,7 +200,7 @@ func (m *GetRange) Reset() { *m = GetRange{} }
func (m *GetRange) String() string { return proto.CompactTextString(m) }
func (*GetRange) ProtoMessage() {}
func (*GetRange) Descriptor() ([]byte, []int) {
return fileDescriptor_d1dee042cf9c065c, []int{2}
return fileDescriptor_d1dee042cf9c065c, []int{4}
}
func (m *GetRange) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
......@@ -171,7 +259,7 @@ func (m *Offer) Reset() { *m = Offer{} }
func (m *Offer) String() string { return proto.CompactTextString(m) }
func (*Offer) ProtoMessage() {}
func (*Offer) Descriptor() ([]byte, []int) {
return fileDescriptor_d1dee042cf9c065c, []int{3}
return fileDescriptor_d1dee042cf9c065c, []int{5}
}
func (m *Offer) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
......@@ -222,7 +310,7 @@ func (m *Want) Reset() { *m = Want{} }
func (m *Want) String() string { return proto.CompactTextString(m) }
func (*Want) ProtoMessage() {}
func (*Want) Descriptor() ([]byte, []int) {
return fileDescriptor_d1dee042cf9c065c, []int{4}
return fileDescriptor_d1dee042cf9c065c, []int{6}
}
func (m *Want) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
......@@ -267,7 +355,7 @@ func (m *Delivery) Reset() { *m = Delivery{} }
func (m *Delivery) String() string { return proto.CompactTextString(m) }
func (*Delivery) ProtoMessage() {}
func (*Delivery) Descriptor() ([]byte, []int) {
return fileDescriptor_d1dee042cf9c065c, []int{5}
return fileDescriptor_d1dee042cf9c065c, []int{7}
}
func (m *Delivery) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
......@@ -313,6 +401,8 @@ func (m *Delivery) GetData() []byte {
func init() {
proto.RegisterType((*Syn)(nil), "pullsync.Syn")
proto.RegisterType((*Ack)(nil), "pullsync.Ack")
proto.RegisterType((*Ruid)(nil), "pullsync.Ruid")
proto.RegisterType((*Cancel)(nil), "pullsync.Cancel")
proto.RegisterType((*GetRange)(nil), "pullsync.GetRange")
proto.RegisterType((*Offer)(nil), "pullsync.Offer")
proto.RegisterType((*Want)(nil), "pullsync.Want")
......@@ -322,24 +412,26 @@ func init() {
func init() { proto.RegisterFile("pullsync.proto", fileDescriptor_d1dee042cf9c065c) }
var fileDescriptor_d1dee042cf9c065c = []byte{
// 270 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x44, 0x90, 0x3f, 0x4f, 0xc3, 0x30,
0x10, 0xc5, 0xeb, 0xfc, 0x29, 0xe1, 0x54, 0x55, 0xc8, 0x03, 0xca, 0x50, 0x99, 0xc8, 0x62, 0xc8,
0xc4, 0xc2, 0x02, 0x1b, 0x0d, 0x15, 0xb0, 0x21, 0x99, 0x08, 0x24, 0x36, 0x37, 0x75, 0x21, 0x22,
0xb5, 0x23, 0xdb, 0x45, 0xca, 0xb7, 0xe0, 0x63, 0x31, 0x76, 0x64, 0x44, 0xc9, 0x17, 0x41, 0xb1,
0x12, 0xb1, 0xbd, 0xdf, 0xd3, 0xdd, 0x7b, 0xa7, 0x83, 0x79, 0xbd, 0xaf, 0x2a, 0xd3, 0xc8, 0xe2,
0xa2, 0xd6, 0xca, 0x2a, 0x1c, 0x8d, 0x4c, 0x43, 0xf0, 0x9f, 0x1a, 0x49, 0xcf, 0xc0, 0x5f, 0x16,
0x1f, 0x38, 0x86, 0xa3, 0xdb, 0xbd, 0x36, 0x4a, 0x9b, 0x18, 0x25, 0x7e, 0x1a, 0xb0, 0x11, 0xe9,
0x0d, 0x44, 0xf7, 0xc2, 0x32, 0x2e, 0xdf, 0x04, 0x3e, 0x01, 0x3f, 0x2b, 0x65, 0x8c, 0x12, 0x94,
0x86, 0xac, 0x97, 0x18, 0x43, 0x70, 0xa7, 0xd5, 0x2e, 0xf6, 0x12, 0x94, 0x06, 0xcc, 0x69, 0x3c,
0x07, 0x2f, 0x57, 0xb1, 0xef, 0x1c, 0x2f, 0x57, 0xf4, 0x1a, 0xc2, 0xc7, 0xed, 0x56, 0xe8, 0xbe,
0x24, 0x57, 0xf5, 0x4e, 0x19, 0xeb, 0x22, 0x02, 0x36, 0x22, 0x3e, 0x85, 0xe9, 0x03, 0x37, 0xef,
0xc2, 0xb8, 0xa0, 0x19, 0x1b, 0x88, 0x9e, 0x43, 0xf0, 0xc2, 0xa5, 0xc5, 0x0b, 0x38, 0xce, 0x4a,
0xfb, 0x2c, 0x0a, 0xab, 0xb4, 0xdb, 0x9d, 0xb1, 0x7f, 0x83, 0x5e, 0x41, 0xb4, 0x12, 0x55, 0xf9,
0x29, 0x74, 0xd3, 0x77, 0x2c, 0x37, 0x1b, 0x2d, 0x8c, 0x19, 0xe6, 0x46, 0xec, 0x4f, 0x5d, 0x71,
0xcb, 0x87, 0x06, 0xa7, 0xb3, 0xc5, 0x77, 0x4b, 0xd0, 0xa1, 0x25, 0xe8, 0xb7, 0x25, 0xe8, 0xab,
0x23, 0x93, 0x43, 0x47, 0x26, 0x3f, 0x1d, 0x99, 0xbc, 0x7a, 0xf5, 0x7a, 0x3d, 0x75, 0x3f, 0xbb,
0xfc, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x17, 0x82, 0xda, 0x45, 0x01, 0x00, 0x00,
// 295 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0xbf, 0x4e, 0xf3, 0x30,
0x10, 0xc0, 0xeb, 0x24, 0xed, 0xd7, 0xef, 0x54, 0x2a, 0xe4, 0x01, 0x45, 0xa8, 0x32, 0x95, 0xc5,
0xd0, 0x89, 0x85, 0x05, 0x36, 0x9a, 0x56, 0xc0, 0x86, 0x64, 0x22, 0x90, 0xd8, 0xdc, 0xc4, 0x85,
0x88, 0xd4, 0x8e, 0x6c, 0x07, 0x29, 0x6f, 0xc1, 0x63, 0x31, 0x76, 0x64, 0x44, 0xc9, 0x8b, 0xa0,
0x98, 0x44, 0x2c, 0x4c, 0xfe, 0xfd, 0xee, 0x7c, 0x7f, 0x74, 0x30, 0x2d, 0xca, 0x3c, 0x37, 0x95,
0x4c, 0xce, 0x0a, 0xad, 0xac, 0xc2, 0xe3, 0xde, 0xe9, 0x10, 0xfc, 0xfb, 0x4a, 0xd2, 0x13, 0xf0,
0x97, 0xc9, 0x2b, 0x0e, 0xe1, 0xdf, 0xaa, 0xd4, 0x46, 0x69, 0x13, 0xa2, 0xb9, 0xbf, 0x08, 0x58,
0xaf, 0xf4, 0x18, 0x02, 0x56, 0x66, 0x29, 0xc6, 0x3f, 0x6f, 0x88, 0xe6, 0x68, 0x71, 0xc0, 0x1c,
0xd3, 0x19, 0x8c, 0x56, 0x5c, 0x26, 0x22, 0xff, 0x33, 0x7b, 0x05, 0xe3, 0x1b, 0x61, 0x19, 0x97,
0xcf, 0x02, 0x1f, 0x82, 0x1f, 0x65, 0xd2, 0xa5, 0x87, 0xac, 0xc5, 0xb6, 0xe2, 0x5a, 0xab, 0x5d,
0xe8, 0xcd, 0xd1, 0x22, 0x60, 0x8e, 0xf1, 0x14, 0xbc, 0x58, 0x85, 0xbe, 0x8b, 0x78, 0xb1, 0xa2,
0x97, 0x30, 0xbc, 0xdb, 0x6e, 0x85, 0x6e, 0xd7, 0x8b, 0x55, 0xb1, 0x53, 0xc6, 0xba, 0x16, 0x01,
0xeb, 0x15, 0x1f, 0xc1, 0xe8, 0x96, 0x9b, 0x17, 0x61, 0x5c, 0xa3, 0x09, 0xeb, 0x8c, 0x9e, 0x42,
0xf0, 0xc8, 0xa5, 0xc5, 0x33, 0xf8, 0x1f, 0x65, 0xf6, 0x41, 0x24, 0x56, 0x69, 0x57, 0x3b, 0x61,
0xbf, 0x01, 0x7a, 0x01, 0xe3, 0xb5, 0xc8, 0xb3, 0x37, 0xa1, 0xab, 0x76, 0xc6, 0x32, 0x4d, 0xb5,
0x30, 0xa6, 0xfb, 0xd7, 0x6b, 0xbb, 0xea, 0x9a, 0x5b, 0xde, 0x4d, 0x70, 0x1c, 0xcd, 0x3e, 0x6a,
0x82, 0xf6, 0x35, 0x41, 0x5f, 0x35, 0x41, 0xef, 0x0d, 0x19, 0xec, 0x1b, 0x32, 0xf8, 0x6c, 0xc8,
0xe0, 0xc9, 0x2b, 0x36, 0x9b, 0x91, 0xbb, 0xf6, 0xf9, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xe5,
0x7c, 0x69, 0x94, 0x7f, 0x01, 0x00, 0x00,
}
func (m *Syn) Marshal() (dAtA []byte, err error) {
......@@ -406,6 +498,62 @@ func (m *Ack) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil
}
func (m *Ruid) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *Ruid) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *Ruid) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.Ruid != 0 {
i = encodeVarintPullsync(dAtA, i, uint64(m.Ruid))
i--
dAtA[i] = 0x8
}
return len(dAtA) - i, nil
}
func (m *Cancel) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *Cancel) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *Cancel) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.Ruid != 0 {
i = encodeVarintPullsync(dAtA, i, uint64(m.Ruid))
i--
dAtA[i] = 0x8
}
return len(dAtA) - i, nil
}
func (m *GetRange) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
......@@ -582,6 +730,30 @@ func (m *Ack) Size() (n int) {
return n
}
func (m *Ruid) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
if m.Ruid != 0 {
n += 1 + sovPullsync(uint64(m.Ruid))
}
return n
}
func (m *Cancel) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
if m.Ruid != 0 {
n += 1 + sovPullsync(uint64(m.Ruid))
}
return n
}
func (m *GetRange) Size() (n int) {
if m == nil {
return 0
......@@ -834,6 +1006,150 @@ func (m *Ack) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *Ruid) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPullsync
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Ruid: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Ruid: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Ruid", wireType)
}
m.Ruid = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPullsync
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Ruid |= uint32(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPullsync(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPullsync
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPullsync
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *Cancel) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPullsync
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Cancel: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Cancel: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Ruid", wireType)
}
m.Ruid = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPullsync
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Ruid |= uint32(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPullsync(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPullsync
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPullsync
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *GetRange) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
......
pkg/pullsync/pb/pullsync.proto
View file @ cfb17e33
......@@ -14,6 +14,14 @@ message Ack {
repeated uint64 Cursors = 1;
}
message Ruid {
uint32 Ruid = 1;
}
message Cancel {
uint32 Ruid = 1;
}
message GetRange {
int32 Bin = 1;
uint64 From = 2;
......
pkg/pullsync/pullsync.go
View file @ cfb17e33
......@@ -6,9 +6,12 @@ package pullsync
import (
"context"
"crypto/rand"
"encoding/binary"
"errors"
"fmt"
"io"
"sync"
"time"
"github.com/ethersphere/bee/pkg/bitvector"
......@@ -27,6 +30,7 @@ const (
protocolVersion = "1.0.0"
streamName = "pullsync"
cursorStreamName = "cursors"
cancelStreamName = "cancel"
)
var (
......@@ -37,8 +41,9 @@ var (
var maxPage = 50
type Interface interface {
SyncInterval(ctx context.Context, peer swarm.Address, bin uint8, from, to uint64) (topmost uint64, err error)
SyncInterval(ctx context.Context, peer swarm.Address, bin uint8, from, to uint64) (topmost uint64, ruid uint32, err error)
GetCursors(ctx context.Context, peer swarm.Address) ([]uint64, error)
CancelRuid(peer swarm.Address, ruid uint32) error
}
type Syncer struct {
......@@ -46,6 +51,9 @@ type Syncer struct {
logger logging.Logger
storage pullstorage.Storer
ruidMtx sync.Mutex
ruidCtx map[uint32]func()
Interface
io.Closer
}
......@@ -62,6 +70,7 @@ func New(o Options) *Syncer {
streamer: o.Streamer,
storage: o.Storage,
logger: o.Logger,
ruidCtx: make(map[uint32]func()),
}
}
......@@ -78,6 +87,10 @@ func (s *Syncer) Protocol() p2p.ProtocolSpec {
Name: cursorStreamName,
Handler: s.cursorHandler,
},
{
Name: cancelStreamName,
Handler: s.cancelHandler,
},
},
}
}
......@@ -86,11 +99,20 @@ func (s *Syncer) Protocol() p2p.ProtocolSpec {
// It returns the BinID of highest chunk that was synced from the given interval.
// If the requested interval is too large, the downstream peer has the liberty to
// provide less chunks than requested.
func (s *Syncer) SyncInterval(ctx context.Context, peer swarm.Address, bin uint8, from, to uint64) (topmost uint64, err error) {
func (s *Syncer) SyncInterval(ctx context.Context, peer swarm.Address, bin uint8, from, to uint64) (topmost uint64, ruid uint32, err error) {
stream, err := s.streamer.NewStream(ctx, peer, nil, protocolName, protocolVersion, streamName)
if err != nil {
return 0, fmt.Errorf("new stream: %w", err)
return 0, 0, fmt.Errorf("new stream: %w", err)
}
var ru pb.Ruid
b := make([]byte, 4)
_, err = rand.Read(b)
if err != nil {
return 0, 0, fmt.Errorf("crypto rand: %w", err)
}
ru.Ruid = binary.BigEndian.Uint32(b)
defer func() {
if err != nil {
_ = stream.FullClose()
......@@ -100,25 +122,28 @@ func (s *Syncer) SyncInterval(ctx context.Context, peer swarm.Address, bin uint8
}()
w, r := protobuf.NewWriterAndReader(stream)
if err = w.WriteMsgWithContext(ctx, &ru); err != nil {
return 0, 0, fmt.Errorf("write ruid: %w", err)
}
rangeMsg := &pb.GetRange{Bin: int32(bin), From: from, To: to}
if err = w.WriteMsgWithContext(ctx, rangeMsg); err != nil {
return 0, fmt.Errorf("write get range: %w", err)
return 0, ru.Ruid, fmt.Errorf("write get range: %w", err)
}
var offer pb.Offer
if err = r.ReadMsgWithContext(ctx, &offer); err != nil {
return 0, fmt.Errorf("read offer: %w", err)
return 0, ru.Ruid, fmt.Errorf("read offer: %w", err)
}
if len(offer.Hashes)%swarm.HashSize != 0 {
return 0, fmt.Errorf("inconsistent hash length")
return 0, ru.Ruid, fmt.Errorf("inconsistent hash length")
}
// empty interval (no chunks present in interval).
// return the end of the requested range as topmost.
if len(offer.Hashes) == 0 {
return offer.Topmost, nil
return offer.Topmost, ru.Ruid, nil
}
var (
......@@ -129,7 +154,7 @@ func (s *Syncer) SyncInterval(ctx context.Context, peer swarm.Address, bin uint8
bv, err := bitvector.New(bvLen)
if err != nil {
return 0, fmt.Errorf("new bitvector: %w", err)
return 0, ru.Ruid, fmt.Errorf("new bitvector: %w", err)
}
for i := 0; i < len(offer.Hashes); i += swarm.HashSize {
......@@ -137,11 +162,11 @@ func (s *Syncer) SyncInterval(ctx context.Context, peer swarm.Address, bin uint8
if a.Equal(swarm.ZeroAddress) {
// i'd like to have this around to see we don't see any of these in the logs
s.logger.Errorf("syncer got a zero address hash on offer")
return 0, fmt.Errorf("zero address on offer")
return 0, ru.Ruid, fmt.Errorf("zero address on offer")
}
have, err := s.storage.Has(ctx, a)
if err != nil {
return 0, fmt.Errorf("storage has: %w", err)
return 0, ru.Ruid, fmt.Errorf("storage has: %w", err)
}
if !have {
wantChunks[a.String()] = struct{}{}
......@@ -152,7 +177,7 @@ func (s *Syncer) SyncInterval(ctx context.Context, peer swarm.Address, bin uint8
wantMsg := &pb.Want{BitVector: bv.Bytes()}
if err = w.WriteMsgWithContext(ctx, wantMsg); err != nil {
return 0, fmt.Errorf("write want: %w", err)
return 0, ru.Ruid, fmt.Errorf("write want: %w", err)
}
// if ctr is zero, it means we don't want any chunk in the batch
......@@ -163,21 +188,21 @@ func (s *Syncer) SyncInterval(ctx context.Context, peer swarm.Address, bin uint8
for ; ctr > 0; ctr-- {
var delivery pb.Delivery
if err = r.ReadMsgWithContext(ctx, &delivery); err != nil {
return 0, fmt.Errorf("read delivery: %w", err)
return 0, ru.Ruid, fmt.Errorf("read delivery: %w", err)
}
addr := swarm.NewAddress(delivery.Address)
if _, ok := wantChunks[addr.String()]; !ok {
return 0, ErrUnsolicitedChunk
return 0, ru.Ruid, ErrUnsolicitedChunk
}
delete(wantChunks, addr.String())
if err = s.storage.Put(ctx, storage.ModePutSync, swarm.NewChunk(addr, delivery.Data)); err != nil {
return 0, fmt.Errorf("delivery put: %w", err)
return 0, ru.Ruid, fmt.Errorf("delivery put: %w", err)
}
}
return offer.Topmost, nil
return offer.Topmost, ru.Ruid, nil
}
// handler handles an incoming request to sync an interval
......@@ -185,6 +210,22 @@ func (s *Syncer) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) err
w, r := protobuf.NewWriterAndReader(stream)
defer stream.Close()
var ru pb.Ruid
if err := r.ReadMsgWithContext(ctx, &ru); err != nil {
return fmt.Errorf("send ruid: %w", err)
}
ctx, cancel := context.WithCancel(ctx)
s.ruidMtx.Lock()
s.ruidCtx[ru.Ruid] = cancel
s.ruidMtx.Unlock()
defer func() {
s.ruidMtx.Lock()
delete(s.ruidCtx, ru.Ruid)
s.ruidMtx.Unlock()
}()
defer cancel()
var rn pb.GetRange
if err := r.ReadMsgWithContext(ctx, &rn); err != nil {
return fmt.Errorf("read get range: %w", err)
......@@ -320,6 +361,42 @@ func (s *Syncer) cursorHandler(ctx context.Context, p p2p.Peer, stream p2p.Strea
return nil
}
func (s *Syncer) CancelRuid(peer swarm.Address, ruid uint32) error {
stream, err := s.streamer.NewStream(context.Background(), peer, nil, protocolName, protocolVersion, cancelStreamName)
if err != nil {
return fmt.Errorf("new stream: %w", err)
}
w := protobuf.NewWriter(stream)
defer stream.Close()
var c pb.Cancel
c.Ruid = ruid
if err := w.WriteMsgWithTimeout(5*time.Second, &c); err != nil {
return fmt.Errorf("send cancellation: %w", err)
}
return nil
}
// handler handles an incoming request to explicitly cancel a ruid
func (s *Syncer) cancelHandler(ctx context.Context, p p2p.Peer, stream p2p.Stream) error {
r := protobuf.NewReader(stream)
defer stream.Close()
var c pb.Cancel
if err := r.ReadMsgWithContext(ctx, &c); err != nil {
return fmt.Errorf("read cancel: %w", err)
}
if cancel, ok := s.ruidCtx[c.Ruid]; ok {
cancel()
}
s.ruidMtx.Lock()
delete(s.ruidCtx, c.Ruid)
s.ruidMtx.Unlock()
return nil
}
func (s *Syncer) Close() error {
return nil
}
pkg/pullsync/pullsync_test.go
View file @ cfb17e33
......@@ -61,7 +61,7 @@ func TestIncoming_WantEmptyInterval(t *testing.T) {
psClient, clientDb = newPullSync(recorder)
)
topmost, err := psClient.SyncInterval(context.Background(), swarm.ZeroAddress, 1, 0, 5)
topmost, _, err := psClient.SyncInterval(context.Background(), swarm.ZeroAddress, 1, 0, 5)
if err != nil {
t.Fatal(err)
}
......@@ -84,7 +84,7 @@ func TestIncoming_WantNone(t *testing.T) {
psClient, clientDb = newPullSync(recorder, mock.WithChunks(chunks...))
)
topmost, err := psClient.SyncInterval(context.Background(), swarm.ZeroAddress, 0, 0, 5)
topmost, _, err := psClient.SyncInterval(context.Background(), swarm.ZeroAddress, 0, 0, 5)
if err != nil {
t.Fatal(err)
}
......@@ -107,7 +107,7 @@ func TestIncoming_WantOne(t *testing.T) {
psClient, clientDb = newPullSync(recorder, mock.WithChunks(someChunks(1, 2, 3, 4)...))
)
topmost, err := psClient.SyncInterval(context.Background(), swarm.ZeroAddress, 0, 0, 5)
topmost, _, err := psClient.SyncInterval(context.Background(), swarm.ZeroAddress, 0, 0, 5)
if err != nil {
t.Fatal(err)
}
......@@ -132,7 +132,7 @@ func TestIncoming_WantAll(t *testing.T) {
psClient, clientDb = newPullSync(recorder)
)
topmost, err := psClient.SyncInterval(context.Background(), swarm.ZeroAddress, 0, 0, 5)
topmost, _, err := psClient.SyncInterval(context.Background(), swarm.ZeroAddress, 0, 0, 5)
if err != nil {
t.Fatal(err)
}
......@@ -161,7 +161,7 @@ func TestIncoming_UnsolicitedChunk(t *testing.T) {
psClient, _ = newPullSync(recorder)
)
_, err := psClient.SyncInterval(context.Background(), swarm.ZeroAddress, 0, 0, 5)
_, _, err := psClient.SyncInterval(context.Background(), swarm.ZeroAddress, 0, 0, 5)
if !errors.Is(err, pullsync.ErrUnsolicitedChunk) {
t.Fatalf("expected ErrUnsolicitedChunk but got %v", err)
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment