Commit 5fb8cd84 authored by Zahoor Mohamed's avatar Zahoor Mohamed Committed by GitHub

[bee #118] Receipts for pushsync (#122)

* pushsync: add receipt forwarding logic
Co-authored-by: default avataracud <12988138+acud@users.noreply.github.com>
parent 8e958f02
...@@ -14,29 +14,58 @@ type metrics struct { ...@@ -14,29 +14,58 @@ type metrics struct {
// to be able to return them by Metrics() // to be able to return them by Metrics()
// using reflection // using reflection
SendChunkCounter prometheus.Counter TotalChunksToBeSentCounter prometheus.Counter
SendChunkTimer prometheus.Counter TotalChunksSynced prometheus.Counter
SendChunkErrorCounter prometheus.Counter TotalChunksStoredInDB prometheus.Counter
MarkAndSweepTimer prometheus.Counter ChunksSentCounter prometheus.Counter
ChunksReceivedCounter prometheus.Counter
ChunksInBatch prometheus.Gauge SendChunkErrorCounter prometheus.Counter
ReceivedChunkErrorCounter prometheus.Counter
ReceiptsReceivedCounter prometheus.Counter
ReceiptsSentCounter prometheus.Counter
SendReceiptErrorCounter prometheus.Counter
ReceiveReceiptErrorCounter prometheus.Counter
ErrorSettingChunkToSynced prometheus.Counter
RetriesExhaustedCounter prometheus.Counter
InvalidReceiptReceived prometheus.Counter
SendChunkTimer prometheus.Histogram
ReceiptRTT prometheus.Histogram
MarkAndSweepTimer prometheus.Histogram
} }
func newMetrics() metrics { func newMetrics() metrics {
subsystem := "pushsync" subsystem := "pushsync"
return metrics{ return metrics{
SendChunkCounter: prometheus.NewCounter(prometheus.CounterOpts{ TotalChunksToBeSentCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace, Namespace: m.Namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "send_chunk", Name: "total_chunk_to_be_sent",
Help: "Total chunks to be sent.", Help: "Total chunks to be sent.",
}), }),
SendChunkTimer: prometheus.NewCounter(prometheus.CounterOpts{ TotalChunksSynced: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "total_chunk_synced",
Help: "Total chunks synced succesfully with valid receipts.",
}),
TotalChunksStoredInDB: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "total_chunk_stored_in_DB",
Help: "Total chunks stored succesfully in local store.",
}),
ChunksSentCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "sent_chunk",
Help: "Total chunks sent.",
}),
ChunksReceivedCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace, Namespace: m.Namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "send_chunk_time_taken", Name: "received_chunk",
Help: "Total time taken to send a chunk.", Help: "Total chunks received.",
}), }),
SendChunkErrorCounter: prometheus.NewCounter(prometheus.CounterOpts{ SendChunkErrorCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace, Namespace: m.Namespace,
...@@ -44,18 +73,75 @@ func newMetrics() metrics { ...@@ -44,18 +73,75 @@ func newMetrics() metrics {
Name: "send_chunk_error", Name: "send_chunk_error",
Help: "Total no of time error received while sending chunk.", Help: "Total no of time error received while sending chunk.",
}), }),
MarkAndSweepTimer: prometheus.NewCounter(prometheus.CounterOpts{ ReceivedChunkErrorCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "received_chunk_error",
Help: "Total no of time error received while receiving chunk.",
}),
ReceiptsReceivedCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "received_receipts",
Help: "Total no of times receipts received.",
}),
ReceiptsSentCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "sent_receipts",
Help: "Total no of times receipts are sent.",
}),
SendReceiptErrorCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace, Namespace: m.Namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "mark_and_sweep_time", Name: "sent_receipts_error",
Help: "Total time spent in mark and sweep.", Help: "Total no of times receipts were sent and error was encountered.",
}),
ReceiveReceiptErrorCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "receive_receipt_error",
Help: "Total no of time error received while receiving receipt.",
}), }),
ChunksInBatch: prometheus.NewGauge(prometheus.GaugeOpts{ ErrorSettingChunkToSynced: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "cannot_set_chunk_sync_in_DB",
Help: "Total no of times the chunk cannot be synced in DB.",
}),
RetriesExhaustedCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "chunk_retries_exhausted",
Help: "CHunk retries exhausted.",
}),
InvalidReceiptReceived: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "invalid_receipt_receipt",
Help: "Invalid receipt received from peer.",
}),
SendChunkTimer: prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "send_chunk_time_histogram",
Help: "Histogram for Time taken to send a chunk.",
Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60},
}),
MarkAndSweepTimer: prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "mark_and_sweep_time_histogram",
Help: "Histogram of time spent in mark and sweep.",
Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60},
}),
ReceiptRTT: prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: m.Namespace, Namespace: m.Namespace,
Subsystem: subsystem, Subsystem: subsystem,
Name: "chunks_in_batch", Name: "receipt_rtt_histogram",
Help: "Chunks in batch at a given time.", Help: "Histogram of RTT for receiving receipt for a pushed chunk.",
Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60},
}), }),
} }
} }
......
...@@ -74,22 +74,68 @@ func (m *Delivery) GetData() []byte { ...@@ -74,22 +74,68 @@ func (m *Delivery) GetData() []byte {
return nil return nil
} }
type Receipt struct {
Address []byte `protobuf:"bytes,1,opt,name=Address,proto3" json:"Address,omitempty"`
}
func (m *Receipt) Reset() { *m = Receipt{} }
func (m *Receipt) String() string { return proto.CompactTextString(m) }
func (*Receipt) ProtoMessage() {}
func (*Receipt) Descriptor() ([]byte, []int) {
return fileDescriptor_723cf31bfc02bfd6, []int{1}
}
func (m *Receipt) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *Receipt) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_Receipt.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *Receipt) XXX_Merge(src proto.Message) {
xxx_messageInfo_Receipt.Merge(m, src)
}
func (m *Receipt) XXX_Size() int {
return m.Size()
}
func (m *Receipt) XXX_DiscardUnknown() {
xxx_messageInfo_Receipt.DiscardUnknown(m)
}
var xxx_messageInfo_Receipt proto.InternalMessageInfo
func (m *Receipt) GetAddress() []byte {
if m != nil {
return m.Address
}
return nil
}
func init() { func init() {
proto.RegisterType((*Delivery)(nil), "pb.Delivery") proto.RegisterType((*Delivery)(nil), "pb.Delivery")
proto.RegisterType((*Receipt)(nil), "pb.Receipt")
} }
func init() { proto.RegisterFile("pushsync.proto", fileDescriptor_723cf31bfc02bfd6) } func init() { proto.RegisterFile("pushsync.proto", fileDescriptor_723cf31bfc02bfd6) }
var fileDescriptor_723cf31bfc02bfd6 = []byte{ var fileDescriptor_723cf31bfc02bfd6 = []byte{
// 122 bytes of a gzipped FileDescriptorProto // 135 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2b, 0x28, 0x2d, 0xce, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2b, 0x28, 0x2d, 0xce,
0x28, 0xae, 0xcc, 0x4b, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x2a, 0x48, 0x52, 0xb2, 0x28, 0xae, 0xcc, 0x4b, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x2a, 0x48, 0x52, 0xb2,
0xe0, 0xe2, 0x70, 0x49, 0xcd, 0xc9, 0x2c, 0x4b, 0x2d, 0xaa, 0x14, 0x92, 0xe0, 0x62, 0x77, 0x4c, 0xe0, 0xe2, 0x70, 0x49, 0xcd, 0xc9, 0x2c, 0x4b, 0x2d, 0xaa, 0x14, 0x92, 0xe0, 0x62, 0x77, 0x4c,
0x49, 0x29, 0x4a, 0x2d, 0x2e, 0x96, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x09, 0x82, 0x71, 0x85, 0x84, 0x49, 0x29, 0x4a, 0x2d, 0x2e, 0x96, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x09, 0x82, 0x71, 0x85, 0x84,
0xb8, 0x58, 0x5c, 0x12, 0x4b, 0x12, 0x25, 0x98, 0xc0, 0xc2, 0x60, 0xb6, 0x93, 0xc4, 0x89, 0x47, 0xb8, 0x58, 0x5c, 0x12, 0x4b, 0x12, 0x25, 0x98, 0xc0, 0xc2, 0x60, 0xb6, 0x92, 0x32, 0x17, 0x7b,
0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, 0xb1, 0x1c, 0xc3, 0x85, 0x50, 0x6a, 0x72, 0x6a, 0x66, 0x41, 0x09, 0x6e, 0x8d, 0x4e, 0x12, 0x27, 0x1e, 0xc9, 0x31, 0x5e,
0xc7, 0x72, 0x0c, 0x37, 0x1e, 0xcb, 0x31, 0x24, 0xb1, 0x81, 0x8d, 0x37, 0x06, 0x04, 0x00, 0x00, 0x78, 0x24, 0xc7, 0xf8, 0xe0, 0x91, 0x1c, 0xe3, 0x84, 0xc7, 0x72, 0x0c, 0x17, 0x1e, 0xcb, 0x31,
0xff, 0xff, 0x79, 0xb1, 0x76, 0x9e, 0x70, 0x00, 0x00, 0x00, 0xdc, 0x78, 0x2c, 0xc7, 0x90, 0xc4, 0x06, 0x76, 0x83, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x9d,
0x84, 0x4d, 0xb6, 0x95, 0x00, 0x00, 0x00,
} }
func (m *Delivery) Marshal() (dAtA []byte, err error) { func (m *Delivery) Marshal() (dAtA []byte, err error) {
...@@ -129,6 +175,36 @@ func (m *Delivery) MarshalToSizedBuffer(dAtA []byte) (int, error) { ...@@ -129,6 +175,36 @@ func (m *Delivery) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil return len(dAtA) - i, nil
} }
func (m *Receipt) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *Receipt) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *Receipt) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if len(m.Address) > 0 {
i -= len(m.Address)
copy(dAtA[i:], m.Address)
i = encodeVarintPushsync(dAtA, i, uint64(len(m.Address)))
i--
dAtA[i] = 0xa
}
return len(dAtA) - i, nil
}
func encodeVarintPushsync(dAtA []byte, offset int, v uint64) int { func encodeVarintPushsync(dAtA []byte, offset int, v uint64) int {
offset -= sovPushsync(v) offset -= sovPushsync(v)
base := offset base := offset
...@@ -157,6 +233,19 @@ func (m *Delivery) Size() (n int) { ...@@ -157,6 +233,19 @@ func (m *Delivery) Size() (n int) {
return n return n
} }
func (m *Receipt) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.Address)
if l > 0 {
n += 1 + l + sovPushsync(uint64(l))
}
return n
}
func sovPushsync(x uint64) (n int) { func sovPushsync(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7 return (math_bits.Len64(x|1) + 6) / 7
} }
...@@ -284,6 +373,93 @@ func (m *Delivery) Unmarshal(dAtA []byte) error { ...@@ -284,6 +373,93 @@ func (m *Delivery) Unmarshal(dAtA []byte) error {
} }
return nil return nil
} }
func (m *Receipt) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPushsync
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Receipt: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Receipt: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Address", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPushsync
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthPushsync
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthPushsync
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Address = append(m.Address[:0], dAtA[iNdEx:postIndex]...)
if m.Address == nil {
m.Address = []byte{}
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPushsync(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPushsync
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPushsync
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipPushsync(dAtA []byte) (n int, err error) { func skipPushsync(dAtA []byte) (n int, err error) {
l := len(dAtA) l := len(dAtA)
iNdEx := 0 iNdEx := 0
......
...@@ -10,3 +10,7 @@ message Delivery { ...@@ -10,3 +10,7 @@ message Delivery {
bytes Address = 1; bytes Address = 1;
bytes Data = 2; bytes Data = 2;
} }
message Receipt {
bytes Address = 1;
}
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment