[bee #118] Receipts for pushsync (#122)

* pushsync: add receipt forwarding logic
Co-authored-by: acud <12988138+acud@users.noreply.github.com>

pkg/pushsync/metrics.go

@@ -14,29 +14,58 @@ type metrics struct {
 	// to be able to return them by Metrics()
 	// using reflection
 
-	SendChunkCounter      prometheus.Counter
-	SendChunkTimer        prometheus.Counter
-	SendChunkErrorCounter prometheus.Counter
-	MarkAndSweepTimer     prometheus.Counter
-
-	ChunksInBatch prometheus.Gauge
+	TotalChunksToBeSentCounter prometheus.Counter
+	TotalChunksSynced          prometheus.Counter
+	TotalChunksStoredInDB      prometheus.Counter
+	ChunksSentCounter          prometheus.Counter
+	ChunksReceivedCounter      prometheus.Counter
+	SendChunkErrorCounter      prometheus.Counter
+	ReceivedChunkErrorCounter  prometheus.Counter
+	ReceiptsReceivedCounter    prometheus.Counter
+	ReceiptsSentCounter        prometheus.Counter
+	SendReceiptErrorCounter    prometheus.Counter
+	ReceiveReceiptErrorCounter prometheus.Counter
+	ErrorSettingChunkToSynced  prometheus.Counter
+	RetriesExhaustedCounter    prometheus.Counter
+	InvalidReceiptReceived     prometheus.Counter
+	SendChunkTimer             prometheus.Histogram
+	ReceiptRTT                 prometheus.Histogram
+	MarkAndSweepTimer          prometheus.Histogram
 }
 
 func newMetrics() metrics {
 	subsystem := "pushsync"
 
 	return metrics{
-		SendChunkCounter: prometheus.NewCounter(prometheus.CounterOpts{
+		TotalChunksToBeSentCounter: prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: m.Namespace,
 			Subsystem: subsystem,
-			Name:      "send_chunk",
+			Name:      "total_chunk_to_be_sent",
 			Help:      "Total chunks to be sent.",
 		}),
-		SendChunkTimer: prometheus.NewCounter(prometheus.CounterOpts{
+		TotalChunksSynced: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "total_chunk_synced",
+			Help:      "Total chunks synced succesfully with valid receipts.",
+		}),
+		TotalChunksStoredInDB: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "total_chunk_stored_in_DB",
+			Help:      "Total chunks stored succesfully in local store.",
+		}),
+		ChunksSentCounter: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "sent_chunk",
+			Help:      "Total chunks sent.",
+		}),
+		ChunksReceivedCounter: prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: m.Namespace,
 			Subsystem: subsystem,
-			Name:      "send_chunk_time_taken",
-			Help:      "Total time taken to send a chunk.",
+			Name:      "received_chunk",
+			Help:      "Total chunks received.",
 		}),
 		SendChunkErrorCounter: prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: m.Namespace,
@@ -44,18 +73,75 @@ func newMetrics() metrics {
 			Name:      "send_chunk_error",
 			Help:      "Total no of time error received while sending chunk.",
 		}),
-		MarkAndSweepTimer: prometheus.NewCounter(prometheus.CounterOpts{
+		ReceivedChunkErrorCounter: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "received_chunk_error",
+			Help:      "Total no of time error received while receiving chunk.",
+		}),
+		ReceiptsReceivedCounter: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "received_receipts",
+			Help:      "Total no of times receipts received.",
+		}),
+		ReceiptsSentCounter: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "sent_receipts",
+			Help:      "Total no of times receipts are sent.",
+		}),
+		SendReceiptErrorCounter: prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: m.Namespace,
 			Subsystem: subsystem,
-			Name:      "mark_and_sweep_time",
-			Help:      "Total time spent in mark and sweep.",
+			Name:      "sent_receipts_error",
+			Help:      "Total no of times receipts were sent and error was encountered.",
+		}),
+		ReceiveReceiptErrorCounter: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "receive_receipt_error",
+			Help:      "Total no of time error received while receiving receipt.",
 		}),
 
-		ChunksInBatch: prometheus.NewGauge(prometheus.GaugeOpts{
+		ErrorSettingChunkToSynced: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "cannot_set_chunk_sync_in_DB",
+			Help:      "Total no of times the chunk cannot be synced in DB.",
+		}),
+		RetriesExhaustedCounter: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "chunk_retries_exhausted",
+			Help:      "CHunk retries exhausted.",
+		}),
+		InvalidReceiptReceived: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "invalid_receipt_receipt",
+			Help:      "Invalid receipt received from peer.",
+		}),
+		SendChunkTimer: prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "send_chunk_time_histogram",
+			Help:      "Histogram for Time taken to send a chunk.",
+			Buckets:   []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60},
+		}),
+		MarkAndSweepTimer: prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "mark_and_sweep_time_histogram",
+			Help:      "Histogram of time spent in mark and sweep.",
+			Buckets:   []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60},
+		}),
+		ReceiptRTT: prometheus.NewHistogram(prometheus.HistogramOpts{
 			Namespace: m.Namespace,
 			Subsystem: subsystem,
-			Name:      "chunks_in_batch",
-			Help:      "Chunks in batch at a given time.",
+			Name:      "receipt_rtt_histogram",
+			Help:      "Histogram of RTT for receiving receipt for a pushed chunk.",
+			Buckets:   []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60},
 		}),
 	}
 }

pkg/pushsync/pb/pushsync.pb.go

@@ -74,22 +74,68 @@ func (m *Delivery) GetData() []byte {
 	return nil
 }
 
+type Receipt struct {
+	Address []byte `protobuf:"bytes,1,opt,name=Address,proto3" json:"Address,omitempty"`
+}
+
+func (m *Receipt) Reset()         { *m = Receipt{} }
+func (m *Receipt) String() string { return proto.CompactTextString(m) }
+func (*Receipt) ProtoMessage()    {}
+func (*Receipt) Descriptor() ([]byte, []int) {
+	return fileDescriptor_723cf31bfc02bfd6, []int{1}
+}
+func (m *Receipt) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *Receipt) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_Receipt.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *Receipt) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Receipt.Merge(m, src)
+}
+func (m *Receipt) XXX_Size() int {
+	return m.Size()
+}
+func (m *Receipt) XXX_DiscardUnknown() {
+	xxx_messageInfo_Receipt.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Receipt proto.InternalMessageInfo
+
+func (m *Receipt) GetAddress() []byte {
+	if m != nil {
+		return m.Address
+	}
+	return nil
+}
+
 func init() {
 	proto.RegisterType((*Delivery)(nil), "pb.Delivery")
+	proto.RegisterType((*Receipt)(nil), "pb.Receipt")
 }
 
 func init() { proto.RegisterFile("pushsync.proto", fileDescriptor_723cf31bfc02bfd6) }
 
 var fileDescriptor_723cf31bfc02bfd6 = []byte{
-	// 122 bytes of a gzipped FileDescriptorProto
+	// 135 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2b, 0x28, 0x2d, 0xce,
 	0x28, 0xae, 0xcc, 0x4b, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x2a, 0x48, 0x52, 0xb2,
 	0xe0, 0xe2, 0x70, 0x49, 0xcd, 0xc9, 0x2c, 0x4b, 0x2d, 0xaa, 0x14, 0x92, 0xe0, 0x62, 0x77, 0x4c,
 	0x49, 0x29, 0x4a, 0x2d, 0x2e, 0x96, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x09, 0x82, 0x71, 0x85, 0x84,
-	0xb8, 0x58, 0x5c, 0x12, 0x4b, 0x12, 0x25, 0x98, 0xc0, 0xc2, 0x60, 0xb6, 0x93, 0xc4, 0x89, 0x47,
-	0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, 0xb1, 0x1c, 0xc3, 0x85,
-	0xc7, 0x72, 0x0c, 0x37, 0x1e, 0xcb, 0x31, 0x24, 0xb1, 0x81, 0x8d, 0x37, 0x06, 0x04, 0x00, 0x00,
-	0xff, 0xff, 0x79, 0xb1, 0x76, 0x9e, 0x70, 0x00, 0x00, 0x00,
+	0xb8, 0x58, 0x5c, 0x12, 0x4b, 0x12, 0x25, 0x98, 0xc0, 0xc2, 0x60, 0xb6, 0x92, 0x32, 0x17, 0x7b,
+	0x50, 0x6a, 0x72, 0x6a, 0x66, 0x41, 0x09, 0x6e, 0x8d, 0x4e, 0x12, 0x27, 0x1e, 0xc9, 0x31, 0x5e,
+	0x78, 0x24, 0xc7, 0xf8, 0xe0, 0x91, 0x1c, 0xe3, 0x84, 0xc7, 0x72, 0x0c, 0x17, 0x1e, 0xcb, 0x31,
+	0xdc, 0x78, 0x2c, 0xc7, 0x90, 0xc4, 0x06, 0x76, 0x83, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x9d,
+	0x84, 0x4d, 0xb6, 0x95, 0x00, 0x00, 0x00,
 }
 
 func (m *Delivery) Marshal() (dAtA []byte, err error) {
@@ -129,6 +175,36 @@ func (m *Delivery) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+func (m *Receipt) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *Receipt) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *Receipt) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Address) > 0 {
+		i -= len(m.Address)
+		copy(dAtA[i:], m.Address)
+		i = encodeVarintPushsync(dAtA, i, uint64(len(m.Address)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
 func encodeVarintPushsync(dAtA []byte, offset int, v uint64) int {
 	offset -= sovPushsync(v)
 	base := offset
@@ -157,6 +233,19 @@ func (m *Delivery) Size() (n int) {
 	return n
 }
 
+func (m *Receipt) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Address)
+	if l > 0 {
+		n += 1 + l + sovPushsync(uint64(l))
+	}
+	return n
+}
+
 func sovPushsync(x uint64) (n int) {
 	return (math_bits.Len64(x|1) + 6) / 7
 }
@@ -284,6 +373,93 @@ func (m *Delivery) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *Receipt) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPushsync
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: Receipt: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: Receipt: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Address", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPushsync
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				byteLen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthPushsync
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPushsync
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Address = append(m.Address[:0], dAtA[iNdEx:postIndex]...)
+			if m.Address == nil {
+				m.Address = []byte{}
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPushsync(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthPushsync
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthPushsync
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func skipPushsync(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0

pkg/pushsync/pb/pushsync.proto

@@ -10,3 +10,7 @@ message Delivery {
   bytes Address = 1;
   bytes Data = 2;
 }
+
+message Receipt {
+  bytes Address = 1;
+}