Skip to content

M
mybee
Commit 5fb8cd84 authored by Zahoor Mohamed's avatar Zahoor Mohamed Committed by GitHub
Browse files
Options

[bee #118] Receipts for pushsync (#122) 

* pushsync: add receipt forwarding logic
Co-authored-by: default avataracud <12988138+acud@users.noreply.github.com>
parent 8e958f02
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 601 additions and 239 deletions
pkg/pushsync/metrics.go
View file @ 5fb8cd84
......@@ -14,29 +14,58 @@ type metrics struct {
// to be able to return them by Metrics()
// using reflection
SendChunkCounter prometheus.Counter
SendChunkTimer prometheus.Counter
SendChunkErrorCounter prometheus.Counter
MarkAndSweepTimer prometheus.Counter
ChunksInBatch prometheus.Gauge
TotalChunksToBeSentCounter prometheus.Counter
TotalChunksSynced prometheus.Counter
TotalChunksStoredInDB prometheus.Counter
ChunksSentCounter prometheus.Counter
ChunksReceivedCounter prometheus.Counter
SendChunkErrorCounter prometheus.Counter
ReceivedChunkErrorCounter prometheus.Counter
ReceiptsReceivedCounter prometheus.Counter
ReceiptsSentCounter prometheus.Counter
SendReceiptErrorCounter prometheus.Counter
ReceiveReceiptErrorCounter prometheus.Counter
ErrorSettingChunkToSynced prometheus.Counter
RetriesExhaustedCounter prometheus.Counter
InvalidReceiptReceived prometheus.Counter
SendChunkTimer prometheus.Histogram
ReceiptRTT prometheus.Histogram
MarkAndSweepTimer prometheus.Histogram
}
func newMetrics() metrics {
subsystem := "pushsync"
return metrics{
SendChunkCounter: prometheus.NewCounter(prometheus.CounterOpts{
TotalChunksToBeSentCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "send_chunk",
Name: "total_chunk_to_be_sent",
Help: "Total chunks to be sent.",
}),
SendChunkTimer: prometheus.NewCounter(prometheus.CounterOpts{
TotalChunksSynced: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "total_chunk_synced",
Help: "Total chunks synced succesfully with valid receipts.",
}),
TotalChunksStoredInDB: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "total_chunk_stored_in_DB",
Help: "Total chunks stored succesfully in local store.",
}),
ChunksSentCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "sent_chunk",
Help: "Total chunks sent.",
}),
ChunksReceivedCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "send_chunk_time_taken",
Help: "Total time taken to send a chunk.",
Name: "received_chunk",
Help: "Total chunks received.",
}),
SendChunkErrorCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
......@@ -44,18 +73,75 @@ func newMetrics() metrics {
Name: "send_chunk_error",
Help: "Total no of time error received while sending chunk.",
}),
MarkAndSweepTimer: prometheus.NewCounter(prometheus.CounterOpts{
ReceivedChunkErrorCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "received_chunk_error",
Help: "Total no of time error received while receiving chunk.",
}),
ReceiptsReceivedCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "received_receipts",
Help: "Total no of times receipts received.",
}),
ReceiptsSentCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "sent_receipts",
Help: "Total no of times receipts are sent.",
}),
SendReceiptErrorCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "mark_and_sweep_time",
Help: "Total time spent in mark and sweep.",
Name: "sent_receipts_error",
Help: "Total no of times receipts were sent and error was encountered.",
}),
ReceiveReceiptErrorCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "receive_receipt_error",
Help: "Total no of time error received while receiving receipt.",
}),
ChunksInBatch: prometheus.NewGauge(prometheus.GaugeOpts{
ErrorSettingChunkToSynced: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "cannot_set_chunk_sync_in_DB",
Help: "Total no of times the chunk cannot be synced in DB.",
}),
RetriesExhaustedCounter: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "chunk_retries_exhausted",
Help: "CHunk retries exhausted.",
}),
InvalidReceiptReceived: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "invalid_receipt_receipt",
Help: "Invalid receipt received from peer.",
}),
SendChunkTimer: prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "send_chunk_time_histogram",
Help: "Histogram for Time taken to send a chunk.",
Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60},
}),
MarkAndSweepTimer: prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "mark_and_sweep_time_histogram",
Help: "Histogram of time spent in mark and sweep.",
Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60},
}),
ReceiptRTT: prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: m.Namespace,
Subsystem: subsystem,
Name: "chunks_in_batch",
Help: "Chunks in batch at a given time.",
Name: "receipt_rtt_histogram",
Help: "Histogram of RTT for receiving receipt for a pushed chunk.",
Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60},
}),
}
}
......
pkg/pushsync/pb/pushsync.pb.go
View file @ 5fb8cd84
......@@ -74,22 +74,68 @@ func (m *Delivery) GetData() []byte {
return nil
}
type Receipt struct {
Address []byte `protobuf:"bytes,1,opt,name=Address,proto3" json:"Address,omitempty"`
}
func (m *Receipt) Reset() { *m = Receipt{} }
func (m *Receipt) String() string { return proto.CompactTextString(m) }
func (*Receipt) ProtoMessage() {}
func (*Receipt) Descriptor() ([]byte, []int) {
return fileDescriptor_723cf31bfc02bfd6, []int{1}
}
func (m *Receipt) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *Receipt) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_Receipt.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *Receipt) XXX_Merge(src proto.Message) {
xxx_messageInfo_Receipt.Merge(m, src)
}
func (m *Receipt) XXX_Size() int {
return m.Size()
}
func (m *Receipt) XXX_DiscardUnknown() {
xxx_messageInfo_Receipt.DiscardUnknown(m)
}
var xxx_messageInfo_Receipt proto.InternalMessageInfo
func (m *Receipt) GetAddress() []byte {
if m != nil {
return m.Address
}
return nil
}
func init() {
proto.RegisterType((*Delivery)(nil), "pb.Delivery")
proto.RegisterType((*Receipt)(nil), "pb.Receipt")
}
func init() { proto.RegisterFile("pushsync.proto", fileDescriptor_723cf31bfc02bfd6) }
var fileDescriptor_723cf31bfc02bfd6 = []byte{
// 122 bytes of a gzipped FileDescriptorProto
// 135 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2b, 0x28, 0x2d, 0xce,
0x28, 0xae, 0xcc, 0x4b, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x2a, 0x48, 0x52, 0xb2,
0xe0, 0xe2, 0x70, 0x49, 0xcd, 0xc9, 0x2c, 0x4b, 0x2d, 0xaa, 0x14, 0x92, 0xe0, 0x62, 0x77, 0x4c,
0x49, 0x29, 0x4a, 0x2d, 0x2e, 0x96, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x09, 0x82, 0x71, 0x85, 0x84,
0xb8, 0x58, 0x5c, 0x12, 0x4b, 0x12, 0x25, 0x98, 0xc0, 0xc2, 0x60, 0xb6, 0x93, 0xc4, 0x89, 0x47,
0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, 0xb1, 0x1c, 0xc3, 0x85,
0xc7, 0x72, 0x0c, 0x37, 0x1e, 0xcb, 0x31, 0x24, 0xb1, 0x81, 0x8d, 0x37, 0x06, 0x04, 0x00, 0x00,
0xff, 0xff, 0x79, 0xb1, 0x76, 0x9e, 0x70, 0x00, 0x00, 0x00,
0xb8, 0x58, 0x5c, 0x12, 0x4b, 0x12, 0x25, 0x98, 0xc0, 0xc2, 0x60, 0xb6, 0x92, 0x32, 0x17, 0x7b,
0x50, 0x6a, 0x72, 0x6a, 0x66, 0x41, 0x09, 0x6e, 0x8d, 0x4e, 0x12, 0x27, 0x1e, 0xc9, 0x31, 0x5e,
0x78, 0x24, 0xc7, 0xf8, 0xe0, 0x91, 0x1c, 0xe3, 0x84, 0xc7, 0x72, 0x0c, 0x17, 0x1e, 0xcb, 0x31,
0xdc, 0x78, 0x2c, 0xc7, 0x90, 0xc4, 0x06, 0x76, 0x83, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x9d,
0x84, 0x4d, 0xb6, 0x95, 0x00, 0x00, 0x00,
}
func (m *Delivery) Marshal() (dAtA []byte, err error) {
......@@ -129,6 +175,36 @@ func (m *Delivery) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil
}
func (m *Receipt) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *Receipt) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *Receipt) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if len(m.Address) > 0 {
i -= len(m.Address)
copy(dAtA[i:], m.Address)
i = encodeVarintPushsync(dAtA, i, uint64(len(m.Address)))
i--
dAtA[i] = 0xa
}
return len(dAtA) - i, nil
}
func encodeVarintPushsync(dAtA []byte, offset int, v uint64) int {
offset -= sovPushsync(v)
base := offset
......@@ -157,6 +233,19 @@ func (m *Delivery) Size() (n int) {
return n
}
func (m *Receipt) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.Address)
if l > 0 {
n += 1 + l + sovPushsync(uint64(l))
}
return n
}
func sovPushsync(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7
}
......@@ -284,6 +373,93 @@ func (m *Delivery) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *Receipt) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPushsync
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Receipt: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Receipt: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Address", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPushsync
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthPushsync
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthPushsync
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Address = append(m.Address[:0], dAtA[iNdEx:postIndex]...)
if m.Address == nil {
m.Address = []byte{}
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPushsync(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPushsync
}
if (iNdEx + skippy) < 0 {
return ErrInvalidLengthPushsync
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipPushsync(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
......
pkg/pushsync/pb/pushsync.proto
View file @ 5fb8cd84
......@@ -10,3 +10,7 @@ message Delivery {
bytes Address = 1;
bytes Data = 2;
}
message Receipt {
bytes Address = 1;
}
pkg/pushsync/pushsync.go
View file @ 5fb8cd84
This diff is collapsed.
pkg/pushsync/pushsync_test.go
View file @ 5fb8cd84
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment