Add proxy method metrics

go/proxyd/backend.go

@@ -31,7 +31,7 @@ var (
 		Name:      "backend_requests_total",
 		Help:      "Count of backend requests.",
 	}, []string{
-    "name",
+		"backend_name",
 	})
 
 	backendErrorsCtr = promauto.NewCounterVec(prometheus.CounterOpts{
@@ -39,7 +39,7 @@ var (
 		Name:      "backend_errors_total",
 		Help:      "Count of backend errors.",
 	}, []string{
-    "name",
+		"backend_name",
 	})
 
 	backendPermanentErrorsCtr = promauto.NewCounterVec(prometheus.CounterOpts{
@@ -47,7 +47,17 @@ var (
 		Name:      "backend_permanent_errors_total",
 		Help:      "Count of backend errors that mark a backend as offline.",
 	}, []string{
-    "name",
+		"backend_name",
+	})
+
+	backendResponseTimeSummary = promauto.NewSummaryVec(prometheus.SummaryOpts{
+		Namespace:  "proxyd",
+		Name:       "backend_response_time_seconds",
+		Help:       "Summary of backend response times broken down by backend and method name.",
+		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.005, 0.99: 0.001},
+	}, []string{
+		"backend_name",
+		"method_name",
 	})
 )
 
@@ -113,7 +123,7 @@ func NewBackend(name, baseURL string, opts ...BackendOpt) *Backend {
 	return backend
 }
 
-func (b *Backend) Forward(body []byte) (*RPCRes, error) {
+func (b *Backend) Forward(req *RPCReq) (*RPCRes, error) {
 	if time.Now().Unix()-atomic.LoadInt64(&b.lastPermError) < b.unhealthyRetryInterval {
 		return nil, ErrBackendOffline
 	}
@@ -122,7 +132,7 @@ func (b *Backend) Forward(body []byte) (*RPCRes, error) {
 	// <= to account for the first attempt not technically being
 	// a retry
 	for i := 0; i <= b.maxRetries; i++ {
-    resB, err := b.doForward(body)
+		resB, err := b.doForward(req)
 		if err != nil {
 			lastError = err
 			log.Warn("backend request failed, trying again", "err", err, "name", b.Name)
@@ -144,18 +154,26 @@ func (b *Backend) Forward(body []byte) (*RPCRes, error) {
 	return nil, wrapErr(lastError, "permanent error forwarding request")
 }
 
-func (b *Backend) doForward(body []byte) ([]byte, error) {
-  req, err := http.NewRequest("POST", b.baseURL, bytes.NewReader(body))
+func (b *Backend) doForward(rpcReq *RPCReq) ([]byte, error) {
+	body, err := json.Marshal(rpcReq)
+	if err != nil {
+		return nil, wrapErr(err, "error marshaling request in forward")
+	}
+
+	httpReq, err := http.NewRequest("POST", b.baseURL, bytes.NewReader(body))
 	if err != nil {
 		backendErrorsCtr.WithLabelValues(b.Name).Inc()
 		return nil, wrapErr(err, "error creating backend request")
 	}
 
 	if b.authPassword != "" {
-    req.SetBasicAuth(b.authUsername, b.authPassword)
+		httpReq.SetBasicAuth(b.authUsername, b.authPassword)
 	}
 
-  res, err := b.client.Do(req)
+	timer := prometheus.NewTimer(backendResponseTimeSummary.WithLabelValues(b.Name, rpcReq.Method))
+	defer timer.ObserveDuration()
+	defer backendRequestsCtr.WithLabelValues(b.Name).Inc()
+	res, err := b.client.Do(httpReq)
 	if err != nil {
 		backendErrorsCtr.WithLabelValues(b.Name).Inc()
 		return nil, wrapErr(err, "error in backend request")
@@ -173,7 +191,6 @@ func (b *Backend) doForward(body []byte) ([]byte, error) {
 		return nil, wrapErr(err, "error reading response body")
 	}
 
-  backendRequestsCtr.WithLabelValues(b.Name).Inc()
 	return resB, nil
 }
 
@@ -183,10 +200,10 @@ type BackendGroup struct {
 	i        int64
 }
 
-func (b *BackendGroup) Forward(body []byte) (*RPCRes, error) {
+func (b *BackendGroup) Forward(rpcReq *RPCReq) (*RPCRes, error) {
 	var outRes *RPCRes
 	for _, back := range b.backends {
-    res, err := back.Forward(body)
+		res, err := back.Forward(rpcReq)
 		if err == ErrBackendOffline {
 			log.Debug("skipping offline backend", "name", back.Name)
 			continue

go/proxyd/server.go

@@ -22,17 +22,11 @@ var (
 		Help:      "Count of total HTTP requests.",
 	})
 
-	httpRequestDurationHisto = promauto.NewHistogram(prometheus.HistogramOpts{
+	httpRequestDurationSummary = promauto.NewSummary(prometheus.SummaryOpts{
 		Namespace:  "proxyd",
-		Name:      "http_request_duration_histogram_seconds",
-		Help:      "Histogram of HTTP request durations.",
-		Buckets: []float64{
-			0,
-			0.1,
-			0.25,
-			0.75,
-			1,
-		},
+		Name:       "http_request_duration_seconds",
+		Help:       "Summary of HTTP request durations, in seconds.",
+		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.95: 0.005, 0.99: 0.001},
 	})
 
 	rpcRequestsCtr = promauto.NewCounterVec(prometheus.CounterOpts{
@@ -142,7 +136,7 @@ func (s *Server) HandleRPC(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	backendRes, err := group.Forward(body)
+	backendRes, err := group.Forward(req)
 	if err != nil {
 		log.Error("error forwarding RPC request", "group", group.Name, "method", req.Method, "err", err)
 		rpcErrorsCtr.WithLabelValues("-32603").Inc()
@@ -184,6 +178,6 @@ func instrumentedHdlr(h http.Handler) http.HandlerFunc {
 		start := time.Now()
 		h.ServeHTTP(w, r)
 		dur := time.Since(start)
-		httpRequestDurationHisto.Observe(float64(dur) / float64(time.Second))
+		httpRequestDurationSummary.Observe(float64(dur) / float64(time.Second))
 	}
 }