Support tracing in HTTP headers (#1071)

pkg/api/api.go

@@ -189,9 +189,21 @@ func requestEncrypt(r *http.Request) bool {
 func (s *server) newTracingHandler(spanName string) func(h http.Handler) http.Handler {
 	return func(h http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			span, _, ctx := s.Tracer.StartSpanFromContext(r.Context(), spanName, s.Logger)
+			ctx, err := s.Tracer.WithContextFromHTTPHeaders(r.Context(), r.Header)
+			if err != nil && !errors.Is(err, tracing.ErrContextNotFound) {
+				s.Logger.Debugf("span '%s': extract tracing context: %v", spanName, err)
+				// ignore
+			}
+
+			span, _, ctx := s.Tracer.StartSpanFromContext(ctx, spanName, s.Logger)
 			defer span.Finish()
 
+			err = s.Tracer.AddContextHTTPHeader(ctx, r.Header)
+			if err != nil {
+				s.Logger.Debugf("span '%s': inject tracing context: %v", spanName, err)
+				// ignore
+			}
+
 			h.ServeHTTP(w, r.WithContext(ctx))
 		})
 	}

pkg/api/router.go

@@ -9,13 +9,14 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/ethersphere/bee/pkg/jsonhttp"
-	"github.com/ethersphere/bee/pkg/logging"
-	"github.com/ethersphere/bee/pkg/swarm"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
 	"github.com/sirupsen/logrus"
 	"resenje.org/web"
+
+	"github.com/ethersphere/bee/pkg/jsonhttp"
+	"github.com/ethersphere/bee/pkg/logging/httpaccess"
+	"github.com/ethersphere/bee/pkg/swarm"
 )
 
 func (s *server) setupRouting() {
@@ -170,7 +171,7 @@ func (s *server) setupRouting() {
 	)
 
 	s.Handler = web.ChainHandlers(
-		logging.NewHTTPAccessLogHandler(s.Logger, logrus.InfoLevel, "api access"),
+		httpaccess.NewHTTPAccessLogHandler(s.Logger, logrus.InfoLevel, s.Tracer, "api access"),
 		handlers.CompressHandler,
 		// todo: add recovery handler
 		s.pageviewMetricsHandler,

pkg/debugapi/router.go

@@ -9,20 +9,21 @@ import (
 	"net/http"
 	"net/http/pprof"
 
-	"github.com/ethersphere/bee/pkg/jsonhttp"
-	"github.com/ethersphere/bee/pkg/logging"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 	"resenje.org/web"
+
+	"github.com/ethersphere/bee/pkg/jsonhttp"
+	"github.com/ethersphere/bee/pkg/logging/httpaccess"
 )
 
 func (s *server) setupRouting() {
 	baseRouter := http.NewServeMux()
 
 	baseRouter.Handle("/metrics", web.ChainHandlers(
-		logging.SetAccessLogLevelHandler(0), // suppress access log messages
+		httpaccess.SetAccessLogLevelHandler(0), // suppress access log messages
 		web.FinalHandler(promhttp.InstrumentMetricHandler(
 			s.metricsRegistry,
 			promhttp.HandlerFor(s.metricsRegistry, promhttp.HandlerOpts{}),
@@ -46,11 +47,11 @@ func (s *server) setupRouting() {
 	router.Handle("/debug/vars", expvar.Handler())
 
 	router.Handle("/health", web.ChainHandlers(
-		logging.SetAccessLogLevelHandler(0), // suppress access log messages
+		httpaccess.SetAccessLogLevelHandler(0), // suppress access log messages
 		web.FinalHandlerFunc(s.statusHandler),
 	))
 	router.Handle("/readiness", web.ChainHandlers(
-		logging.SetAccessLogLevelHandler(0), // suppress access log messages
+		httpaccess.SetAccessLogLevelHandler(0), // suppress access log messages
 		web.FinalHandlerFunc(s.statusHandler),
 	))
 
@@ -141,7 +142,7 @@ func (s *server) setupRouting() {
 	})
 
 	baseRouter.Handle("/", web.ChainHandlers(
-		logging.NewHTTPAccessLogHandler(s.Logger, logrus.InfoLevel, "debug api access"),
+		httpaccess.NewHTTPAccessLogHandler(s.Logger, logrus.InfoLevel, s.Tracer, "debug api access"),
 		handlers.CompressHandler,
 		// todo: add recovery handler
 		web.NoCacheHeadersHandler,

pkg/logging/http_access.go → pkg/logging/httpaccess/http_access.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package logging
+package httpaccess
 
 import (
 	"bufio"
@@ -11,11 +11,14 @@ import (
 	"time"
 
 	"github.com/sirupsen/logrus"
+
+	"github.com/ethersphere/bee/pkg/logging"
+	"github.com/ethersphere/bee/pkg/tracing"
 )
 
 // NewHTTPAccessLogHandler creates a handler that will log a message after a
 // request has been served.
-func NewHTTPAccessLogHandler(logger Logger, level logrus.Level, message string) func(h http.Handler) http.Handler {
+func NewHTTPAccessLogHandler(logger logging.Logger, level logrus.Level, tracer *tracing.Tracer, message string) func(h http.Handler) http.Handler {
 	return func(h http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			startTime := time.Now()
@@ -27,6 +30,10 @@ func NewHTTPAccessLogHandler(logger Logger, level logrus.Level, message string)
 				return
 			}
 
+			ctx, _ := tracer.WithContextFromHTTPHeaders(r.Context(), r.Header)
+
+			logger := tracing.NewLoggerWithTraceID(ctx, logger)
+
 			status := rl.status
 			if status == 0 {
 				status = http.StatusOK
@@ -56,6 +63,7 @@ func NewHTTPAccessLogHandler(logger Logger, level logrus.Level, message string)
 			if v := r.Header.Get("X-Real-Ip"); v != "" {
 				fields["x-real-ip"] = v
 			}
+
 			logger.WithFields(fields).Log(rl.level, message)
 		})
 	}

pkg/tracing/tracing.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"errors"
 	"io"
+	"net/http"
 	"time"
 
 	"github.com/ethersphere/bee/pkg/logging"
@@ -35,6 +36,14 @@ type contextKey struct{}
 // LogField is the key in log message field that holds tracing id value.
 const LogField = "traceid"
 
+const (
+	// TraceContextHeaderName is the http header name used to propagate tracing context.
+	TraceContextHeaderName = "swarm-trace-id"
+
+	// TraceBaggageHeaderPrefix is the prefix for http headers used to propagate baggage.
+	TraceBaggageHeaderPrefix = "swarmctx-"
+)
+
 // Tracer connect to a tracing server and handles tracing spans and contexts
 // by using opentracing Tracer.
 type Tracer struct {
@@ -67,6 +76,10 @@ func NewTracer(o *Options) (*Tracer, io.Closer, error) {
 			BufferFlushInterval: 1 * time.Second,
 			LocalAgentHostPort:  o.Endpoint,
 		},
+		Headers: &jaeger.HeadersConfig{
+			TraceContextHeaderName:   TraceContextHeaderName,
+			TraceBaggageHeaderPrefix: TraceBaggageHeaderPrefix,
+		},
 	}
 
 	t, closer, err := cfg.NewTracer()
@@ -159,6 +172,62 @@ func (t *Tracer) WithContextFromHeaders(ctx context.Context, headers p2p.Headers
 	return WithContext(ctx, c), nil
 }
 
+// AddContextHTTPHeader adds a tracing span context to provided HTTP headers
+// from the go context. If the tracing span context is not present in
+// go context, ErrContextNotFound is returned.
+func (t *Tracer) AddContextHTTPHeader(ctx context.Context, headers http.Header) error {
+	if t == nil {
+		t = noopTracer
+	}
+
+	c := FromContext(ctx)
+	if c == nil {
+		return ErrContextNotFound
+	}
+
+	carrier := opentracing.HTTPHeadersCarrier(headers)
+	if err := t.tracer.Inject(c, opentracing.HTTPHeaders, carrier); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// FromHTTPHeaders returns tracing span context from HTTP headers. If the tracing
+// span context is not present in go context, ErrContextNotFound is returned.
+func (t *Tracer) FromHTTPHeaders(headers http.Header) (opentracing.SpanContext, error) {
+	if t == nil {
+		t = noopTracer
+	}
+
+	carrier := opentracing.HTTPHeadersCarrier(headers)
+	c, err := t.tracer.Extract(opentracing.HTTPHeaders, carrier)
+	if err != nil {
+		if errors.Is(err, opentracing.ErrSpanContextNotFound) {
+			return nil, ErrContextNotFound
+		}
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// WithContextFromHTTPHeaders returns a new context with injected tracing span
+// context if they are found in HTTP headers. If the tracing span context is not
+// present in go context, ErrContextNotFound is returned.
+func (t *Tracer) WithContextFromHTTPHeaders(ctx context.Context, headers http.Header) (context.Context, error) {
+	if t == nil {
+		t = noopTracer
+	}
+
+	c, err := t.FromHTTPHeaders(headers)
+	if err != nil {
+		return ctx, err
+	}
+
+	return WithContext(ctx, c), nil
+}
+
 // WithContext adds tracing span context to go context.
 func WithContext(ctx context.Context, c opentracing.SpanContext) context.Context {
 	return context.WithValue(ctx, contextKey{}, c)