HTTP Client: Outgoing tracing middleware (#34466)

Following #33439 this adds support for outgoing tracing middleware 
in HTTP client provider.

Fixes #24004

pkg/infra/httpclient/httpclientprovider/http_client_provider.go

@@ -5,6 +5,7 @@ import (
 
 	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
 	"github.com/grafana/grafana/pkg/infra/httpclient"
+	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
@@ -12,8 +13,10 @@ var newProviderFunc = sdkhttpclient.NewProvider
 
 // New creates a new HTTP client provider with pre-configured middlewares.
 func New(cfg *setting.Cfg) httpclient.Provider {
+	logger := log.New("httpclient")
 	userAgent := fmt.Sprintf("Grafana/%s", cfg.BuildVersion)
 	middlewares := []sdkhttpclient.Middleware{
+		TracingMiddleware(logger),
 		DataSourceMetricsMiddleware(),
 		SetUserAgentMiddleware(userAgent),
 		sdkhttpclient.BasicAuthenticationMiddleware(),

pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go

@@ -22,11 +22,12 @@ func TestHTTPClientProvider(t *testing.T) {
 		_ = New(&setting.Cfg{SigV4AuthEnabled: false})
 		require.Len(t, providerOpts, 1)
 		o := providerOpts[0]
-		require.Len(t, o.Middlewares, 4)
+		require.Len(t, o.Middlewares, 5)
-		require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, TracingMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName())
-		require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName())
-		require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName())
-		require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName())
 	})
 
 	t.Run("When creating new provider and SigV4 is enabled should apply expected middleware", func(t *testing.T) {
@@ -42,11 +43,12 @@ func TestHTTPClientProvider(t *testing.T) {
 		_ = New(&setting.Cfg{SigV4AuthEnabled: true})
 		require.Len(t, providerOpts, 1)
 		o := providerOpts[0]
-		require.Len(t, o.Middlewares, 5)
+		require.Len(t, o.Middlewares, 6)
-		require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, TracingMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName())
-		require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName())
-		require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName())
-		require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName())
-		require.Equal(t, SigV4MiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, SigV4MiddlewareName, o.Middlewares[5].(sdkhttpclient.MiddlewareName).MiddlewareName())
 	})
 }

pkg/infra/httpclient/httpclientprovider/testing.go

@@ -13,6 +13,6 @@ type testContext struct {
 func (c *testContext) createRoundTripper(name string) http.RoundTripper {
 	return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
 		c.callChain = append(c.callChain, name)
-		return &http.Response{StatusCode: http.StatusOK}, nil
+		return &http.Response{StatusCode: http.StatusOK, Request: req}, nil
 	})
 }

pkg/infra/httpclient/httpclientprovider/tracing_middleware.go

@@ -0,0 +1,62 @@
+package httpclientprovider
+
+import (
+	"net/http"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
+)
+
+const (
+	TracingMiddlewareName   = "tracing"
+	httpContentLengthTagKey = "http.content_length"
+)
+
+func TracingMiddleware(logger log.Logger) httpclient.Middleware {
+	return httpclient.NamedMiddlewareFunc(TracingMiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper {
+		return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			span, ctx := opentracing.StartSpanFromContext(req.Context(), "HTTP Outgoing Request")
+			defer span.Finish()
+
+			req = req.WithContext(ctx)
+			for k, v := range opts.Labels {
+				span.SetTag(k, v)
+			}
+
+			if err := opentracing.GlobalTracer().Inject(
+				span.Context(),
+				opentracing.HTTPHeaders,
+				opentracing.HTTPHeadersCarrier(req.Header)); err != nil {
+				logger.Error("Failed to inject span context instance", "err", err)
+			}
+
+			res, err := next.RoundTrip(req)
+
+			ext.HTTPUrl.Set(span, req.URL.String())
+			ext.HTTPMethod.Set(span, req.Method)
+			ext.SpanKind.Set(span, ext.SpanKindRPCClientEnum)
+
+			if err != nil {
+				ext.Error.Set(span, true)
+				return res, err
+			}
+
+			if res != nil {
+				// we avoid measuring contentlength less than zero because it indicates
+				// that the content size is unknown. https://godoc.org/github.com/badu/http#Response
+				if res.ContentLength > 0 {
+					span.SetTag(httpContentLengthTagKey, res.ContentLength)
+				}
+
+				ext.HTTPStatusCode.Set(span, uint16(res.StatusCode))
+				if res.StatusCode >= 400 {
+					ext.Error.Set(span, true)
+				}
+			}
+
+			return res, err
+		})
+	})
+}

pkg/infra/httpclient/httpclientprovider/tracing_middleware_test.go

@@ -0,0 +1,173 @@
+package httpclientprovider
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
+	"github.com/stretchr/testify/require"
+	jaeger "github.com/uber/jaeger-client-go"
+)
+
+func TestTracingMiddleware(t *testing.T) {
+	setupTracing(t)
+
+	t.Run("GET request that returns 200 OK should start and capture span", func(t *testing.T) {
+		finalRoundTripper := httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			return &http.Response{StatusCode: http.StatusOK, Request: req}, nil
+		})
+
+		mw := TracingMiddleware(log.New("test"))
+		rt := mw.CreateMiddleware(httpclient.Options{
+			Labels: map[string]string{
+				"l1": "v1",
+				"l2": "v2",
+			},
+		}, finalRoundTripper)
+		require.NotNil(t, rt)
+		middlewareName, ok := mw.(httpclient.MiddlewareName)
+		require.True(t, ok)
+		require.Equal(t, TracingMiddlewareName, middlewareName.MiddlewareName())
+
+		ctx := context.Background()
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://test.com/query", nil)
+		require.NoError(t, err)
+		res, err := rt.RoundTrip(req)
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		if res.Body != nil {
+			require.NoError(t, res.Body.Close())
+		}
+
+		sp := opentracing.SpanFromContext(res.Request.Context())
+		require.NotNil(t, sp)
+		jsp, ok := sp.(*jaeger.Span)
+		require.True(t, ok)
+		require.Equal(t, "HTTP Outgoing Request", jsp.OperationName())
+		require.Len(t, jsp.Tags(), 8)
+		expectedTags := opentracing.Tags{
+			string(ext.HTTPMethod):     http.MethodGet,
+			string(ext.HTTPStatusCode): uint16(http.StatusOK),
+			string(ext.HTTPUrl):        "http://test.com/query",
+			"l1":                       "v1",
+			"l2":                       "v2",
+			jaeger.SamplerParamTagKey:  true,
+			jaeger.SamplerTypeTagKey:   jaeger.SamplerTypeConst,
+			string(ext.SpanKind):       ext.SpanKindRPCClientEnum,
+		}
+		require.EqualValues(t, expectedTags, jsp.Tags())
+		require.Contains(t, req.Header, "Uber-Trace-Id")
+	})
+
+	t.Run("GET request that returns 400 Bad Request should start and capture span", func(t *testing.T) {
+		finalRoundTripper := httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			return &http.Response{StatusCode: http.StatusBadRequest, Request: req}, nil
+		})
+
+		mw := TracingMiddleware(log.New("test"))
+		rt := mw.CreateMiddleware(httpclient.Options{
+			Labels: map[string]string{
+				"l1": "v1",
+				"l2": "v2",
+			},
+		}, finalRoundTripper)
+		require.NotNil(t, rt)
+		middlewareName, ok := mw.(httpclient.MiddlewareName)
+		require.True(t, ok)
+		require.Equal(t, TracingMiddlewareName, middlewareName.MiddlewareName())
+
+		ctx := context.Background()
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://test.com/query", nil)
+		require.NoError(t, err)
+		res, err := rt.RoundTrip(req)
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		if res.Body != nil {
+			require.NoError(t, res.Body.Close())
+		}
+
+		sp := opentracing.SpanFromContext(res.Request.Context())
+		require.NotNil(t, sp)
+		jsp, ok := sp.(*jaeger.Span)
+		require.True(t, ok)
+		require.Equal(t, "HTTP Outgoing Request", jsp.OperationName())
+		require.Len(t, jsp.Tags(), 9)
+		expectedTags := opentracing.Tags{
+			string(ext.Error):          true,
+			string(ext.HTTPMethod):     http.MethodGet,
+			string(ext.HTTPStatusCode): uint16(http.StatusBadRequest),
+			string(ext.HTTPUrl):        "http://test.com/query",
+			"l1":                       "v1",
+			"l2":                       "v2",
+			jaeger.SamplerParamTagKey:  true,
+			jaeger.SamplerTypeTagKey:   jaeger.SamplerTypeConst,
+			string(ext.SpanKind):       ext.SpanKindRPCClientEnum,
+		}
+		require.EqualValues(t, expectedTags, jsp.Tags())
+		require.Contains(t, req.Header, "Uber-Trace-Id")
+	})
+
+	t.Run("POST request that returns 200 OK should start and capture span", func(t *testing.T) {
+		finalRoundTripper := httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			return &http.Response{StatusCode: http.StatusOK, Request: req, ContentLength: 10}, nil
+		})
+
+		mw := TracingMiddleware(log.New("test"))
+		rt := mw.CreateMiddleware(httpclient.Options{
+			Labels: map[string]string{
+				"l1": "v1",
+				"l2": "v2",
+			},
+		}, finalRoundTripper)
+		require.NotNil(t, rt)
+		middlewareName, ok := mw.(httpclient.MiddlewareName)
+		require.True(t, ok)
+		require.Equal(t, TracingMiddlewareName, middlewareName.MiddlewareName())
+
+		ctx := context.Background()
+		req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://test.com/query", bytes.NewBufferString("{ \"message\": \"ok\"}"))
+		require.NoError(t, err)
+		res, err := rt.RoundTrip(req)
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		if res.Body != nil {
+			require.NoError(t, res.Body.Close())
+		}
+
+		sp := opentracing.SpanFromContext(res.Request.Context())
+		require.NotNil(t, sp)
+		jsp, ok := sp.(*jaeger.Span)
+		require.True(t, ok)
+		require.Equal(t, "HTTP Outgoing Request", jsp.OperationName())
+		require.Len(t, jsp.Tags(), 9)
+		expectedTags := opentracing.Tags{
+			httpContentLengthTagKey:    int64(10),
+			string(ext.HTTPMethod):     http.MethodPost,
+			string(ext.HTTPStatusCode): uint16(http.StatusOK),
+			string(ext.HTTPUrl):        "http://test.com/query",
+			"l1":                       "v1",
+			"l2":                       "v2",
+			jaeger.SamplerParamTagKey:  true,
+			jaeger.SamplerTypeTagKey:   jaeger.SamplerTypeConst,
+			string(ext.SpanKind):       ext.SpanKindRPCClientEnum,
+		}
+		require.EqualValues(t, expectedTags, jsp.Tags())
+		require.Contains(t, req.Header, "Uber-Trace-Id")
+	})
+}
+
+func setupTracing(t *testing.T) {
+	t.Helper()
+
+	tracer, closer := jaeger.NewTracer("test", jaeger.NewConstSampler(true), jaeger.NewNullReporter())
+	opentracing.SetGlobalTracer(tracer)
+	t.Cleanup(func() {
+		require.NoError(t, closer.Close())
+		opentracing.SetGlobalTracer(opentracing.NoopTracer{})
+	})
+}