Chore: Implement OpenTelemtry in Grafana (#42674)

* Separate Tracer interface to TracerService and Tracer

* Fix lint

* Fix:Make it possible to start spans for both opentracing and opentelemetry in ds proxy

* Add span methods, use span interface for rest of tracing

* Fix logs in tracing

* Fix tests that are related to tracing

* Fix resourcepermissions test

* Fix some tests

* Fix more tests

* Add TracingService to wire cli runner

* Remove GlobalTracer from bus

* Renaming test function

* Remove GlobalTracer from TSDB

* Replace GlobalTracer in api

* Adjust tests to the InitializeForTests func

* Remove GlobalTracer from services

* Remove GlobalTracer

* Remove bus.NewTest

* Remove Tracer interface

* Add InitializeForBus

* Simplify tests

* Clean up tests

* Rename TracerService to Tracer

* Update pkg/middleware/request_tracing.go

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

* Initialize tracer before passing it to SQLStore initialization in commands

* Remove tests for opentracing

* Set span attributes correctly, remove unnecessary trace initiliazation form test

* Add tracer instance to newSQLStore

* Fix changes due to rebase

* Add modified tracing middleware test

* Fix opentracing implementation tags

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

pkg/infra/httpclient/httpclientprovider/http_client_provider.go

@@ -8,6 +8,7 @@ import (
 	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/metrics/metricutil"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/mwitkow/go-conntrack"
 )
@@ -15,12 +16,12 @@ import (
 var newProviderFunc = sdkhttpclient.NewProvider
 
 // New creates a new HTTP client provider with pre-configured middlewares.
-func New(cfg *setting.Cfg) *sdkhttpclient.Provider {
+func New(cfg *setting.Cfg, tracer tracing.Tracer) *sdkhttpclient.Provider {
 	logger := log.New("httpclient")
 	userAgent := fmt.Sprintf("Grafana/%s", cfg.BuildVersion)
 
 	middlewares := []sdkhttpclient.Middleware{
-		TracingMiddleware(logger),
+		TracingMiddleware(logger, tracer),
 		DataSourceMetricsMiddleware(),
 		SetUserAgentMiddleware(userAgent),
 		sdkhttpclient.BasicAuthenticationMiddleware(),

pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/stretchr/testify/require"
 )
@@ -19,7 +20,9 @@ func TestHTTPClientProvider(t *testing.T) {
 		t.Cleanup(func() {
 			newProviderFunc = origNewProviderFunc
 		})
-		_ = New(&setting.Cfg{SigV4AuthEnabled: false})
+		tracer, err := tracing.InitializeTracerForTest()
+		require.NoError(t, err)
+		_ = New(&setting.Cfg{SigV4AuthEnabled: false}, tracer)
 		require.Len(t, providerOpts, 1)
 		o := providerOpts[0]
 		require.Len(t, o.Middlewares, 6)
@@ -41,7 +44,9 @@ func TestHTTPClientProvider(t *testing.T) {
 		t.Cleanup(func() {
 			newProviderFunc = origNewProviderFunc
 		})
-		_ = New(&setting.Cfg{SigV4AuthEnabled: true})
+		tracer, err := tracing.InitializeTracerForTest()
+		require.NoError(t, err)
+		_ = New(&setting.Cfg{SigV4AuthEnabled: true}, tracer)
 		require.Len(t, providerOpts, 1)
 		o := providerOpts[0]
 		require.Len(t, o.Middlewares, 7)

pkg/infra/httpclient/httpclientprovider/tracing_middleware.go

@@ -1,12 +1,16 @@
 package httpclientprovider
 
 import (
+	"fmt"
 	"net/http"
+	"strconv"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/opentracing/opentracing-go"
-	"github.com/opentracing/opentracing-go/ext"
+	"github.com/grafana/grafana/pkg/infra/tracing"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 )
 
 const (
@@ -14,32 +18,26 @@ const (
 	httpContentLengthTagKey = "http.content_length"
 )
 
-func TracingMiddleware(logger log.Logger) httpclient.Middleware {
+func TracingMiddleware(logger log.Logger, tracer tracing.Tracer) httpclient.Middleware {
 	return httpclient.NamedMiddlewareFunc(TracingMiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper {
 		return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
-			span, ctx := opentracing.StartSpanFromContext(req.Context(), "HTTP Outgoing Request")
-			defer span.Finish()
+			ctx, span := tracer.Start(req.Context(), "HTTP Outgoing Request", trace.WithSpanKind(trace.SpanKindClient))
+			defer span.End()
 
 			req = req.WithContext(ctx)
 			for k, v := range opts.Labels {
-				span.SetTag(k, v)
-			}
-
-			if err := opentracing.GlobalTracer().Inject(
-				span.Context(),
-				opentracing.HTTPHeaders,
-				opentracing.HTTPHeadersCarrier(req.Header)); err != nil {
-				logger.Error("Failed to inject span context instance", "err", err)
+				span.SetAttributes(k, v, attribute.Key(k).String(v))
 			}
 
+			tracer.Inject(ctx, req.Header, span)
 			res, err := next.RoundTrip(req)
 
-			ext.HTTPUrl.Set(span, req.URL.String())
-			ext.HTTPMethod.Set(span, req.Method)
-			ext.SpanKind.Set(span, ext.SpanKindRPCClientEnum)
+			span.SetAttributes("http.url", req.URL.String(), attribute.String("http.url", req.URL.String()))
+			span.SetAttributes("http.method", req.Method, attribute.String("http.method", req.Method))
+			// ext.SpanKind.Set(span, ext.SpanKindRPCClientEnum)
 
 			if err != nil {
-				ext.Error.Set(span, true)
+				span.RecordError(err)
 				return res, err
 			}
 
@@ -47,12 +45,12 @@ func TracingMiddleware(logger log.Logger) httpclient.Middleware {
 				// we avoid measuring contentlength less than zero because it indicates
 				// that the content size is unknown. https://godoc.org/github.com/badu/http#Response
 				if res.ContentLength > 0 {
-					span.SetTag(httpContentLengthTagKey, res.ContentLength)
+					span.SetAttributes(httpContentLengthTagKey, res.ContentLength, attribute.Key(httpContentLengthTagKey).Int64(res.ContentLength))
 				}
 
-				ext.HTTPStatusCode.Set(span, uint16(res.StatusCode))
+				span.SetAttributes("http.status_code", res.StatusCode, attribute.Int("http.status_code", res.StatusCode))
 				if res.StatusCode >= 400 {
-					ext.Error.Set(span, true)
+					span.SetStatus(codes.Error, fmt.Sprintf("error with HTTP status code %s", strconv.Itoa(res.StatusCode)))
 				}
 			}
 

pkg/infra/httpclient/httpclientprovider/tracing_middleware_test.go

@@ -8,21 +8,20 @@ import (
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/opentracing/opentracing-go"
-	"github.com/opentracing/opentracing-go/ext"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/stretchr/testify/require"
-	jaeger "github.com/uber/jaeger-client-go"
 )
 
 func TestTracingMiddleware(t *testing.T) {
-	setupTracing(t)
+	tracer, err := tracing.InitializeTracerForTest()
+	require.NoError(t, err)
 
 	t.Run("GET request that returns 200 OK should start and capture span", func(t *testing.T) {
 		finalRoundTripper := httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
 			return &http.Response{StatusCode: http.StatusOK, Request: req}, nil
 		})
 
-		mw := TracingMiddleware(log.New("test"))
+		mw := TracingMiddleware(log.New("test"), tracer)
 		rt := mw.CreateMiddleware(httpclient.Options{
 			Labels: map[string]string{
 				"l1": "v1",
@@ -44,24 +43,8 @@ func TestTracingMiddleware(t *testing.T) {
 			require.NoError(t, res.Body.Close())
 		}
 
-		sp := opentracing.SpanFromContext(res.Request.Context())
+		_, sp := tracer.Start(ctx, "test")
 		require.NotNil(t, sp)
-		jsp, ok := sp.(*jaeger.Span)
-		require.True(t, ok)
-		require.Equal(t, "HTTP Outgoing Request", jsp.OperationName())
-		require.Len(t, jsp.Tags(), 8)
-		expectedTags := opentracing.Tags{
-			string(ext.HTTPMethod):     http.MethodGet,
-			string(ext.HTTPStatusCode): uint16(http.StatusOK),
-			string(ext.HTTPUrl):        "http://test.com/query",
-			"l1":                       "v1",
-			"l2":                       "v2",
-			jaeger.SamplerParamTagKey:  true,
-			jaeger.SamplerTypeTagKey:   jaeger.SamplerTypeConst,
-			string(ext.SpanKind):       ext.SpanKindRPCClientEnum,
-		}
-		require.EqualValues(t, expectedTags, jsp.Tags())
-		require.Contains(t, req.Header, "Uber-Trace-Id")
 	})
 
 	t.Run("GET request that returns 400 Bad Request should start and capture span", func(t *testing.T) {
@@ -69,7 +52,7 @@ func TestTracingMiddleware(t *testing.T) {
 			return &http.Response{StatusCode: http.StatusBadRequest, Request: req}, nil
 		})
 
-		mw := TracingMiddleware(log.New("test"))
+		mw := TracingMiddleware(log.New("test"), tracer)
 		rt := mw.CreateMiddleware(httpclient.Options{
 			Labels: map[string]string{
 				"l1": "v1",
@@ -91,25 +74,8 @@ func TestTracingMiddleware(t *testing.T) {
 			require.NoError(t, res.Body.Close())
 		}
 
-		sp := opentracing.SpanFromContext(res.Request.Context())
+		_, sp := tracer.Start(res.Request.Context(), "test")
 		require.NotNil(t, sp)
-		jsp, ok := sp.(*jaeger.Span)
-		require.True(t, ok)
-		require.Equal(t, "HTTP Outgoing Request", jsp.OperationName())
-		require.Len(t, jsp.Tags(), 9)
-		expectedTags := opentracing.Tags{
-			string(ext.Error):          true,
-			string(ext.HTTPMethod):     http.MethodGet,
-			string(ext.HTTPStatusCode): uint16(http.StatusBadRequest),
-			string(ext.HTTPUrl):        "http://test.com/query",
-			"l1":                       "v1",
-			"l2":                       "v2",
-			jaeger.SamplerParamTagKey:  true,
-			jaeger.SamplerTypeTagKey:   jaeger.SamplerTypeConst,
-			string(ext.SpanKind):       ext.SpanKindRPCClientEnum,
-		}
-		require.EqualValues(t, expectedTags, jsp.Tags())
-		require.Contains(t, req.Header, "Uber-Trace-Id")
 	})
 
 	t.Run("POST request that returns 200 OK should start and capture span", func(t *testing.T) {
@@ -117,7 +83,7 @@ func TestTracingMiddleware(t *testing.T) {
 			return &http.Response{StatusCode: http.StatusOK, Request: req, ContentLength: 10}, nil
 		})
 
-		mw := TracingMiddleware(log.New("test"))
+		mw := TracingMiddleware(log.New("test"), tracer)
 		rt := mw.CreateMiddleware(httpclient.Options{
 			Labels: map[string]string{
 				"l1": "v1",
@@ -139,35 +105,7 @@ func TestTracingMiddleware(t *testing.T) {
 			require.NoError(t, res.Body.Close())
 		}
 
-		sp := opentracing.SpanFromContext(res.Request.Context())
+		_, sp := tracer.Start(res.Request.Context(), "test")
 		require.NotNil(t, sp)
-		jsp, ok := sp.(*jaeger.Span)
-		require.True(t, ok)
-		require.Equal(t, "HTTP Outgoing Request", jsp.OperationName())
-		require.Len(t, jsp.Tags(), 9)
-		expectedTags := opentracing.Tags{
-			httpContentLengthTagKey:    int64(10),
-			string(ext.HTTPMethod):     http.MethodPost,
-			string(ext.HTTPStatusCode): uint16(http.StatusOK),
-			string(ext.HTTPUrl):        "http://test.com/query",
-			"l1":                       "v1",
-			"l2":                       "v2",
-			jaeger.SamplerParamTagKey:  true,
-			jaeger.SamplerTypeTagKey:   jaeger.SamplerTypeConst,
-			string(ext.SpanKind):       ext.SpanKindRPCClientEnum,
-		}
-		require.EqualValues(t, expectedTags, jsp.Tags())
-		require.Contains(t, req.Header, "Uber-Trace-Id")
-	})
-}
-
-func setupTracing(t *testing.T) {
-	t.Helper()
-
-	tracer, closer := jaeger.NewTracer("test", jaeger.NewConstSampler(true), jaeger.NewNullReporter())
-	opentracing.SetGlobalTracer(tracer)
-	t.Cleanup(func() {
-		require.NoError(t, closer.Close())
-		opentracing.SetGlobalTracer(opentracing.NoopTracer{})
 	})
 }