Tracing: Use tracing.InitializeTracerForTest (#75479)

pkg/api/common_test.go

@@ -186,7 +186,7 @@ func getContextHandler(t *testing.T, cfg *setting.Cfg) *contexthandler.ContextHa
 
 	return contexthandler.ProvideService(
 		cfg,
-		tracing.NewFakeTracer(),
+		tracing.InitializeTracerForTest(),
 		featuremgmt.WithFeatures(),
 		&authntest.FakeService{ExpectedIdentity: &authn.Identity{IsAnonymous: true, SessionToken: &usertoken.UserToken{}}},
 	)

pkg/expr/classic/classic_test.go

@@ -599,7 +599,7 @@ func TestConditionsCmd(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			res, err := tt.cmd.Execute(context.Background(), time.Now(), tt.vars, tracing.NewFakeTracer())
+			res, err := tt.cmd.Execute(context.Background(), time.Now(), tt.vars, tracing.InitializeTracerForTest())
 			require.NoError(t, err)
 			require.Equal(t, tt.expected(), res)
 		})

pkg/expr/commands_test.go

@@ -119,7 +119,7 @@ func TestReduceExecute(t *testing.T) {
 				},
 			}
 
-			execute, err := cmd.Execute(context.Background(), time.Now(), vars, tracing.NewFakeTracer())
+			execute, err := cmd.Execute(context.Background(), time.Now(), vars, tracing.InitializeTracerForTest())
 			require.NoError(t, err)
 
 			require.Len(t, execute.Values, len(numbers))
@@ -163,7 +163,7 @@ func TestReduceExecute(t *testing.T) {
 		t.Run("drop all non numbers if mapper is DropNonNumber", func(t *testing.T) {
 			cmd, err := NewReduceCommand(util.GenerateShortUID(), randomReduceFunc(), varToReduce, &mathexp.DropNonNumber{})
 			require.NoError(t, err)
-			execute, err := cmd.Execute(context.Background(), time.Now(), vars, tracing.NewFakeTracer())
+			execute, err := cmd.Execute(context.Background(), time.Now(), vars, tracing.InitializeTracerForTest())
 			require.NoError(t, err)
 			require.Len(t, execute.Values, 2)
 		})
@@ -171,7 +171,7 @@ func TestReduceExecute(t *testing.T) {
 		t.Run("replace all non numbers if mapper is ReplaceNonNumberWithValue", func(t *testing.T) {
 			cmd, err := NewReduceCommand(util.GenerateShortUID(), randomReduceFunc(), varToReduce, &mathexp.ReplaceNonNumberWithValue{Value: 1})
 			require.NoError(t, err)
-			execute, err := cmd.Execute(context.Background(), time.Now(), vars, tracing.NewFakeTracer())
+			execute, err := cmd.Execute(context.Background(), time.Now(), vars, tracing.InitializeTracerForTest())
 			require.NoError(t, err)
 			require.Len(t, execute.Values, len(numbers))
 			for _, value := range execute.Values[1 : len(numbers)-1] {
@@ -194,7 +194,7 @@ func TestReduceExecute(t *testing.T) {
 		}
 		cmd, err := NewReduceCommand(util.GenerateShortUID(), randomReduceFunc(), varToReduce, nil)
 		require.NoError(t, err)
-		results, err := cmd.Execute(context.Background(), time.Now(), vars, tracing.NewFakeTracer())
+		results, err := cmd.Execute(context.Background(), time.Now(), vars, tracing.InitializeTracerForTest())
 		require.NoError(t, err)
 
 		require.Len(t, results.Values, 1)
@@ -253,7 +253,7 @@ func TestResampleCommand_Execute(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			result, err := cmd.Execute(context.Background(), time.Now(), mathexp.Vars{
 				varToReduce: mathexp.Results{Values: mathexp.Values{test.vals}},
-			}, tracing.NewFakeTracer())
+			}, tracing.InitializeTracerForTest())
 			if test.isError {
 				require.Error(t, err)
 			} else {
@@ -268,7 +268,7 @@ func TestResampleCommand_Execute(t *testing.T) {
 	t.Run("should return empty result if input is nil Value", func(t *testing.T) {
 		result, err := cmd.Execute(context.Background(), time.Now(), mathexp.Vars{
 			varToReduce: mathexp.Results{Values: mathexp.Values{nil}},
-		}, tracing.NewFakeTracer())
+		}, tracing.InitializeTracerForTest())
 		require.Empty(t, result.Values)
 		require.NoError(t, err)
 	})

pkg/expr/mathexp/exp_nan_null_val_test.go

@@ -137,7 +137,7 @@ func TestNaN(t *testing.T) {
 				e, err := New(tt.expr)
 				tt.newErrIs(t, err)
 				if e != nil {
-					res, err := e.Execute("", tt.vars, tracing.NewFakeTracer())
+					res, err := e.Execute("", tt.vars, tracing.InitializeTracerForTest())
 					tt.execErrIs(t, err)
 					if diff := cmp.Diff(res, tt.results, options...); diff != "" {
 						assert.FailNow(t, tt.name, diff)
@@ -343,7 +343,7 @@ func TestNullValues(t *testing.T) {
 				e, err := New(tt.expr)
 				tt.newErrIs(t, err)
 				if e != nil {
-					res, err := e.Execute("", tt.vars, tracing.NewFakeTracer())
+					res, err := e.Execute("", tt.vars, tracing.InitializeTracerForTest())
 					tt.execErrIs(t, err)
 					if diff := cmp.Diff(tt.results, res, options...); diff != "" {
 						t.Errorf("Result mismatch (-want +got):\n%s", diff)
@@ -380,7 +380,7 @@ func TestNoData(t *testing.T) {
 				e, err := New(expr)
 				require.NoError(t, err)
 				if e != nil {
-					res, err := e.Execute("", vars, tracing.NewFakeTracer())
+					res, err := e.Execute("", vars, tracing.InitializeTracerForTest())
 					require.NoError(t, err)
 					require.Len(t, res.Values, 1)
 					require.Equal(t, NewNoData(), res.Values[0])
@@ -421,21 +421,21 @@ func TestNoData(t *testing.T) {
 			require.NoError(t, err)
 			if e != nil {
 				t.Run("$A,$B=nodata", func(t *testing.T) {
-					res, err := e.Execute("", makeVars(NewNoData(), NewNoData()), tracing.NewFakeTracer())
+					res, err := e.Execute("", makeVars(NewNoData(), NewNoData()), tracing.InitializeTracerForTest())
 					require.NoError(t, err)
 					require.Len(t, res.Values, 1)
 					require.Equal(t, parse.TypeNoData, res.Values[0].Type())
 				})
 
 				t.Run("$A=nodata, $B=series", func(t *testing.T) {
-					res, err := e.Execute("", makeVars(NewNoData(), series), tracing.NewFakeTracer())
+					res, err := e.Execute("", makeVars(NewNoData(), series), tracing.InitializeTracerForTest())
 					require.NoError(t, err)
 					require.Len(t, res.Values, 1)
 					require.Equal(t, parse.TypeNoData, res.Values[0].Type())
 				})
 
 				t.Run("$A=series, $B=nodata", func(t *testing.T) {
-					res, err := e.Execute("", makeVars(NewNoData(), series), tracing.NewFakeTracer())
+					res, err := e.Execute("", makeVars(NewNoData(), series), tracing.InitializeTracerForTest())
 					require.NoError(t, err)
 					require.Len(t, res.Values, 1)
 					require.Equal(t, parse.TypeNoData, res.Values[0].Type())

pkg/expr/mathexp/exp_scalar_no_test.go

@@ -79,7 +79,7 @@ func TestScalarExpr(t *testing.T) {
 			e, err := New(tt.expr)
 			tt.newErrIs(t, err)
 			if e != nil {
-				res, err := e.Execute("", tt.vars, tracing.NewFakeTracer())
+				res, err := e.Execute("", tt.vars, tracing.InitializeTracerForTest())
 				tt.execErrIs(t, err)
 				tt.resultIs(t, tt.Results, res)
 			}
@@ -132,7 +132,7 @@ func TestNumberExpr(t *testing.T) {
 			e, err := New(tt.expr)
 			tt.newErrIs(t, err)
 			if e != nil {
-				res, err := e.Execute("", tt.vars, tracing.NewFakeTracer())
+				res, err := e.Execute("", tt.vars, tracing.InitializeTracerForTest())
 				tt.execErrIs(t, err)
 				tt.resultIs(t, tt.results, res)
 			}

pkg/expr/mathexp/exp_series_test.go

@@ -159,7 +159,7 @@ func TestSeriesExpr(t *testing.T) {
 			e, err := New(tt.expr)
 			tt.newErrIs(t, err)
 			if e != nil {
-				res, err := e.Execute("", tt.vars, tracing.NewFakeTracer())
+				res, err := e.Execute("", tt.vars, tracing.InitializeTracerForTest())
 				tt.execErrIs(t, err)
 				if diff := cmp.Diff(tt.results, res, data.FrameTestCompareOptions()...); diff != "" {
 					t.Errorf("Result mismatch (-want +got):\n%s", diff)

pkg/expr/mathexp/funcs_test.go

@@ -74,7 +74,7 @@ func TestAbsFunc(t *testing.T) {
 			e, err := New(tt.expr)
 			tt.newErrIs(t, err)
 			if e != nil {
-				res, err := e.Execute("", tt.vars, tracing.NewFakeTracer())
+				res, err := e.Execute("", tt.vars, tracing.InitializeTracerForTest())
 				tt.execErrIs(t, err)
 				tt.resultIs(t, tt.results, res)
 			}
@@ -133,7 +133,7 @@ func TestIsNumberFunc(t *testing.T) {
 			e, err := New(tt.expr)
 			require.NoError(t, err)
 			if e != nil {
-				res, err := e.Execute("", tt.vars, tracing.NewFakeTracer())
+				res, err := e.Execute("", tt.vars, tracing.InitializeTracerForTest())
 				require.NoError(t, err)
 				require.Equal(t, tt.results, res)
 			}

pkg/infra/tracing/test_helper.go

@@ -1,123 +1,30 @@
 package tracing
 
 import (
-	"context"
-	"net/http"
-
 	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/codes"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
 	"go.opentelemetry.io/otel/sdk/trace/tracetest"
-	"go.opentelemetry.io/otel/trace"
 )
 
-func InitializeTracerForTest() Tracer {
+type TracerForTestOption func(tp *tracesdk.TracerProvider)
+
+func WithSpanProcessor(sp tracesdk.SpanProcessor) TracerForTestOption {
+	return TracerForTestOption(func(tp *tracesdk.TracerProvider) {
+		tp.RegisterSpanProcessor(sp)
+	})
+}
+
+func InitializeTracerForTest(opts ...TracerForTestOption) Tracer {
 	exp := tracetest.NewInMemoryExporter()
 	tp, _ := initTracerProvider(exp, "testing", tracesdk.AlwaysSample())
+
+	for _, opt := range opts {
+		opt(tp)
+	}
+
 	otel.SetTracerProvider(tp)
 
 	ots := &Opentelemetry{Propagation: "jaeger,w3c", tracerProvider: tp}
 	_ = ots.initOpentelemetryTracer()
 	return ots
 }
-
-type FakeSpan struct {
-	Name string
-
-	ended       bool
-	Attributes  map[attribute.Key]attribute.Value
-	StatusCode  codes.Code
-	Description string
-	Err         error
-	Events      map[string]EventValue
-}
-
-func newFakeSpan(name string) *FakeSpan {
-	return &FakeSpan{
-		Name:       name,
-		Attributes: map[attribute.Key]attribute.Value{},
-		Events:     map[string]EventValue{},
-	}
-}
-
-func (t *FakeSpan) End() {
-	if t.ended {
-		panic("End already called")
-	}
-	t.ended = true
-}
-
-func (t *FakeSpan) IsEnded() bool {
-	return t.ended
-}
-
-func (t *FakeSpan) SetAttributes(key string, value any, kv attribute.KeyValue) {
-	if t.IsEnded() {
-		panic("span already ended")
-	}
-	t.Attributes[kv.Key] = kv.Value
-}
-
-func (t *FakeSpan) SetName(name string) {
-	if t.IsEnded() {
-		panic("span already ended")
-	}
-	t.Name = name
-}
-
-func (t *FakeSpan) SetStatus(code codes.Code, description string) {
-	if t.IsEnded() {
-		panic("span already ended")
-	}
-	t.StatusCode = code
-	t.Description = description
-}
-
-func (t *FakeSpan) RecordError(err error, options ...trace.EventOption) {
-	if t.IsEnded() {
-		panic("span already ended")
-	}
-	t.Err = err
-}
-
-func (t *FakeSpan) AddEvents(keys []string, values []EventValue) {
-	if t.IsEnded() {
-		panic("span already ended")
-	}
-	if len(keys) != len(values) {
-		panic("different number of keys and values")
-	}
-	for i := 0; i < len(keys); i++ {
-		t.Events[keys[i]] = values[i]
-	}
-}
-
-func (t *FakeSpan) ContextWithSpan(ctx context.Context) context.Context {
-	return ctx
-}
-
-type FakeTracer struct {
-	Spans []*FakeSpan
-}
-
-func (t *FakeTracer) Run(ctx context.Context) error {
-	return nil
-}
-
-func (t *FakeTracer) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
-	span := newFakeSpan(spanName)
-	t.Spans = append(t.Spans, span)
-	return ctx, span
-}
-
-func (t *FakeTracer) Inject(ctx context.Context, header http.Header, span Span) {
-}
-
-func (t *FakeTracer) OtelTracer() trace.Tracer {
-	return nil
-}
-
-func NewFakeTracer() *FakeTracer {
-	return &FakeTracer{Spans: []*FakeSpan{}}
-}

pkg/middleware/auth_test.go

@@ -21,7 +21,7 @@ import (
 )
 
 func setupAuthMiddlewareTest(t *testing.T, identity *authn.Identity, authErr error) *contexthandler.ContextHandler {
-	return contexthandler.ProvideService(setting.NewCfg(), tracing.NewFakeTracer(), featuremgmt.WithFeatures(), &authntest.FakeService{
+	return contexthandler.ProvideService(setting.NewCfg(), tracing.InitializeTracerForTest(), featuremgmt.WithFeatures(), &authntest.FakeService{
 		ExpectedErr:      authErr,
 		ExpectedIdentity: identity,
 	})

pkg/middleware/middleware_test.go

@@ -251,6 +251,6 @@ func middlewareScenario(t *testing.T, desc string, fn scenarioFunc, cbs ...func(
 func getContextHandler(t *testing.T, cfg *setting.Cfg, authnService authn.Service) *contexthandler.ContextHandler {
 	t.Helper()
 
-	tracer := tracing.NewFakeTracer()
+	tracer := tracing.InitializeTracerForTest()
 	return contexthandler.ProvideService(cfg, tracer, featuremgmt.WithFeatures(), authnService)
 }

pkg/services/contexthandler/contexthandler_test.go

@@ -25,7 +25,7 @@ func TestContextHandler(t *testing.T) {
 	t.Run("should set auth error if authentication was unsuccessful", func(t *testing.T) {
 		handler := contexthandler.ProvideService(
 			setting.NewCfg(),
-			tracing.NewFakeTracer(),
+			tracing.InitializeTracerForTest(),
 			featuremgmt.WithFeatures(),
 			&authntest.FakeService{ExpectedErr: errors.New("some error")},
 		)
@@ -46,7 +46,7 @@ func TestContextHandler(t *testing.T) {
 		identity := &authn.Identity{ID: authn.NamespacedID(authn.NamespaceUser, 1), OrgID: 1}
 		handler := contexthandler.ProvideService(
 			setting.NewCfg(),
-			tracing.NewFakeTracer(),
+			tracing.InitializeTracerForTest(),
 			featuremgmt.WithFeatures(),
 			&authntest.FakeService{ExpectedIdentity: identity},
 		)
@@ -67,7 +67,7 @@ func TestContextHandler(t *testing.T) {
 		identity := &authn.Identity{IsAnonymous: true, OrgID: 1}
 		handler := contexthandler.ProvideService(
 			setting.NewCfg(),
-			tracing.NewFakeTracer(),
+			tracing.InitializeTracerForTest(),
 			featuremgmt.WithFeatures(),
 			&authntest.FakeService{ExpectedIdentity: identity},
 		)
@@ -88,7 +88,7 @@ func TestContextHandler(t *testing.T) {
 		identity := &authn.Identity{OrgID: 1, AuthenticatedBy: login.RenderModule}
 		handler := contexthandler.ProvideService(
 			setting.NewCfg(),
-			tracing.NewFakeTracer(),
+			tracing.InitializeTracerForTest(),
 			featuremgmt.WithFeatures(),
 			&authntest.FakeService{ExpectedIdentity: identity},
 		)
@@ -109,7 +109,7 @@ func TestContextHandler(t *testing.T) {
 	t.Run("should delete session cookie on invalid session", func(t *testing.T) {
 		handler := contexthandler.ProvideService(
 			setting.NewCfg(),
-			tracing.NewFakeTracer(),
+			tracing.InitializeTracerForTest(),
 			featuremgmt.WithFeatures(),
 			&authntest.FakeService{ExpectedErr: auth.ErrInvalidSessionToken},
 		)
@@ -129,7 +129,7 @@ func TestContextHandler(t *testing.T) {
 	t.Run("should delete session cookie when oauth token refresh failed", func(t *testing.T) {
 		handler := contexthandler.ProvideService(
 			setting.NewCfg(),
-			tracing.NewFakeTracer(),
+			tracing.InitializeTracerForTest(),
 			featuremgmt.WithFeatures(),
 			&authntest.FakeService{ExpectedErr: authn.ErrExpiredAccessToken.Errorf("")},
 		)
@@ -158,7 +158,7 @@ func TestContextHandler(t *testing.T) {
 
 		handler := contexthandler.ProvideService(
 			cfg,
-			tracing.NewFakeTracer(),
+			tracing.InitializeTracerForTest(),
 			featuremgmt.WithFeatures(),
 			&authntest.FakeService{ExpectedIdentity: &authn.Identity{}},
 		)

pkg/services/pluginsintegration/clientmiddleware/tracing_middleware_test.go

@@ -11,6 +11,9 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
 
 	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -101,7 +104,8 @@ func TestTracingMiddleware(t *testing.T) {
 	} {
 		t.Run("Creates spans on "+tc.name, func(t *testing.T) {
 			t.Run("successful", func(t *testing.T) {
-				tracer := tracing.NewFakeTracer()
+				spanRecorder := tracetest.NewSpanRecorder()
+				tracer := tracing.InitializeTracerForTest(tracing.WithSpanProcessor(spanRecorder))
 
 				cdt := clienttest.NewClientDecoratorTest(
 					t,
@@ -110,16 +114,17 @@ func TestTracingMiddleware(t *testing.T) {
 
 				err := tc.run(pluginCtx, cdt)
 				require.NoError(t, err)
-				require.Len(t, tracer.Spans, 1, "must have 1 span")
+				spans := spanRecorder.Ended()
-				span := tracer.Spans[0]
+				require.Len(t, spans, 1, "must have 1 span")
-				assert.True(t, span.IsEnded(), "span should be ended")
+				span := spans[0]
-				assert.NoError(t, span.Err, "span should not have an error")
+				assert.Empty(t, span.Events(), "span should not have an error")
-				assert.Equal(t, codes.Unset, span.StatusCode, "span should not have a status code")
+				assert.Equal(t, codes.Unset, span.Status().Code, "span should not have a status code")
-				assert.Equal(t, tc.expSpanName, span.Name)
+				assert.Equal(t, tc.expSpanName, span.Name())
 			})
 
 			t.Run("error", func(t *testing.T) {
-				tracer := tracing.NewFakeTracer()
+				spanRecorder := tracetest.NewSpanRecorder()
+				tracer := tracing.InitializeTracerForTest(tracing.WithSpanProcessor(spanRecorder))
 
 				cdt := clienttest.NewClientDecoratorTest(
 					t,
@@ -131,17 +136,19 @@ func TestTracingMiddleware(t *testing.T) {
 
 				err := tc.run(pluginCtx, cdt)
 				require.Error(t, err)
-				require.Len(t, tracer.Spans, 1, "must have 1 span")
+				spans := spanRecorder.Ended()
-				span := tracer.Spans[0]
+				require.Len(t, spans, 1, "must have 1 span")
-				assert.True(t, span.IsEnded(), "span should be ended")
+				span := spans[0]
-				assert.Error(t, span.Err, "span should contain an error")
+				require.Len(t, span.Events(), 1, "span should contain an error")
-				assert.Equal(t, codes.Error, span.StatusCode, "span code should be error")
+				require.Equal(t, semconv.ExceptionEventName, span.Events()[0].Name)
+				require.Equal(t, codes.Error, span.Status().Code, "span code should be error")
 			})
 
 			t.Run("panic", func(t *testing.T) {
 				var didPanic bool
 
-				tracer := tracing.NewFakeTracer()
+				spanRecorder := tracetest.NewSpanRecorder()
+				tracer := tracing.InitializeTracerForTest(tracing.WithSpanProcessor(spanRecorder))
 
 				cdt := clienttest.NewClientDecoratorTest(
 					t,
@@ -163,9 +170,7 @@ func TestTracingMiddleware(t *testing.T) {
 				}()
 
 				require.True(t, didPanic, "should have panicked")
-				require.Len(t, tracer.Spans, 1, "must have 1 span")
+				require.Len(t, spanRecorder.Ended(), 1, "must have 1 span")
-				span := tracer.Spans[0]
-				assert.True(t, span.IsEnded(), "span should be ended")
 			})
 		})
 	}
@@ -180,19 +185,18 @@ func TestTracingMiddlewareAttributes(t *testing.T) {
 	for _, tc := range []struct {
 		name       string
 		requestMut []func(ctx *context.Context, req *backend.QueryDataRequest)
-		assert     func(t *testing.T, span *tracing.FakeSpan)
+		assert     func(t *testing.T, span trace.ReadOnlySpan)
 	}{
 		{
 			name: "default",
 			requestMut: []func(ctx *context.Context, req *backend.QueryDataRequest){
 				defaultPluginContextRequestMut,
 			},
-			assert: func(t *testing.T, span *tracing.FakeSpan) {
+			assert: func(t *testing.T, span trace.ReadOnlySpan) {
-				assert.Len(t, span.Attributes, 2, "should have correct number of span attributes")
+				attribs := span.Attributes()
-				assert.Equal(t, "my_plugin_id", span.Attributes["plugin_id"].AsString(), "should have correct plugin_id")
+				require.Len(t, attribs, 2, "should have correct number of span attributes")
-				assert.Equal(t, int64(1337), span.Attributes["org_id"].AsInt64(), "should have correct org_id")
+				require.True(t, spanAttributesContains(attribs, attribute.String("plugin_id", "my_plugin_id")))
-				_, ok := span.Attributes["user"]
+				require.True(t, spanAttributesContains(attribs, attribute.Int("org_id", 1337)))
-				assert.False(t, ok, "should not have user attribute")
 			},
 		},
 		{
@@ -203,22 +207,22 @@ func TestTracingMiddlewareAttributes(t *testing.T) {
 					req.PluginContext.User = &backend.User{Login: "admin"}
 				},
 			},
-			assert: func(t *testing.T, span *tracing.FakeSpan) {
+			assert: func(t *testing.T, span trace.ReadOnlySpan) {
-				assert.Len(t, span.Attributes, 3, "should have correct number of span attributes")
+				attribs := span.Attributes()
-				assert.Equal(t, "my_plugin_id", span.Attributes["plugin_id"].AsString(), "should have correct plugin_id")
+				assert.Len(t, attribs, 3, "should have correct number of span attributes")
-				assert.Equal(t, int64(1337), span.Attributes["org_id"].AsInt64(), "should have correct org_id")
+				require.True(t, spanAttributesContains(attribs, attribute.String("plugin_id", "my_plugin_id")))
-				assert.Equal(t, "admin", span.Attributes["user"].AsString(), "should have correct user attribute")
+				require.True(t, spanAttributesContains(attribs, attribute.Int("org_id", 1337)))
+				require.True(t, spanAttributesContains(attribs, attribute.String("user", "admin")))
 			},
 		},
 		{
 			name:       "empty retains zero values",
 			requestMut: []func(ctx *context.Context, req *backend.QueryDataRequest){},
-			assert: func(t *testing.T, span *tracing.FakeSpan) {
+			assert: func(t *testing.T, span trace.ReadOnlySpan) {
-				assert.Len(t, span.Attributes, 2, "should have correct number of span attributes")
+				attribs := span.Attributes()
-				assert.Zero(t, span.Attributes["plugin_id"].AsString(), "should have correct plugin_id")
+				require.Len(t, attribs, 2, "should have correct number of span attributes")
-				assert.Zero(t, span.Attributes["org_id"].AsInt64(), "should have correct org_id")
+				require.True(t, spanAttributesContains(attribs, attribute.String("plugin_id", "")))
-				_, ok := span.Attributes["user"]
+				require.True(t, spanAttributesContains(attribs, attribute.Int("org_id", 0)))
-				assert.False(t, ok, "should not have user attribute")
 			},
 		},
 		{
@@ -228,9 +232,10 @@ func TestTracingMiddlewareAttributes(t *testing.T) {
 					*ctx = ctxkey.Set(*ctx, &contextmodel.ReqContext{Context: &web.Context{Req: &http.Request{Header: nil}}})
 				},
 			},
-			assert: func(t *testing.T, span *tracing.FakeSpan) {
+			assert: func(t *testing.T, span trace.ReadOnlySpan) {
-				assert.Empty(t, span.Attributes["panel_id"])
+				attribs := span.Attributes()
-				assert.Empty(t, span.Attributes["dashboard_id"])
+				require.True(t, spanAttributesContains(attribs, attribute.String("plugin_id", "")))
+				require.True(t, spanAttributesContains(attribs, attribute.Int("org_id", 0)))
 			},
 		},
 		{
@@ -244,14 +249,13 @@ func TestTracingMiddlewareAttributes(t *testing.T) {
 					}
 				},
 			},
-			assert: func(t *testing.T, span *tracing.FakeSpan) {
+			assert: func(t *testing.T, span trace.ReadOnlySpan) {
-				require.Len(t, span.Attributes, 4)
+				attribs := span.Attributes()
-				for _, k := range []string{"plugin_id", "org_id"} {
+				require.Len(t, attribs, 4)
-					_, ok := span.Attributes[attribute.Key(k)]
+				require.True(t, spanAttributesContains(attribs, attribute.String("plugin_id", "")))
-					assert.True(t, ok)
+				require.True(t, spanAttributesContains(attribs, attribute.Int("org_id", 0)))
-				}
+				require.True(t, spanAttributesContains(attribs, attribute.String("datasource_uid", "uid")))
-				assert.Equal(t, "uid", span.Attributes["datasource_uid"].AsString())
+				require.True(t, spanAttributesContains(attribs, attribute.String("datasource_name", "name")))
-				assert.Equal(t, "name", span.Attributes["datasource_name"].AsString())
 			},
 		},
 		{
@@ -268,15 +272,14 @@ func TestTracingMiddlewareAttributes(t *testing.T) {
 					}))
 				},
 			},
-			assert: func(t *testing.T, span *tracing.FakeSpan) {
+			assert: func(t *testing.T, span trace.ReadOnlySpan) {
-				require.Len(t, span.Attributes, 5)
+				attribs := span.Attributes()
-				for _, k := range []string{"plugin_id", "org_id"} {
+				require.Len(t, attribs, 5)
-					_, ok := span.Attributes[attribute.Key(k)]
+				require.True(t, spanAttributesContains(attribs, attribute.String("plugin_id", "")))
-					assert.True(t, ok)
+				require.True(t, spanAttributesContains(attribs, attribute.Int("org_id", 0)))
-				}
+				require.True(t, spanAttributesContains(attribs, attribute.Int("panel_id", 10)))
-				assert.Equal(t, int64(10), span.Attributes["panel_id"].AsInt64())
+				require.True(t, spanAttributesContains(attribs, attribute.String("query_group_id", "query group id")))
-				assert.Equal(t, "dashboard uid", span.Attributes["dashboard_uid"].AsString())
+				require.True(t, spanAttributesContains(attribs, attribute.String("dashboard_uid", "dashboard uid")))
-				assert.Equal(t, "query group id", span.Attributes["query_group_id"].AsString())
 			},
 		},
 		{
@@ -291,12 +294,11 @@ func TestTracingMiddlewareAttributes(t *testing.T) {
 					}))
 				},
 			},
-			assert: func(t *testing.T, span *tracing.FakeSpan) {
+			assert: func(t *testing.T, span trace.ReadOnlySpan) {
-				require.Len(t, span.Attributes, 2)
+				attribs := span.Attributes()
-				for _, k := range []string{"plugin_id", "org_id"} {
+				require.Len(t, attribs, 2)
-					_, ok := span.Attributes[attribute.Key(k)]
+				require.True(t, spanAttributesContains(attribs, attribute.String("plugin_id", "")))
-					assert.True(t, ok)
+				require.True(t, spanAttributesContains(attribs, attribute.Int("org_id", 0)))
-				}
 			},
 		},
 	} {
@@ -309,7 +311,8 @@ func TestTracingMiddlewareAttributes(t *testing.T) {
 				mut(&ctx, req)
 			}
 
-			tracer := tracing.NewFakeTracer()
+			spanRecorder := tracetest.NewSpanRecorder()
+			tracer := tracing.InitializeTracerForTest(tracing.WithSpanProcessor(spanRecorder))
 
 			cdt := clienttest.NewClientDecoratorTest(
 				t,
@@ -318,11 +321,11 @@ func TestTracingMiddlewareAttributes(t *testing.T) {
 
 			_, err := cdt.Decorator.QueryData(ctx, req)
 			require.NoError(t, err)
-			require.Len(t, tracer.Spans, 1, "must have 1 span")
+			spans := spanRecorder.Ended()
-			span := tracer.Spans[0]
+			require.Len(t, spans, 1, "must have 1 span")
-			assert.True(t, span.IsEnded(), "span should be ended")
+			span := spans[0]
-			assert.NoError(t, span.Err, "span should not have an error")
+			assert.Len(t, span.Events(), 0, "span should not have an error")
-			assert.Equal(t, codes.Unset, span.StatusCode, "span should not have a status code")
+			assert.Equal(t, codes.Unset, span.Status().Code, "span should not have a status code")
 
 			if tc.assert != nil {
 				tc.assert(t, span)
@@ -331,6 +334,16 @@ func TestTracingMiddlewareAttributes(t *testing.T) {
 	}
 }
 
+func spanAttributesContains(attribs []attribute.KeyValue, attrib attribute.KeyValue) bool {
+	for _, v := range attribs {
+		if v.Key == attrib.Key && v.Value == attrib.Value {
+			return true
+		}
+	}
+
+	return false
+}
+
 func newReqContextWithRequest(req *http.Request) *contextmodel.ReqContext {
 	return &contextmodel.ReqContext{
 		Context: &web.Context{

pkg/tsdb/elasticsearch/client/client_test.go

@@ -68,7 +68,7 @@ func TestClient_ExecuteMultisearch(t *testing.T) {
 			To:   to,
 		}
 
-		c, err := NewClient(context.Background(), &ds, timeRange, log.New("test", "test"), tracing.NewFakeTracer())
+		c, err := NewClient(context.Background(), &ds, timeRange, log.New("test", "test"), tracing.InitializeTracerForTest())
 		require.NoError(t, err)
 		require.NotNil(t, c)
 
@@ -190,7 +190,7 @@ func TestClient_Index(t *testing.T) {
 				To:   to,
 			}
 
-			c, err := NewClient(context.Background(), &ds, timeRange, log.New("test", "test"), tracing.NewFakeTracer())
+			c, err := NewClient(context.Background(), &ds, timeRange, log.New("test", "test"), tracing.InitializeTracerForTest())
 			require.NoError(t, err)
 			require.NotNil(t, c)
 

pkg/tsdb/elasticsearch/data_query_test.go

@@ -1819,6 +1819,6 @@ func executeElasticsearchDataQuery(c es.Client, body string, from, to time.Time)
 			},
 		},
 	}
-	query := newElasticsearchDataQuery(context.Background(), c, dataRequest.Queries, log.New("test.logger"), tracing.NewFakeTracer())
+	query := newElasticsearchDataQuery(context.Background(), c, dataRequest.Queries, log.New("test.logger"), tracing.InitializeTracerForTest())
 	return query.execute()
 }

pkg/tsdb/elasticsearch/querydata_test.go

@@ -139,7 +139,7 @@ func queryDataTestWithResponseCode(queriesBytes []byte, responseStatusCode int,
 		return nil
 	})
 
-	result, err := queryData(context.Background(), queries, dsInfo, log.New("test.logger"), tracing.NewFakeTracer())
+	result, err := queryData(context.Background(), queries, dsInfo, log.New("test.logger"), tracing.InitializeTracerForTest())
 	if err != nil {
 		return queryDataTestResult{}, err
 	}

pkg/tsdb/elasticsearch/response_parser_test.go

@@ -3534,7 +3534,7 @@ func parseTestResponse(tsdbQueries map[string]string, responseBody string) (*bac
 		return nil, err
 	}
 
-	return parseResponse(context.Background(), response.Responses, queries, configuredFields, log.New("test.logger"), tracing.NewFakeTracer())
+	return parseResponse(context.Background(), response.Responses, queries, configuredFields, log.New("test.logger"), tracing.InitializeTracerForTest())
 }
 
 func requireTimeValue(t *testing.T, expected int64, frame *data.Frame, index int) {

pkg/tsdb/loki/api_mock.go

@@ -65,7 +65,7 @@ func makeMockedAPIWithUrl(url string, statusCode int, contentType string, respon
 		Transport: &mockedRoundTripper{statusCode: statusCode, contentType: contentType, responseBytes: responseBytes, requestCallback: requestCallback},
 	}
 
-	return newLokiAPI(&client, url, log.New("test"), tracing.NewFakeTracer())
+	return newLokiAPI(&client, url, log.New("test"), tracing.InitializeTracerForTest())
 }
 
 func makeCompressedMockedAPIWithUrl(url string, statusCode int, contentType string, responseBytes []byte, requestCallback mockRequestCallback) *LokiAPI {
@@ -73,5 +73,5 @@ func makeCompressedMockedAPIWithUrl(url string, statusCode int, contentType stri
 		Transport: &mockedCompressedRoundTripper{statusCode: statusCode, contentType: contentType, responseBytes: responseBytes, requestCallback: requestCallback},
 	}
 
-	return newLokiAPI(&client, url, log.New("test"), tracing.NewFakeTracer())
+	return newLokiAPI(&client, url, log.New("test"), tracing.InitializeTracerForTest())
 }

pkg/tsdb/loki/healthcheck_test.go

@@ -87,7 +87,7 @@ func Test_healthcheck(t *testing.T) {
 		s := &Service{
 			im:       datasource.NewInstanceManager(newInstanceSettings(httpProvider)),
 			features: featuremgmt.WithFeatures(featuremgmt.FlagLokiLogsDataplane, featuremgmt.FlagLokiMetricDataplane),
-			tracer:   tracing.NewFakeTracer(),
+			tracer:   tracing.InitializeTracerForTest(),
 			logger:   log.New("loki test"),
 		}
 
@@ -106,7 +106,7 @@ func Test_healthcheck(t *testing.T) {
 		s := &Service{
 			im:       datasource.NewInstanceManager(newInstanceSettings(httpProvider)),
 			features: featuremgmt.WithFeatures(featuremgmt.FlagLokiLogsDataplane, featuremgmt.FlagLokiMetricDataplane),
-			tracer:   tracing.NewFakeTracer(),
+			tracer:   tracing.InitializeTracerForTest(),
 			logger:   log.New("loki test"),
 		}