Tracing: Support multiple OTel propagators (#61199)

* tracing: Support multiple OTel propagators

Signed-off-by: Dave Henderson <dave.henderson@grafana.com>

* tracing: add TraceIDString method to fix up tests

This method will be useful elsewhere if we want to log the trace ID.

* improve propagation docs

Co-authored-by: Christopher Moyer <35463610+chri2547@users.noreply.github.com>

* doc style fix

Signed-off-by: Dave Henderson <dave.henderson@grafana.com>

* Use tracing.TraceIDFromContext instead of adding TraceIDString method

Signed-off-by: Dave Henderson <dave.henderson@grafana.com>

---------

Signed-off-by: Dave Henderson <dave.henderson@grafana.com>
Co-authored-by: Bryan Boreham <bjboreham@gmail.com>
Co-authored-by: Christopher Moyer <35463610+chri2547@users.noreply.github.com>

docs/sources/setup-grafana/configure-grafana/_index.md

@@ -1752,7 +1752,7 @@ The host:port destination for reporting spans. (ex: `localhost:14268/api/traces`
 
 ### propagation
 
-The propagation specifies the text map propagation format.(ex: jaeger, w3c)
+The propagation specifies the text map propagation format. The values `jaeger` and `w3c` are supported. Add a comma (`,`) between values to specify multiple formats (for example, `"jaeger,w3c"`). The default value is `w3c`.
 
 <hr>
 
@@ -1766,7 +1766,7 @@ The host:port destination for reporting spans. (ex: `localhost:4317`)
 
 ### propagation
 
-The propagation specifies the text map propagation format.(ex: jaeger, w3c)
+The propagation specifies the text map propagation format. The values `jaeger` and `w3c` are supported. Add a comma (`,`) between values to specify multiple formats (for example, `"jaeger,w3c"`). The default value is `w3c`.
 
 <hr>
 

pkg/infra/httpclient/httpclientprovider/tracing_middleware_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -46,6 +47,51 @@ func TestTracingMiddleware(t *testing.T) {
 		require.NotNil(t, sp)
 	})
 
+	t.Run("GET request that returns 200 OK should propagate parent span", func(t *testing.T) {
+		expectedTraceID := "<unset>"
+
+		finalRoundTripper := httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			// both Jaeger and w3c headers should be set
+			require.NotEmpty(t, req.Header.Get("Uber-Trace-Id"))
+			require.NotEmpty(t, req.Header.Get("Traceparent"))
+
+			ctx, span := tracer.Start(req.Context(), "inner")
+			defer span.End()
+
+			// child span should have the same trace ID as the parent span
+			require.Equal(t, expectedTraceID, tracing.TraceIDFromContext(ctx, false))
+
+			return &http.Response{StatusCode: http.StatusOK, Request: req}, nil
+		})
+
+		mw := TracingMiddleware(log.New("test"), tracer)
+		rt := mw.CreateMiddleware(httpclient.Options{
+			Labels: map[string]string{
+				"l1": "v1",
+				"l2": "v2",
+			},
+		}, finalRoundTripper)
+		require.NotNil(t, rt)
+		middlewareName, ok := mw.(httpclient.MiddlewareName)
+		require.True(t, ok)
+		require.Equal(t, TracingMiddlewareName, middlewareName.MiddlewareName())
+
+		ctx, span := tracer.Start(context.Background(), "testspan")
+		defer span.End()
+
+		expectedTraceID = tracing.TraceIDFromContext(ctx, false)
+		assert.NotEmpty(t, expectedTraceID)
+
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://test.com/query", nil)
+		require.NoError(t, err)
+		res, err := rt.RoundTrip(req)
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		if res.Body != nil {
+			require.NoError(t, res.Body.Close())
+		}
+	})
+
 	t.Run("GET request that returns 400 Bad Request should start and capture span", func(t *testing.T) {
 		finalRoundTripper := httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
 			return &http.Response{StatusCode: http.StatusBadRequest, Request: req}, nil

pkg/infra/tracing/opentelemetry_tracing.go

@@ -166,13 +166,17 @@ func (ots *Opentelemetry) initOTLPTracerProvider() (*tracesdk.TracerProvider, er
 		return nil, err
 	}
 
+	return initTracerProvider(exp, ots.customAttribs...)
+}
+
+func initTracerProvider(exp tracesdk.SpanExporter, customAttribs ...attribute.KeyValue) (*tracesdk.TracerProvider, error) {
 	res, err := resource.New(
 		context.Background(),
 		resource.WithAttributes(
 			semconv.ServiceNameKey.String("grafana"),
 			semconv.ServiceVersionKey.String(version.Version),
 		),
-		resource.WithAttributes(ots.customAttribs...),
+		resource.WithAttributes(customAttribs...),
 		resource.WithProcessRuntimeDescription(),
 		resource.WithTelemetrySDK(),
 	)
@@ -222,15 +226,34 @@ func (ots *Opentelemetry) initOpentelemetryTracer() error {
 		otel.SetTracerProvider(tp)
 	}
 
-	switch ots.propagation {
-	case w3cPropagator:
-		otel.SetTextMapPropagator(propagation.TraceContext{})
-	case jaegerPropagator:
-		otel.SetTextMapPropagator(jaegerpropagator.Jaeger{})
-	default:
-		otel.SetTextMapPropagator(propagation.TraceContext{})
+	propagators := []propagation.TextMapPropagator{}
+	for _, p := range strings.Split(ots.propagation, ",") {
+		switch p {
+		case w3cPropagator:
+			propagators = append(propagators, propagation.TraceContext{}, propagation.Baggage{})
+		case jaegerPropagator:
+			propagators = append(propagators, jaegerpropagator.Jaeger{})
+		case "":
+		default:
+			return fmt.Errorf("unsupported OpenTelemetry propagator: %q", p)
+		}
 	}
-	ots.tracerProvider = tp
+
+	switch len(propagators) {
+	case 0:
+		otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
+			propagation.TraceContext{}, propagation.Baggage{},
+		))
+	case 1:
+		otel.SetTextMapPropagator(propagators[0])
+	default:
+		otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagators...))
+	}
+
+	if ots.tracerProvider == nil {
+		ots.tracerProvider = tp
+	}
+
 	ots.tracer = otel.GetTracerProvider().Tracer("component-main")
 
 	return nil

pkg/infra/tracing/test_helper.go

@@ -1,7 +1,16 @@
 package tracing
 
+import (
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+)
+
 func InitializeTracerForTest() Tracer {
-	ots := &Opentelemetry{enabled: noopExporter}
+	exp := tracetest.NewInMemoryExporter()
+	tp, _ := initTracerProvider(exp)
+	otel.SetTracerProvider(tp)
+
+	ots := &Opentelemetry{propagation: "jaeger,w3c", tracerProvider: tp}
 	_ = ots.initOpentelemetryTracer()
 	return ots
 }