Tracing: Improve HTTP request/middleware spans and standalone apiserver (#85715)

Fixes so that auth middleware trace/span doesn't wrap the next handlers.
Allow tracing service name to be overridden in standalone apiserver.
Change k8s api tracing operation name to KubernetesAPI from 
grafana-apiserver (which is the service name)

pkg/apiserver/builder/helper.go

@@ -76,7 +76,7 @@ func SetupConfig(
 
 		handler := genericapiserver.DefaultBuildHandlerChain(requestHandler, c)
 		handler = filters.WithAcceptHeader(handler)
-		handler = k8stracing.WithTracing(handler, serverConfig.TracerProvider, "grafana-apiserver")
+		handler = k8stracing.WithTracing(handler, serverConfig.TracerProvider, "KubernetesAPI")
 
 		return handler
 	}

pkg/services/apiserver/standalone/options/tracing.go

@@ -26,7 +26,8 @@ type TracingOptions struct {
 	OTLPAddress     string
 	OTLPPropagation string
 
-	Tags map[string]string
+	ServiceName string
+	Tags        map[string]string
 
 	SamplerType        string
 	SamplerParam       float64
@@ -47,6 +48,7 @@ func (o *TracingOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&o.JaegerPropagation, "grafana.tracing.jaeger.propagation", "jaeger", "Tracing Jaeger propagation specifies the text map propagation format, w3c or jaeger.")
 	fs.StringVar(&o.OTLPAddress, "grafana.tracing.otlp.address", "", "Tracing OTLP exporter destination, e.g. localhost:4317.")
 	fs.StringVar(&o.OTLPPropagation, "grafana.tracing.otlp.propagation", "w3c", "Tracing OTLP propagation specifies the text map propagation format, w3c or jaeger.")
+	fs.StringVar(&o.ServiceName, "grafana.tracing.service-name", "grafana-apiserver", "Override the default service name, grafana-apiserver.")
 	fs.StringToStringVar(&o.Tags, "grafana.tracing.tag", map[string]string{}, "Tracing server tag in 'key=value' format. Specify multiple times to add many.")
 	fs.StringVar(&o.SamplerType, "grafana.tracing.sampler-type", "const", "Tracing sampler type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote.")
 	fs.Float64Var(&o.SamplerParam, "grafana.tracing.sampler-param", 0, "Tracing sampler configuration parameter. For 'const' sampler, 0 or 1 for always false/true respectively. For 'rateLimiting' sampler, the number of spans per second. For 'remote' sampler, param is the same as for 'probabilistic' and indicates the initial sampling rate before the actual one is received from the sampling service.")
@@ -54,21 +56,25 @@ func (o *TracingOptions) AddFlags(fs *pflag.FlagSet) {
 }
 
 func (o *TracingOptions) Validate() []error {
-	errors := []error{}
+	errs := []error{}
 
 	if o.JaegerAddress != "" {
 		if _, err := url.Parse(o.JaegerAddress); err != nil {
-			errors = append(errors, fmt.Errorf("failed to parse tracing.jaeger.address: %w", err))
+			errs = append(errs, fmt.Errorf("failed to parse tracing.jaeger.address: %w", err))
 		}
 	}
 
 	if o.SamplingServiceURL != "" {
 		if _, err := url.Parse(o.SamplingServiceURL); err != nil {
-			errors = append(errors, fmt.Errorf("failed to parse tracing.sampling-service: %w", err))
+			errs = append(errs, fmt.Errorf("failed to parse tracing.sampling-service: %w", err))
 		}
 	}
 
-	return errors
+	if o.ServiceName == "" {
+		errs = append(errs, errors.New("grafana.tracing.service-name cannot be empty"))
+	}
+
+	return errs
 }
 
 func (o *TracingOptions) ApplyTo(config *genericapiserver.RecommendedConfig) error {
@@ -93,7 +99,7 @@ func (o *TracingOptions) ApplyTo(config *genericapiserver.RecommendedConfig) err
 		return err
 	}
 
-	tracingCfg.ServiceName = "grafana-apiserver"
+	tracingCfg.ServiceName = o.ServiceName
 	tracingCfg.ServiceVersion = setting.BuildVersion
 
 	for k, v := range o.Tags {

pkg/services/contexthandler/contexthandler.go

@@ -84,11 +84,11 @@ func CopyWithReqContext(ctx context.Context) context.Context {
 // Middleware provides a middleware to initialize the request context.
 func (h *ContextHandler) Middleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		ctx, span := h.tracer.Start(r.Context(), "Auth - Middleware")
-		defer span.End() // this will span to next handlers as well
+		ctx := r.Context()
+		_, span := h.tracer.Start(ctx, "Auth - Middleware")
 
 		reqContext := &contextmodel.ReqContext{
-			Context: web.FromContext(ctx), // Extract web context from context (no knowledge of the trace)
+			Context: web.FromContext(ctx),
 			SignedInUser: &user.SignedInUser{
 				Permissions: map[int64]map[string][]string{},
 			},
@@ -106,12 +106,13 @@ func (h *ContextHandler) Middleware(next http.Handler) http.Handler {
 		// This modifies both r and reqContext.Req since they point to the same value
 		*reqContext.Req = *reqContext.Req.WithContext(ctx)
 
-		traceID := tracing.TraceIDFromContext(reqContext.Req.Context(), false)
+		ctx = trace.ContextWithSpan(reqContext.Req.Context(), span)
+		traceID := tracing.TraceIDFromContext(ctx, false)
 		if traceID != "" {
 			reqContext.Logger = reqContext.Logger.New("traceID", traceID)
 		}
 
-		identity, err := h.authnService.Authenticate(reqContext.Req.Context(), &authn.Request{HTTPRequest: reqContext.Req, Resp: reqContext.Resp})
+		identity, err := h.authnService.Authenticate(ctx, &authn.Request{HTTPRequest: reqContext.Req, Resp: reqContext.Resp})
 		if err != nil {
 			// Hack: set all errors on LookupTokenErr, so we can check it in auth middlewares
 			reqContext.LookupTokenErr = err
@@ -134,6 +135,9 @@ func (h *ContextHandler) Middleware(next http.Handler) http.Handler {
 			reqContext.Resp.Before(h.addIDHeaderEndOfRequestFunc(reqContext.SignedInUser))
 		}
 
+		// End the span to make next handlers not wrapped within middleware span
+		span.End()
+
 		next.ServeHTTP(w, r)
 	})
 }