Show traceids for failing and successful requests (#64903)

* tracing: show backend trace ids in frontend

* better trace id naming

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

* better trace id naming

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

* better trace id naming

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

* added feature flag

* bind functionality to the feature flag

* use non-generic name for traceid header

* fixed tests

* loki: do not create empty fields

* do not add empty fields

* fixed graphite test mock data

* added unit-tests to queryResponse

* added unit-tests for backend_srv

* more typescript-friendly check

* added unit-tests for runRequest

---------

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

pkg/middleware/middleware.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
 	"github.com/grafana/grafana/pkg/services/org"
 	"github.com/grafana/grafana/pkg/setting"
@@ -29,12 +30,18 @@ func AddDefaultResponseHeaders(cfg *setting.Cfg) web.Handler {
 	t := web.NewTree()
 	t.Add("/api/datasources/uid/:uid/resources/*", nil)
 	t.Add("/api/datasources/:id/resources/*", nil)
+
 	return func(c *web.Context) {
 		c.Resp.Before(func(w web.ResponseWriter) { // if response has already been written, skip.
 			if w.Written() {
 				return
 			}
 
+			traceId := tracing.TraceIDFromContext(c.Req.Context(), false)
+			if traceId != "" {
+				w.Header().Set("grafana-trace-id", traceId)
+			}
+
 			_, _, resourceURLMatch := t.Match(c.Req.URL.Path)
 			resourceCachable := resourceURLMatch && allowCacheControl(c.Resp)
 			if !strings.HasPrefix(c.Req.URL.Path, "/public/plugins/") &&

pkg/services/featuremgmt/registry.go

@@ -315,6 +315,12 @@ var (
 			State:       FeatureStateAlpha,
 			Owner:       grafanaObservabilityLogsSquad,
 		},
+		{
+			Name:        "showTraceId",
+			Description: "Show trace ids for requests",
+			State:       FeatureStateAlpha,
+			Owner:       grafanaObservabilityLogsSquad,
+		},
 		{
 			Name:        "datasourceOnboarding",
 			Description: "Enable data source onboarding page",

pkg/services/featuremgmt/toggles_gen.csv

@@ -46,6 +46,7 @@ accessControlOnCall,beta,@grafana/grafana-authnz-team,false,false,false,false
 nestedFolders,alpha,@grafana/backend-platform,true,false,false,false
 accessTokenExpirationCheck,stable,@grafana/grafana-authnz-team,false,false,false,false
 elasticsearchBackendMigration,alpha,@grafana/observability-logs,false,false,false,false
+showTraceId,alpha,@grafana/observability-logs,false,false,false,false
 datasourceOnboarding,alpha,@grafana/dashboards-squad,false,false,false,false
 emptyDashboardPage,stable,@grafana/dashboards-squad,false,false,false,true
 secureSocksDatasourceProxy,alpha,@grafana/hosted-grafana-team,false,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -195,6 +195,10 @@ const (
 	// Use Elasticsearch as backend data source
 	FlagElasticsearchBackendMigration = "elasticsearchBackendMigration"
 
+	// FlagShowTraceId
+	// Show trace ids for requests
+	FlagShowTraceId = "showTraceId"
+
 	// FlagDatasourceOnboarding
 	// Enable data source onboarding page
 	FlagDatasourceOnboarding = "datasourceOnboarding"