Show traceids for failing and successful requests (#64903)

* tracing: show backend trace ids in frontend

* better trace id naming

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

* better trace id naming

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

* better trace id naming

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

* added feature flag

* bind functionality to the feature flag

* use non-generic name for traceid header

* fixed tests

* loki: do not create empty fields

* do not add empty fields

* fixed graphite test mock data

* added unit-tests to queryResponse

* added unit-tests for backend_srv

* more typescript-friendly check

* added unit-tests for runRequest

---------

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -84,6 +84,7 @@ Alpha features might be changed or removed without prior notice.
 | `showDashboardValidationWarnings`  | Show warnings when dashboards do not validate against the schema                                                                                                                                    |
 | `mysqlAnsiQuotes`                  | Use double quotes to escape keyword in a MySQL query                                                                                                                                                |
 | `elasticsearchBackendMigration`    | Use Elasticsearch as backend data source                                                                                                                                                            |
+| `showTraceId`                      | Show trace ids for requests                                                                                                                                                                         |
 | `datasourceOnboarding`             | Enable data source onboarding page                                                                                                                                                                  |
 | `secureSocksDatasourceProxy`       | Enable secure socks tunneling for supported core datasources                                                                                                                                        |
 | `authnService`                     | Use new auth service to perform authentication                                                                                                                                                      |

packages/grafana-data/src/types/datasource.ts

@@ -465,6 +465,11 @@ export interface DataQueryResponse {
    * Defaults to LoadingState.Done if state is not defined
    */
   state?: LoadingState;
+
+  /**
+   * traceIds related to the response, if available
+   */
+  traceIds?: string[];
 }
 
 export enum DataQueryErrorType {
@@ -488,6 +493,7 @@ export interface DataQueryError {
   status?: number;
   statusText?: string;
   refId?: string;
+  traceId?: string;
   type?: DataQueryErrorType;
 }
 

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -65,6 +65,7 @@ export interface FeatureToggles {
   nestedFolders?: boolean;
   accessTokenExpirationCheck?: boolean;
   elasticsearchBackendMigration?: boolean;
+  showTraceId?: boolean;
   datasourceOnboarding?: boolean;
   emptyDashboardPage?: boolean;
   secureSocksDatasourceProxy?: boolean;

packages/grafana-data/src/types/panel.ts

@@ -66,6 +66,9 @@ export interface PanelData {
 
   /** Contains the range from the request or a shifted time range if a request uses relative time */
   timeRange: TimeRange;
+
+  /** traceIds collected during the processing of the requests */
+  traceIds?: string[];
 }
 
 export interface PanelProps<T = any> {

packages/grafana-runtime/src/services/backendSrv.ts

@@ -96,6 +96,7 @@ export interface FetchResponse<T = any> {
   readonly type: ResponseType;
   readonly url: string;
   readonly config: BackendSrvRequest;
+  readonly traceId?: string;
 }
 
 /**
@@ -122,6 +123,7 @@ export interface FetchError<T = any> {
   cancelled?: boolean;
   isHandled?: boolean;
   config: BackendSrvRequest;
+  traceId?: string;
 }
 
 export function isFetchError(e: unknown): e is FetchError {

packages/grafana-runtime/src/utils/queryResponse.test.ts

@@ -280,6 +280,82 @@ describe('Query Response parser', () => {
     expect(ids).toEqual(['A', 'B']);
   });
 
+  test('should handle a success-response without traceIds', () => {
+    const input = {
+      data: {
+        results: {
+          A: {
+            frames: [],
+          },
+        },
+      },
+    } as unknown as FetchResponse<BackendDataSourceResponse>;
+    const res = toDataQueryResponse(input);
+    expect(res.traceIds).toBeUndefined();
+  });
+
+  test('should handle a success-response with traceIds', () => {
+    const input = {
+      data: {
+        results: {
+          A: {
+            frames: [],
+          },
+        },
+      },
+      traceId: 'traceId1',
+    } as unknown as FetchResponse<BackendDataSourceResponse>;
+    const res = toDataQueryResponse(input);
+    expect(res.traceIds).toStrictEqual(['traceId1']);
+  });
+
+  test('should handle an error-response without traceIds', () => {
+    const input = {
+      data: {
+        results: {
+          A: {
+            error: 'error from A',
+            status: 400,
+          },
+          B: {
+            error: 'error from B',
+            status: 400,
+          },
+        },
+      },
+    } as unknown as FetchResponse<BackendDataSourceResponse>;
+    const res = toDataQueryResponse(input);
+    expect(res.traceIds).toBeUndefined();
+    expect(res.error?.traceId).toBeUndefined();
+    expect(res.errors).toHaveLength(2);
+    expect(res.errors?.[0].traceId).toBeUndefined();
+    expect(res.errors?.[1].traceId).toBeUndefined();
+  });
+
+  test('should handle an error-response with traceIds', () => {
+    const input = {
+      data: {
+        results: {
+          A: {
+            error: 'error from A',
+            status: 400,
+          },
+          B: {
+            error: 'error from B',
+            status: 400,
+          },
+        },
+      },
+      traceId: 'traceId1',
+    } as unknown as FetchResponse<BackendDataSourceResponse>;
+    const res = toDataQueryResponse(input);
+    expect(res.traceIds).toStrictEqual(['traceId1']);
+    expect(res.error?.traceId).toBe('traceId1');
+    expect(res.errors).toHaveLength(2);
+    expect(res.errors?.[0].traceId).toBe('traceId1');
+    expect(res.errors?.[1].traceId).toBe('traceId1');
+  });
+
   describe('Cache notice', () => {
     let resp: FetchResponse<BackendDataSourceResponse>;
 

packages/grafana-runtime/src/utils/queryResponse.ts

@@ -65,6 +65,13 @@ export function toDataQueryResponse(
   queries?: DataQuery[]
 ): DataQueryResponse {
   const rsp: DataQueryResponse = { data: [], state: LoadingState.Done };
+
+  const traceId = 'traceId' in res ? res.traceId : undefined;
+
+  if (traceId != null) {
+    rsp.traceIds = [traceId];
+  }
+
   // If the response isn't in a correct shape we just ignore the data and pass empty DataQueryResponse.
   if ((res as FetchResponse).data?.results) {
     const results = (res as FetchResponse).data.results;
@@ -83,17 +90,21 @@ export function toDataQueryResponse(
 
     for (const dr of data) {
       if (dr.error) {
-        if (!rsp.error) {
-          rsp.error = {
+        const errorObj: DataQueryError = {
           refId: dr.refId,
           message: dr.error,
           status: dr.status,
         };
+        if (traceId != null) {
+          errorObj.traceId = traceId;
+        }
+        if (!rsp.error) {
+          rsp.error = { ...errorObj };
         }
         if (rsp.errors) {
-          rsp.errors.push({ refId: dr.refId, message: dr.error, status: dr.status });
+          rsp.errors.push({ ...errorObj });
         } else {
-          rsp.errors = [{ refId: dr.refId, message: dr.error, status: dr.status }];
+          rsp.errors = [{ ...errorObj }];
         }
         rsp.state = LoadingState.Error;
       }

pkg/middleware/middleware.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
 	"github.com/grafana/grafana/pkg/services/org"
 	"github.com/grafana/grafana/pkg/setting"
@@ -29,12 +30,18 @@ func AddDefaultResponseHeaders(cfg *setting.Cfg) web.Handler {
 	t := web.NewTree()
 	t.Add("/api/datasources/uid/:uid/resources/*", nil)
 	t.Add("/api/datasources/:id/resources/*", nil)
+
 	return func(c *web.Context) {
 		c.Resp.Before(func(w web.ResponseWriter) { // if response has already been written, skip.
 			if w.Written() {
 				return
 			}
 
+			traceId := tracing.TraceIDFromContext(c.Req.Context(), false)
+			if traceId != "" {
+				w.Header().Set("grafana-trace-id", traceId)
+			}
+
 			_, _, resourceURLMatch := t.Match(c.Req.URL.Path)
 			resourceCachable := resourceURLMatch && allowCacheControl(c.Resp)
 			if !strings.HasPrefix(c.Req.URL.Path, "/public/plugins/") &&

pkg/services/featuremgmt/registry.go

@@ -315,6 +315,12 @@ var (
 			State:       FeatureStateAlpha,
 			Owner:       grafanaObservabilityLogsSquad,
 		},
+		{
+			Name:        "showTraceId",
+			Description: "Show trace ids for requests",
+			State:       FeatureStateAlpha,
+			Owner:       grafanaObservabilityLogsSquad,
+		},
 		{
 			Name:        "datasourceOnboarding",
 			Description: "Enable data source onboarding page",

pkg/services/featuremgmt/toggles_gen.csv

@@ -46,6 +46,7 @@ accessControlOnCall,beta,@grafana/grafana-authnz-team,false,false,false,false
 nestedFolders,alpha,@grafana/backend-platform,true,false,false,false
 accessTokenExpirationCheck,stable,@grafana/grafana-authnz-team,false,false,false,false
 elasticsearchBackendMigration,alpha,@grafana/observability-logs,false,false,false,false
+showTraceId,alpha,@grafana/observability-logs,false,false,false,false
 datasourceOnboarding,alpha,@grafana/dashboards-squad,false,false,false,false
 emptyDashboardPage,stable,@grafana/dashboards-squad,false,false,false,true
 secureSocksDatasourceProxy,alpha,@grafana/hosted-grafana-team,false,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -195,6 +195,10 @@ const (
 	// Use Elasticsearch as backend data source
 	FlagElasticsearchBackendMigration = "elasticsearchBackendMigration"
 
+	// FlagShowTraceId
+	// Show trace ids for requests
+	FlagShowTraceId = "showTraceId"
+
 	// FlagDatasourceOnboarding
 	// Enable data source onboarding page
 	FlagDatasourceOnboarding = "datasourceOnboarding"

public/app/core/services/backend_srv.ts

@@ -50,6 +50,8 @@ export interface FolderRequestOptions {
   withAccessControl?: boolean;
 }
 
+const GRAFANA_TRACEID_HEADER = 'grafana-trace-id';
+
 export class BackendSrv implements BackendService {
   private inFlightRequests: Subject<string> = new Subject<string>();
   private HTTP_REQUEST_CANCELED = -1;
@@ -223,6 +225,7 @@ export class BackendSrv implements BackendService {
           type,
           redirected,
           config: options,
+          traceId: response.headers.get(GRAFANA_TRACEID_HEADER) ?? undefined,
         };
         return fetchResponse;
       }),
@@ -238,7 +241,13 @@ export class BackendSrv implements BackendService {
         filter((response) => response.ok === false),
         mergeMap((response) => {
           const { status, statusText, data } = response;
-          const fetchErrorResponse: FetchError = { status, statusText, data, config: options };
+          const fetchErrorResponse: FetchError = {
+            status,
+            statusText,
+            data,
+            config: options,
+            traceId: response.headers.get(GRAFANA_TRACEID_HEADER) ?? undefined,
+          };
           return throwError(fetchErrorResponse);
         }),
         retryWhen((attempts: Observable<any>) =>

public/app/core/specs/backend_srv.test.ts

@@ -1,5 +1,5 @@
 import 'whatwg-fetch'; // fetch polyfill needed for PhantomJs rendering
-import { Observable, of } from 'rxjs';
+import { Observable, of, lastValueFrom } from 'rxjs';
 import { delay } from 'rxjs/operators';
 
 import { AppEvents, DataQueryErrorType, EventBusExtended } from '@grafana/data';
@@ -21,6 +21,7 @@ const getTestContext = (overides?: object) => {
     redirected: false,
     type: 'basic',
     url: 'http://localhost:3000/api/some-mock',
+    headers: new Map(),
   };
   const props = { ...defaults, ...overides };
   const textMock = jest.fn().mockResolvedValue(JSON.stringify(props.data));
@@ -29,6 +30,7 @@ const getTestContext = (overides?: object) => {
       ok: props.ok,
       status: props.status,
       statusText: props.statusText,
+      headers: props.headers,
       text: textMock,
       redirected: false,
       type: 'basic',
@@ -355,6 +357,63 @@ describe('backendSrv', () => {
         });
       });
     });
+
+    describe('traceId handling', () => {
+      const opts = { url: '/something', method: 'GET' };
+      it('should handle a success-response without traceId', async () => {
+        const ctx = getTestContext({ status: 200, statusText: 'OK', headers: new Headers() });
+        const res = await lastValueFrom(ctx.backendSrv.fetch(opts));
+        expect(res.traceId).toBeUndefined();
+      });
+
+      it('should handle a success-response with traceId', async () => {
+        const ctx = getTestContext({
+          status: 200,
+          statusText: 'OK',
+          headers: new Headers({
+            'grafana-trace-id': 'traceId1',
+          }),
+        });
+        const res = await lastValueFrom(ctx.backendSrv.fetch(opts));
+        expect(res.traceId).toBe('traceId1');
+      });
+
+      it('should handle an error-response without traceId', () => {
+        const ctx = getTestContext({
+          ok: false,
+          status: 500,
+          statusText: 'INTERNAL SERVER ERROR',
+          headers: new Headers(),
+        });
+        return lastValueFrom(ctx.backendSrv.fetch(opts)).then(
+          (data) => {
+            throw new Error('must not get here');
+          },
+          (error) => {
+            expect(error.traceId).toBeUndefined();
+          }
+        );
+      });
+
+      it('should handle an error-response with traceId', () => {
+        const ctx = getTestContext({
+          ok: false,
+          status: 500,
+          statusText: 'INTERNAL SERVER ERROR',
+          headers: new Headers({
+            'grafana-trace-id': 'traceId1',
+          }),
+        });
+        return lastValueFrom(ctx.backendSrv.fetch(opts)).then(
+          (data) => {
+            throw new Error('must not get here');
+          },
+          (error) => {
+            expect(error.traceId).toBe('traceId1');
+          }
+        );
+      });
+    });
   });
 
   describe('datasourceRequest', () => {
@@ -369,6 +428,7 @@ describe('backendSrv', () => {
             ok: true,
             status: 200,
             statusText: 'Ok',
+            headers: new Map(),
             text: () => Promise.resolve(JSON.stringify(slowData)),
             redirected: false,
             type: 'basic',
@@ -382,6 +442,7 @@ describe('backendSrv', () => {
           ok: true,
           status: 200,
           statusText: 'Ok',
+          headers: new Map(),
           text: () => Promise.resolve(JSON.stringify(fastData)),
           redirected: false,
           type: 'basic',

public/app/features/inspector/InspectErrorTab.tsx

@@ -1,6 +1,7 @@
 import React from 'react';
 
 import { DataQueryError } from '@grafana/data';
+import { config } from '@grafana/runtime';
 import { Alert, JSONFormatter } from '@grafana/ui';
 
 interface InspectErrorTabProps {
@@ -36,6 +37,12 @@ function renderError(error: DataQueryError) {
         <>
           {error.status && <>Status: {error.status}. Message: </>}
           {msg}
+          {config.featureToggles.showTraceId && error.traceId != null && (
+            <>
+              <br />
+              (Trace ID: {error.traceId})
+            </>
+          )}
         </>
       );
     } else {

public/app/features/inspector/InspectStatsTab.tsx

@@ -2,9 +2,11 @@ import React from 'react';
 
 import { PanelData, QueryResultMetaStat, TimeZone } from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
+import { config } from '@grafana/runtime';
 import { t } from 'app/core/internationalization';
 
 import { InspectStatsTable } from './InspectStatsTable';
+import { InspectStatsTraceIdsTable } from './InspectStatsTraceIdsTable';
 
 interface InspectStatsTabProps {
   data: PanelData;
@@ -59,11 +61,15 @@ export const InspectStatsTab = ({ data, timeZone }: InspectStatsTabProps) => {
 
   const statsTableName = t('dashboard.inspect-stats.table-title', 'Stats');
   const dataStatsTableName = t('dashboard.inspect-stats.data-title', 'Data source stats');
+  const traceIdsStatsTableName = t('dashboard.inspect-stats.data-traceids', 'Trace IDs');
 
   return (
     <div aria-label={selectors.components.PanelInspector.Stats.content}>
       <InspectStatsTable timeZone={timeZone} name={statsTableName} stats={stats} />
       <InspectStatsTable timeZone={timeZone} name={dataStatsTableName} stats={dataStats} />
+      {config.featureToggles.showTraceId && (
+        <InspectStatsTraceIdsTable name={traceIdsStatsTableName} traceIds={data.traceIds ?? []} />
+      )}
     </div>
   );
 };

public/app/features/inspector/InspectStatsTraceIdsTable.tsx

@@ -0,0 +1,47 @@
+import { css } from '@emotion/css';
+import React from 'react';
+
+import { GrafanaTheme2 } from '@grafana/data';
+import { stylesFactory, useTheme2 } from '@grafana/ui';
+
+type Props = {
+  name: string;
+  traceIds: string[];
+};
+
+export const InspectStatsTraceIdsTable = ({ name, traceIds }: Props) => {
+  const theme = useTheme2();
+  const styles = getStyles(theme);
+
+  if (traceIds.length === 0) {
+    return null;
+  }
+
+  return (
+    <div className={styles.wrapper}>
+      <div className="section-heading">{name}</div>
+      <table className="filter-table width-30">
+        <tbody>
+          {traceIds.map((traceId, index) => {
+            return (
+              <tr key={`${traceId}-${index}`}>
+                <td>{traceId}</td>
+              </tr>
+            );
+          })}
+        </tbody>
+      </table>
+    </div>
+  );
+};
+
+const getStyles = stylesFactory((theme: GrafanaTheme2) => {
+  return {
+    wrapper: css`
+      padding-bottom: ${theme.spacing(2)};
+    `,
+    cell: css`
+      text-align: right;
+    `,
+  };
+});

public/app/features/query/components/QueryErrorAlert.tsx

@@ -2,6 +2,7 @@ import { css } from '@emotion/css';
 import React from 'react';
 
 import { DataQueryError, GrafanaTheme2 } from '@grafana/data';
+import { config } from '@grafana/runtime';
 import { Icon, useStyles2 } from '@grafana/ui';
 
 export interface Props {
@@ -18,7 +19,14 @@ export function QueryErrorAlert({ error }: Props) {
       <div className={styles.icon}>
         <Icon name="exclamation-triangle" />
       </div>
-      <div className={styles.message}>{message}</div>
+      <div className={styles.message}>
+        {message}
+        {config.featureToggles.showTraceId && error.traceId != null && (
+          <>
+            <br /> <span>(Trace ID: {error.traceId})</span>
+          </>
+        )}
+      </div>
     </div>
   );
 }

public/app/features/query/state/runRequest.test.ts

@@ -223,6 +223,49 @@ describe('runRequest', () => {
     });
   });
 
+  runRequestScenario('When the response contains traceIds', (ctx) => {
+    ctx.setup(() => {
+      ctx.start();
+      ctx.emitPacket({
+        data: [{ name: 'data-a', refId: 'A' } as DataFrame],
+      });
+      ctx.emitPacket({
+        data: [{ name: 'data-b', refId: 'B' } as DataFrame],
+      });
+      ctx.emitPacket({
+        data: [{ name: 'data-c', refId: 'C' } as DataFrame],
+        traceIds: ['t1', 't2'],
+      });
+      ctx.emitPacket({
+        data: [{ name: 'data-d', refId: 'D' } as DataFrame],
+      });
+      ctx.emitPacket({
+        data: [{ name: 'data-e', refId: 'E' } as DataFrame],
+        traceIds: ['t3', 't4'],
+      });
+      ctx.emitPacket({
+        data: [{ name: 'data-e', refId: 'E' } as DataFrame],
+        traceIds: ['t4', 't4'],
+      });
+    });
+    it('should collect traceIds correctly', () => {
+      const { results } = ctx;
+      expect(results).toHaveLength(6);
+      expect(results[0].traceIds).toBeUndefined();
+
+      // this is the result of adding no-traces data to no-traces state
+      expect(results[1].traceIds).toBeUndefined();
+      // this is the result of adding with-traces data to no-traces state
+      expect(results[2].traceIds).toStrictEqual(['t1', 't2']);
+      // this is the result of adding no-traces data to with-traces state
+      expect(results[3].traceIds).toStrictEqual(['t1', 't2']);
+      // this is the result of adding with-traces data to with-traces state
+      expect(results[4].traceIds).toStrictEqual(['t1', 't2', 't3', 't4']);
+      // this is the result of adding with-traces data to with-traces state with duplicate traceIds
+      expect(results[5].traceIds).toStrictEqual(['t1', 't2', 't3', 't4']);
+    });
+  });
+
   runRequestScenario('After response with state Streaming', (ctx) => {
     ctx.setup(() => {
       ctx.start();

public/app/features/query/state/runRequest.ts

@@ -88,6 +88,13 @@ export function processResponsePacket(packet: DataQueryResponse, state: RunningQ
     timeRange,
   };
 
+  // we use a Set to deduplicate the traceIds
+  const traceIdSet = new Set([...(state.panelData.traceIds ?? []), ...(packet.traceIds ?? [])]);
+
+  if (traceIdSet.size > 0) {
+    panelData.traceIds = Array.from(traceIdSet);
+  }
+
   return { packets, panelData };
 }
 

public/app/plugins/datasource/graphite/datasource_integration.test.ts

@@ -116,12 +116,12 @@ function mockBackendSrv(data: string) {
       redirected: false,
       type: 'basic',
       url: 'http://localhost:3000/api/some-mock',
-      headers: {
+      headers: new Headers({
         method: 'GET',
         url: '/functions',
         // to work around Graphite returning invalid JSON
         responseType: 'text',
-      },
+      }),
     };
     return of(mockedResponse);
   });

public/app/plugins/datasource/loki/responseUtils.ts

@@ -154,7 +154,15 @@ export function combineResponses(currentResult: DataQueryResponse | null, newRes
   // some grafana parts do not behave well.
   // we just choose the old error, if it exists,
   // otherwise the new error, if it exists.
-  currentResult.error = currentResult.error ?? newResult.error;
+  const mergedError = currentResult.error ?? newResult.error;
+  if (mergedError != null) {
+    currentResult.error = mergedError;
+  }
+
+  const mergedTraceIds = [...(currentResult.traceIds ?? []), ...(newResult.traceIds ?? [])];
+  if (mergedTraceIds.length > 0) {
+    currentResult.traceIds = mergedTraceIds;
+  }
 
   return currentResult;
 }