Show traceids for failing and successful requests (#64903)

* tracing: show backend trace ids in frontend * better trace id naming Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com> * better trace id naming Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com> * better trace id naming Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com> * added feature flag * bind functionality to the feature flag * use non-generic name for traceid header * fixed tests * loki: do not create empty fields * do not add empty fields * fixed graphite test mock data * added unit-tests to queryResponse * added unit-tests for backend_srv * more typescript-friendly check * added unit-tests for runRequest --------- Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>
2025-02-25 18:55:37 -06:00 · 2023-04-05 09:13:24 +02:00 · 2023-04-05 09:13:24 +02:00 · 4ded937c79
commit 4ded937c79
parent 9719ee9bd3
21 changed files with 327 additions and 13 deletions
--- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md
+++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md
@ -84,6 +84,7 @@ Alpha features might be changed or removed without prior notice.
 | `showDashboardValidationWarnings`  | Show warnings when dashboards do not validate against the schema                                                                                                                                    |
 | `mysqlAnsiQuotes`                  | Use double quotes to escape keyword in a MySQL query                                                                                                                                                |
 | `elasticsearchBackendMigration`    | Use Elasticsearch as backend data source                                                                                                                                                            |
+| `showTraceId`                      | Show trace ids for requests                                                                                                                                                                         |
 | `datasourceOnboarding`             | Enable data source onboarding page                                                                                                                                                                  |
 | `secureSocksDatasourceProxy`       | Enable secure socks tunneling for supported core datasources                                                                                                                                        |
 | `authnService`                     | Use new auth service to perform authentication                                                                                                                                                      |
--- a/packages/grafana-data/src/types/datasource.ts
+++ b/packages/grafana-data/src/types/datasource.ts
@ -465,6 +465,11 @@ export interface DataQueryResponse {
   * Defaults to LoadingState.Done if state is not defined
   */
  state?: LoadingState;
+
+  /**
+   * traceIds related to the response, if available
+   */
+  traceIds?: string[];
 }

 export enum DataQueryErrorType {
@ -488,6 +493,7 @@ export interface DataQueryError {
  status?: number;
  statusText?: string;
  refId?: string;
+  traceId?: string;
  type?: DataQueryErrorType;
 }

--- a/packages/grafana-data/src/types/featureToggles.gen.ts
+++ b/packages/grafana-data/src/types/featureToggles.gen.ts
@ -65,6 +65,7 @@ export interface FeatureToggles {
  nestedFolders?: boolean;
  accessTokenExpirationCheck?: boolean;
  elasticsearchBackendMigration?: boolean;
+  showTraceId?: boolean;
  datasourceOnboarding?: boolean;
  emptyDashboardPage?: boolean;
  secureSocksDatasourceProxy?: boolean;
--- a/packages/grafana-data/src/types/panel.ts
+++ b/packages/grafana-data/src/types/panel.ts
@ -66,6 +66,9 @@ export interface PanelData {

  /** Contains the range from the request or a shifted time range if a request uses relative time */
  timeRange: TimeRange;
+
+  /** traceIds collected during the processing of the requests */
+  traceIds?: string[];
 }

 export interface PanelProps<T = any> {
--- a/packages/grafana-runtime/src/services/backendSrv.ts
+++ b/packages/grafana-runtime/src/services/backendSrv.ts
@ -96,6 +96,7 @@ export interface FetchResponse<T = any> {
  readonly type: ResponseType;
  readonly url: string;
  readonly config: BackendSrvRequest;
+  readonly traceId?: string;
 }

 /**
@ -122,6 +123,7 @@ export interface FetchError<T = any> {
  cancelled?: boolean;
  isHandled?: boolean;
  config: BackendSrvRequest;
+  traceId?: string;
 }

 export function isFetchError(e: unknown): e is FetchError {
--- a/packages/grafana-runtime/src/utils/queryResponse.test.ts
+++ b/packages/grafana-runtime/src/utils/queryResponse.test.ts
@ -280,6 +280,82 @@ describe('Query Response parser', () => {
    expect(ids).toEqual(['A', 'B']);
  });

+  test('should handle a success-response without traceIds', () => {
+    const input = {
+      data: {
+        results: {
+          A: {
+            frames: [],
+          },
+        },
+      },
+    } as unknown as FetchResponse<BackendDataSourceResponse>;
+    const res = toDataQueryResponse(input);
+    expect(res.traceIds).toBeUndefined();
+  });
+
+  test('should handle a success-response with traceIds', () => {
+    const input = {
+      data: {
+        results: {
+          A: {
+            frames: [],
+          },
+        },
+      },
+      traceId: 'traceId1',
+    } as unknown as FetchResponse<BackendDataSourceResponse>;
+    const res = toDataQueryResponse(input);
+    expect(res.traceIds).toStrictEqual(['traceId1']);
+  });
+
+  test('should handle an error-response without traceIds', () => {
+    const input = {
+      data: {
+        results: {
+          A: {
+            error: 'error from A',
+            status: 400,
+          },
+          B: {
+            error: 'error from B',
+            status: 400,
+          },
+        },
+      },
+    } as unknown as FetchResponse<BackendDataSourceResponse>;
+    const res = toDataQueryResponse(input);
+    expect(res.traceIds).toBeUndefined();
+    expect(res.error?.traceId).toBeUndefined();
+    expect(res.errors).toHaveLength(2);
+    expect(res.errors?.[0].traceId).toBeUndefined();
+    expect(res.errors?.[1].traceId).toBeUndefined();
+  });
+
+  test('should handle an error-response with traceIds', () => {
+    const input = {
+      data: {
+        results: {
+          A: {
+            error: 'error from A',
+            status: 400,
+          },
+          B: {
+            error: 'error from B',
+            status: 400,
+          },
+        },
+      },
+      traceId: 'traceId1',
+    } as unknown as FetchResponse<BackendDataSourceResponse>;
+    const res = toDataQueryResponse(input);
+    expect(res.traceIds).toStrictEqual(['traceId1']);
+    expect(res.error?.traceId).toBe('traceId1');
+    expect(res.errors).toHaveLength(2);
+    expect(res.errors?.[0].traceId).toBe('traceId1');
+    expect(res.errors?.[1].traceId).toBe('traceId1');
+  });
+
  describe('Cache notice', () => {
    let resp: FetchResponse<BackendDataSourceResponse>;

--- a/packages/grafana-runtime/src/utils/queryResponse.ts
+++ b/packages/grafana-runtime/src/utils/queryResponse.ts
@ -65,6 +65,13 @@ export function toDataQueryResponse(
  queries?: DataQuery[]
 ): DataQueryResponse {
  const rsp: DataQueryResponse = { data: [], state: LoadingState.Done };
+
+  const traceId = 'traceId' in res ? res.traceId : undefined;
+
+  if (traceId != null) {
+    rsp.traceIds = [traceId];
+  }
+
  // If the response isn't in a correct shape we just ignore the data and pass empty DataQueryResponse.
  if ((res as FetchResponse).data?.results) {
    const results = (res as FetchResponse).data.results;
@ -83,17 +90,21 @@ export function toDataQueryResponse(

    for (const dr of data) {
      if (dr.error) {
+        const errorObj: DataQueryError = {
+          refId: dr.refId,
+          message: dr.error,
+          status: dr.status,
+        };
+        if (traceId != null) {
+          errorObj.traceId = traceId;
+        }
        if (!rsp.error) {
-          rsp.error = {
-            refId: dr.refId,
-            message: dr.error,
-            status: dr.status,
-          };
+          rsp.error = { ...errorObj };
        }
        if (rsp.errors) {
-          rsp.errors.push({ refId: dr.refId, message: dr.error, status: dr.status });
+          rsp.errors.push({ ...errorObj });
        } else {
-          rsp.errors = [{ refId: dr.refId, message: dr.error, status: dr.status }];
+          rsp.errors = [{ ...errorObj }];
        }
        rsp.state = LoadingState.Error;
      }
--- a/pkg/middleware/middleware.go
+++ b/pkg/middleware/middleware.go
@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strings"

+	"github.com/grafana/grafana/pkg/infra/tracing"
 	contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
 	"github.com/grafana/grafana/pkg/services/org"
 	"github.com/grafana/grafana/pkg/setting"
@ -29,12 +30,18 @@ func AddDefaultResponseHeaders(cfg *setting.Cfg) web.Handler {
 	t := web.NewTree()
 	t.Add("/api/datasources/uid/:uid/resources/*", nil)
 	t.Add("/api/datasources/:id/resources/*", nil)
+
 	return func(c *web.Context) {
 		c.Resp.Before(func(w web.ResponseWriter) { // if response has already been written, skip.
 			if w.Written() {
 				return
 			}

+			traceId := tracing.TraceIDFromContext(c.Req.Context(), false)
+			if traceId != "" {
+				w.Header().Set("grafana-trace-id", traceId)
+			}
+
 			_, _, resourceURLMatch := t.Match(c.Req.URL.Path)
 			resourceCachable := resourceURLMatch && allowCacheControl(c.Resp)
 			if !strings.HasPrefix(c.Req.URL.Path, "/public/plugins/") &&
--- a/pkg/services/featuremgmt/registry.go
+++ b/pkg/services/featuremgmt/registry.go
@ -315,6 +315,12 @@ var (
 			State:       FeatureStateAlpha,
 			Owner:       grafanaObservabilityLogsSquad,
 		},
+		{
+			Name:        "showTraceId",
+			Description: "Show trace ids for requests",
+			State:       FeatureStateAlpha,
+			Owner:       grafanaObservabilityLogsSquad,
+		},
 		{
 			Name:        "datasourceOnboarding",
 			Description: "Enable data source onboarding page",
--- a/pkg/services/featuremgmt/toggles_gen.csv
+++ b/pkg/services/featuremgmt/toggles_gen.csv
@ -46,6 +46,7 @@ accessControlOnCall,beta,@grafana/grafana-authnz-team,false,false,false,false
 nestedFolders,alpha,@grafana/backend-platform,true,false,false,false
 accessTokenExpirationCheck,stable,@grafana/grafana-authnz-team,false,false,false,false
 elasticsearchBackendMigration,alpha,@grafana/observability-logs,false,false,false,false
+showTraceId,alpha,@grafana/observability-logs,false,false,false,false
 datasourceOnboarding,alpha,@grafana/dashboards-squad,false,false,false,false
 emptyDashboardPage,stable,@grafana/dashboards-squad,false,false,false,true
 secureSocksDatasourceProxy,alpha,@grafana/hosted-grafana-team,false,false,false,false
--- a/pkg/services/featuremgmt/toggles_gen.go
+++ b/pkg/services/featuremgmt/toggles_gen.go
@ -195,6 +195,10 @@ const (
 	// Use Elasticsearch as backend data source
 	FlagElasticsearchBackendMigration = "elasticsearchBackendMigration"

+	// FlagShowTraceId
+	// Show trace ids for requests
+	FlagShowTraceId = "showTraceId"
+
 	// FlagDatasourceOnboarding
 	// Enable data source onboarding page
 	FlagDatasourceOnboarding = "datasourceOnboarding"
--- a/public/app/core/services/backend_srv.ts
+++ b/public/app/core/services/backend_srv.ts
@ -50,6 +50,8 @@ export interface FolderRequestOptions {
  withAccessControl?: boolean;
 }

+const GRAFANA_TRACEID_HEADER = 'grafana-trace-id';
+
 export class BackendSrv implements BackendService {
  private inFlightRequests: Subject<string> = new Subject<string>();
  private HTTP_REQUEST_CANCELED = -1;
@ -223,6 +225,7 @@ export class BackendSrv implements BackendService {
          type,
          redirected,
          config: options,
+          traceId: response.headers.get(GRAFANA_TRACEID_HEADER) ?? undefined,
        };
        return fetchResponse;
      }),
@ -238,7 +241,13 @@ export class BackendSrv implements BackendService {
        filter((response) => response.ok === false),
        mergeMap((response) => {
          const { status, statusText, data } = response;
-          const fetchErrorResponse: FetchError = { status, statusText, data, config: options };
+          const fetchErrorResponse: FetchError = {
+            status,
+            statusText,
+            data,
+            config: options,
+            traceId: response.headers.get(GRAFANA_TRACEID_HEADER) ?? undefined,
+          };
          return throwError(fetchErrorResponse);
        }),
        retryWhen((attempts: Observable<any>) =>
--- a/public/app/core/specs/backend_srv.test.ts
+++ b/public/app/core/specs/backend_srv.test.ts
@ -1,5 +1,5 @@
 import 'whatwg-fetch'; // fetch polyfill needed for PhantomJs rendering
-import { Observable, of } from 'rxjs';
+import { Observable, of, lastValueFrom } from 'rxjs';
 import { delay } from 'rxjs/operators';

 import { AppEvents, DataQueryErrorType, EventBusExtended } from '@grafana/data';
@ -21,6 +21,7 @@ const getTestContext = (overides?: object) => {
    redirected: false,
    type: 'basic',
    url: 'http://localhost:3000/api/some-mock',
+    headers: new Map(),
  };
  const props = { ...defaults, ...overides };
  const textMock = jest.fn().mockResolvedValue(JSON.stringify(props.data));
@ -29,6 +30,7 @@ const getTestContext = (overides?: object) => {
      ok: props.ok,
      status: props.status,
      statusText: props.statusText,
+      headers: props.headers,
      text: textMock,
      redirected: false,
      type: 'basic',
@ -355,6 +357,63 @@ describe('backendSrv', () => {
        });
      });
    });
+
+    describe('traceId handling', () => {
+      const opts = { url: '/something', method: 'GET' };
+      it('should handle a success-response without traceId', async () => {
+        const ctx = getTestContext({ status: 200, statusText: 'OK', headers: new Headers() });
+        const res = await lastValueFrom(ctx.backendSrv.fetch(opts));
+        expect(res.traceId).toBeUndefined();
+      });
+
+      it('should handle a success-response with traceId', async () => {
+        const ctx = getTestContext({
+          status: 200,
+          statusText: 'OK',
+          headers: new Headers({
+            'grafana-trace-id': 'traceId1',
+          }),
+        });
+        const res = await lastValueFrom(ctx.backendSrv.fetch(opts));
+        expect(res.traceId).toBe('traceId1');
+      });
+
+      it('should handle an error-response without traceId', () => {
+        const ctx = getTestContext({
+          ok: false,
+          status: 500,
+          statusText: 'INTERNAL SERVER ERROR',
+          headers: new Headers(),
+        });
+        return lastValueFrom(ctx.backendSrv.fetch(opts)).then(
+          (data) => {
+            throw new Error('must not get here');
+          },
+          (error) => {
+            expect(error.traceId).toBeUndefined();
+          }
+        );
+      });
+
+      it('should handle an error-response with traceId', () => {
+        const ctx = getTestContext({
+          ok: false,
+          status: 500,
+          statusText: 'INTERNAL SERVER ERROR',
+          headers: new Headers({
+            'grafana-trace-id': 'traceId1',
+          }),
+        });
+        return lastValueFrom(ctx.backendSrv.fetch(opts)).then(
+          (data) => {
+            throw new Error('must not get here');
+          },
+          (error) => {
+            expect(error.traceId).toBe('traceId1');
+          }
+        );
+      });
+    });
  });

  describe('datasourceRequest', () => {
@ -369,6 +428,7 @@ describe('backendSrv', () => {
            ok: true,
            status: 200,
            statusText: 'Ok',
+            headers: new Map(),
            text: () => Promise.resolve(JSON.stringify(slowData)),
            redirected: false,
            type: 'basic',
@ -382,6 +442,7 @@ describe('backendSrv', () => {
          ok: true,
          status: 200,
          statusText: 'Ok',
+          headers: new Map(),
          text: () => Promise.resolve(JSON.stringify(fastData)),
          redirected: false,
          type: 'basic',
--- a/public/app/features/inspector/InspectErrorTab.tsx
+++ b/public/app/features/inspector/InspectErrorTab.tsx
@ -1,6 +1,7 @@
 import React from 'react';

 import { DataQueryError } from '@grafana/data';
+import { config } from '@grafana/runtime';
 import { Alert, JSONFormatter } from '@grafana/ui';

 interface InspectErrorTabProps {
@ -36,6 +37,12 @@ function renderError(error: DataQueryError) {
        <>
          {error.status && <>Status: {error.status}. Message: </>}
          {msg}
+          {config.featureToggles.showTraceId && error.traceId != null && (
+            <>
+              <br />
+              (Trace ID: {error.traceId})
+            </>
+          )}
        </>
      );
    } else {
--- a/public/app/features/inspector/InspectStatsTab.tsx
+++ b/public/app/features/inspector/InspectStatsTab.tsx
@ -2,9 +2,11 @@ import React from 'react';

 import { PanelData, QueryResultMetaStat, TimeZone } from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
+import { config } from '@grafana/runtime';
 import { t } from 'app/core/internationalization';

 import { InspectStatsTable } from './InspectStatsTable';
+import { InspectStatsTraceIdsTable } from './InspectStatsTraceIdsTable';

 interface InspectStatsTabProps {
  data: PanelData;
@ -59,11 +61,15 @@ export const InspectStatsTab = ({ data, timeZone }: InspectStatsTabProps) => {

  const statsTableName = t('dashboard.inspect-stats.table-title', 'Stats');
  const dataStatsTableName = t('dashboard.inspect-stats.data-title', 'Data source stats');
+  const traceIdsStatsTableName = t('dashboard.inspect-stats.data-traceids', 'Trace IDs');

  return (
    <div aria-label={selectors.components.PanelInspector.Stats.content}>
      <InspectStatsTable timeZone={timeZone} name={statsTableName} stats={stats} />
      <InspectStatsTable timeZone={timeZone} name={dataStatsTableName} stats={dataStats} />
+      {config.featureToggles.showTraceId && (
+        <InspectStatsTraceIdsTable name={traceIdsStatsTableName} traceIds={data.traceIds ?? []} />
+      )}
    </div>
  );
 };
--- a/public/app/features/inspector/InspectStatsTraceIdsTable.tsx
+++ b/public/app/features/inspector/InspectStatsTraceIdsTable.tsx
@ -0,0 +1,47 @@
+import { css } from '@emotion/css';
+import React from 'react';
+
+import { GrafanaTheme2 } from '@grafana/data';
+import { stylesFactory, useTheme2 } from '@grafana/ui';
+
+type Props = {
+  name: string;
+  traceIds: string[];
+};
+
+export const InspectStatsTraceIdsTable = ({ name, traceIds }: Props) => {
+  const theme = useTheme2();
+  const styles = getStyles(theme);
+
+  if (traceIds.length === 0) {
+    return null;
+  }
+
+  return (
+    <div className={styles.wrapper}>
+      <div className="section-heading">{name}</div>
+      <table className="filter-table width-30">
+        <tbody>
+          {traceIds.map((traceId, index) => {
+            return (
+              <tr key={`${traceId}-${index}`}>
+                <td>{traceId}</td>
+              </tr>
+            );
+          })}
+        </tbody>
+      </table>
+    </div>
+  );
+};
+
+const getStyles = stylesFactory((theme: GrafanaTheme2) => {
+  return {
+    wrapper: css`
+      padding-bottom: ${theme.spacing(2)};
+    `,
+    cell: css`
+      text-align: right;
+    `,
+  };
+});
--- a/public/app/features/query/components/QueryErrorAlert.tsx
+++ b/public/app/features/query/components/QueryErrorAlert.tsx
@ -2,6 +2,7 @@ import { css } from '@emotion/css';
 import React from 'react';

 import { DataQueryError, GrafanaTheme2 } from '@grafana/data';
+import { config } from '@grafana/runtime';
 import { Icon, useStyles2 } from '@grafana/ui';

 export interface Props {
@ -18,7 +19,14 @@ export function QueryErrorAlert({ error }: Props) {
      <div className={styles.icon}>
        <Icon name="exclamation-triangle" />
      </div>
-      <div className={styles.message}>{message}</div>
+      <div className={styles.message}>
+        {message}
+        {config.featureToggles.showTraceId && error.traceId != null && (
+          <>
+            <br /> <span>(Trace ID: {error.traceId})</span>
+          </>
+        )}
+      </div>
    </div>
  );
 }
--- a/public/app/features/query/state/runRequest.test.ts
+++ b/public/app/features/query/state/runRequest.test.ts
@ -223,6 +223,49 @@ describe('runRequest', () => {
    });
  });

+  runRequestScenario('When the response contains traceIds', (ctx) => {
+    ctx.setup(() => {
+      ctx.start();
+      ctx.emitPacket({
+        data: [{ name: 'data-a', refId: 'A' } as DataFrame],
+      });
+      ctx.emitPacket({
+        data: [{ name: 'data-b', refId: 'B' } as DataFrame],
+      });
+      ctx.emitPacket({
+        data: [{ name: 'data-c', refId: 'C' } as DataFrame],
+        traceIds: ['t1', 't2'],
+      });
+      ctx.emitPacket({
+        data: [{ name: 'data-d', refId: 'D' } as DataFrame],
+      });
+      ctx.emitPacket({
+        data: [{ name: 'data-e', refId: 'E' } as DataFrame],
+        traceIds: ['t3', 't4'],
+      });
+      ctx.emitPacket({
+        data: [{ name: 'data-e', refId: 'E' } as DataFrame],
+        traceIds: ['t4', 't4'],
+      });
+    });
+    it('should collect traceIds correctly', () => {
+      const { results } = ctx;
+      expect(results).toHaveLength(6);
+      expect(results[0].traceIds).toBeUndefined();
+
+      // this is the result of adding no-traces data to no-traces state
+      expect(results[1].traceIds).toBeUndefined();
+      // this is the result of adding with-traces data to no-traces state
+      expect(results[2].traceIds).toStrictEqual(['t1', 't2']);
+      // this is the result of adding no-traces data to with-traces state
+      expect(results[3].traceIds).toStrictEqual(['t1', 't2']);
+      // this is the result of adding with-traces data to with-traces state
+      expect(results[4].traceIds).toStrictEqual(['t1', 't2', 't3', 't4']);
+      // this is the result of adding with-traces data to with-traces state with duplicate traceIds
+      expect(results[5].traceIds).toStrictEqual(['t1', 't2', 't3', 't4']);
+    });
+  });
+
  runRequestScenario('After response with state Streaming', (ctx) => {
    ctx.setup(() => {
      ctx.start();
--- a/public/app/features/query/state/runRequest.ts
+++ b/public/app/features/query/state/runRequest.ts
@ -88,6 +88,13 @@ export function processResponsePacket(packet: DataQueryResponse, state: RunningQ
    timeRange,
  };

+  // we use a Set to deduplicate the traceIds
+  const traceIdSet = new Set([...(state.panelData.traceIds ?? []), ...(packet.traceIds ?? [])]);
+
+  if (traceIdSet.size > 0) {
+    panelData.traceIds = Array.from(traceIdSet);
+  }
+
  return { packets, panelData };
 }

--- a/public/app/plugins/datasource/graphite/datasource_integration.test.ts
+++ b/public/app/plugins/datasource/graphite/datasource_integration.test.ts
@ -116,12 +116,12 @@ function mockBackendSrv(data: string) {
      redirected: false,
      type: 'basic',
      url: 'http://localhost:3000/api/some-mock',
-      headers: {
+      headers: new Headers({
        method: 'GET',
        url: '/functions',
        // to work around Graphite returning invalid JSON
        responseType: 'text',
-      },
+      }),
    };
    return of(mockedResponse);
  });
--- a/public/app/plugins/datasource/loki/responseUtils.ts
+++ b/public/app/plugins/datasource/loki/responseUtils.ts
@ -154,7 +154,15 @@ export function combineResponses(currentResult: DataQueryResponse | null, newRes
  // some grafana parts do not behave well.
  // we just choose the old error, if it exists,
  // otherwise the new error, if it exists.
-  currentResult.error = currentResult.error ?? newResult.error;
+  const mergedError = currentResult.error ?? newResult.error;
+  if (mergedError != null) {
+    currentResult.error = mergedError;
+  }
+
+  const mergedTraceIds = [...(currentResult.traceIds ?? []), ...(newResult.traceIds ?? [])];
+  if (mergedTraceIds.length > 0) {
+    currentResult.traceIds = mergedTraceIds;
+  }

  return currentResult;
 }