diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 1da4beab3fd..3559bb4464e 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -84,6 +84,7 @@ Alpha features might be changed or removed without prior notice. | `showDashboardValidationWarnings` | Show warnings when dashboards do not validate against the schema | | `mysqlAnsiQuotes` | Use double quotes to escape keyword in a MySQL query | | `elasticsearchBackendMigration` | Use Elasticsearch as backend data source | +| `showTraceId` | Show trace ids for requests | | `datasourceOnboarding` | Enable data source onboarding page | | `secureSocksDatasourceProxy` | Enable secure socks tunneling for supported core datasources | | `authnService` | Use new auth service to perform authentication | diff --git a/packages/grafana-data/src/types/datasource.ts b/packages/grafana-data/src/types/datasource.ts index 3fe8cf36a09..02a196f0240 100644 --- a/packages/grafana-data/src/types/datasource.ts +++ b/packages/grafana-data/src/types/datasource.ts @@ -465,6 +465,11 @@ export interface DataQueryResponse { * Defaults to LoadingState.Done if state is not defined */ state?: LoadingState; + + /** + * traceIds related to the response, if available + */ + traceIds?: string[]; } export enum DataQueryErrorType { @@ -488,6 +493,7 @@ export interface DataQueryError { status?: number; statusText?: string; refId?: string; + traceId?: string; type?: DataQueryErrorType; } diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index d889e89e2f1..0c84cb2fb0b 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -65,6 +65,7 @@ export interface FeatureToggles { nestedFolders?: boolean; accessTokenExpirationCheck?: boolean; elasticsearchBackendMigration?: boolean; + showTraceId?: boolean; datasourceOnboarding?: boolean; emptyDashboardPage?: boolean; secureSocksDatasourceProxy?: boolean; diff --git a/packages/grafana-data/src/types/panel.ts b/packages/grafana-data/src/types/panel.ts index 7e72b7680a7..99c580deeac 100644 --- a/packages/grafana-data/src/types/panel.ts +++ b/packages/grafana-data/src/types/panel.ts @@ -66,6 +66,9 @@ export interface PanelData { /** Contains the range from the request or a shifted time range if a request uses relative time */ timeRange: TimeRange; + + /** traceIds collected during the processing of the requests */ + traceIds?: string[]; } export interface PanelProps { diff --git a/packages/grafana-runtime/src/services/backendSrv.ts b/packages/grafana-runtime/src/services/backendSrv.ts index 15671891069..341794835b9 100644 --- a/packages/grafana-runtime/src/services/backendSrv.ts +++ b/packages/grafana-runtime/src/services/backendSrv.ts @@ -96,6 +96,7 @@ export interface FetchResponse { readonly type: ResponseType; readonly url: string; readonly config: BackendSrvRequest; + readonly traceId?: string; } /** @@ -122,6 +123,7 @@ export interface FetchError { cancelled?: boolean; isHandled?: boolean; config: BackendSrvRequest; + traceId?: string; } export function isFetchError(e: unknown): e is FetchError { diff --git a/packages/grafana-runtime/src/utils/queryResponse.test.ts b/packages/grafana-runtime/src/utils/queryResponse.test.ts index 56be442fd27..adec09b9f9d 100644 --- a/packages/grafana-runtime/src/utils/queryResponse.test.ts +++ b/packages/grafana-runtime/src/utils/queryResponse.test.ts @@ -280,6 +280,82 @@ describe('Query Response parser', () => { expect(ids).toEqual(['A', 'B']); }); + test('should handle a success-response without traceIds', () => { + const input = { + data: { + results: { + A: { + frames: [], + }, + }, + }, + } as unknown as FetchResponse; + const res = toDataQueryResponse(input); + expect(res.traceIds).toBeUndefined(); + }); + + test('should handle a success-response with traceIds', () => { + const input = { + data: { + results: { + A: { + frames: [], + }, + }, + }, + traceId: 'traceId1', + } as unknown as FetchResponse; + const res = toDataQueryResponse(input); + expect(res.traceIds).toStrictEqual(['traceId1']); + }); + + test('should handle an error-response without traceIds', () => { + const input = { + data: { + results: { + A: { + error: 'error from A', + status: 400, + }, + B: { + error: 'error from B', + status: 400, + }, + }, + }, + } as unknown as FetchResponse; + const res = toDataQueryResponse(input); + expect(res.traceIds).toBeUndefined(); + expect(res.error?.traceId).toBeUndefined(); + expect(res.errors).toHaveLength(2); + expect(res.errors?.[0].traceId).toBeUndefined(); + expect(res.errors?.[1].traceId).toBeUndefined(); + }); + + test('should handle an error-response with traceIds', () => { + const input = { + data: { + results: { + A: { + error: 'error from A', + status: 400, + }, + B: { + error: 'error from B', + status: 400, + }, + }, + }, + traceId: 'traceId1', + } as unknown as FetchResponse; + const res = toDataQueryResponse(input); + expect(res.traceIds).toStrictEqual(['traceId1']); + expect(res.error?.traceId).toBe('traceId1'); + expect(res.errors).toHaveLength(2); + expect(res.errors?.[0].traceId).toBe('traceId1'); + expect(res.errors?.[1].traceId).toBe('traceId1'); + }); + describe('Cache notice', () => { let resp: FetchResponse; diff --git a/packages/grafana-runtime/src/utils/queryResponse.ts b/packages/grafana-runtime/src/utils/queryResponse.ts index 919c003ef03..707550f9909 100644 --- a/packages/grafana-runtime/src/utils/queryResponse.ts +++ b/packages/grafana-runtime/src/utils/queryResponse.ts @@ -65,6 +65,13 @@ export function toDataQueryResponse( queries?: DataQuery[] ): DataQueryResponse { const rsp: DataQueryResponse = { data: [], state: LoadingState.Done }; + + const traceId = 'traceId' in res ? res.traceId : undefined; + + if (traceId != null) { + rsp.traceIds = [traceId]; + } + // If the response isn't in a correct shape we just ignore the data and pass empty DataQueryResponse. if ((res as FetchResponse).data?.results) { const results = (res as FetchResponse).data.results; @@ -83,17 +90,21 @@ export function toDataQueryResponse( for (const dr of data) { if (dr.error) { + const errorObj: DataQueryError = { + refId: dr.refId, + message: dr.error, + status: dr.status, + }; + if (traceId != null) { + errorObj.traceId = traceId; + } if (!rsp.error) { - rsp.error = { - refId: dr.refId, - message: dr.error, - status: dr.status, - }; + rsp.error = { ...errorObj }; } if (rsp.errors) { - rsp.errors.push({ refId: dr.refId, message: dr.error, status: dr.status }); + rsp.errors.push({ ...errorObj }); } else { - rsp.errors = [{ refId: dr.refId, message: dr.error, status: dr.status }]; + rsp.errors = [{ ...errorObj }]; } rsp.state = LoadingState.Error; } diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index 4980d7cb2a4..eb4861dbbeb 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/grafana/grafana/pkg/infra/tracing" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/setting" @@ -29,12 +30,18 @@ func AddDefaultResponseHeaders(cfg *setting.Cfg) web.Handler { t := web.NewTree() t.Add("/api/datasources/uid/:uid/resources/*", nil) t.Add("/api/datasources/:id/resources/*", nil) + return func(c *web.Context) { c.Resp.Before(func(w web.ResponseWriter) { // if response has already been written, skip. if w.Written() { return } + traceId := tracing.TraceIDFromContext(c.Req.Context(), false) + if traceId != "" { + w.Header().Set("grafana-trace-id", traceId) + } + _, _, resourceURLMatch := t.Match(c.Req.URL.Path) resourceCachable := resourceURLMatch && allowCacheControl(c.Resp) if !strings.HasPrefix(c.Req.URL.Path, "/public/plugins/") && diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index c3f84f0dee1..11bf3812990 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -315,6 +315,12 @@ var ( State: FeatureStateAlpha, Owner: grafanaObservabilityLogsSquad, }, + { + Name: "showTraceId", + Description: "Show trace ids for requests", + State: FeatureStateAlpha, + Owner: grafanaObservabilityLogsSquad, + }, { Name: "datasourceOnboarding", Description: "Enable data source onboarding page", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index ee34a99f725..d7bbe3f1e55 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -46,6 +46,7 @@ accessControlOnCall,beta,@grafana/grafana-authnz-team,false,false,false,false nestedFolders,alpha,@grafana/backend-platform,true,false,false,false accessTokenExpirationCheck,stable,@grafana/grafana-authnz-team,false,false,false,false elasticsearchBackendMigration,alpha,@grafana/observability-logs,false,false,false,false +showTraceId,alpha,@grafana/observability-logs,false,false,false,false datasourceOnboarding,alpha,@grafana/dashboards-squad,false,false,false,false emptyDashboardPage,stable,@grafana/dashboards-squad,false,false,false,true secureSocksDatasourceProxy,alpha,@grafana/hosted-grafana-team,false,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index e7c96cbe473..4595ec528e1 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -195,6 +195,10 @@ const ( // Use Elasticsearch as backend data source FlagElasticsearchBackendMigration = "elasticsearchBackendMigration" + // FlagShowTraceId + // Show trace ids for requests + FlagShowTraceId = "showTraceId" + // FlagDatasourceOnboarding // Enable data source onboarding page FlagDatasourceOnboarding = "datasourceOnboarding" diff --git a/public/app/core/services/backend_srv.ts b/public/app/core/services/backend_srv.ts index d2e9b6c42f0..16acfd7c110 100644 --- a/public/app/core/services/backend_srv.ts +++ b/public/app/core/services/backend_srv.ts @@ -50,6 +50,8 @@ export interface FolderRequestOptions { withAccessControl?: boolean; } +const GRAFANA_TRACEID_HEADER = 'grafana-trace-id'; + export class BackendSrv implements BackendService { private inFlightRequests: Subject = new Subject(); private HTTP_REQUEST_CANCELED = -1; @@ -223,6 +225,7 @@ export class BackendSrv implements BackendService { type, redirected, config: options, + traceId: response.headers.get(GRAFANA_TRACEID_HEADER) ?? undefined, }; return fetchResponse; }), @@ -238,7 +241,13 @@ export class BackendSrv implements BackendService { filter((response) => response.ok === false), mergeMap((response) => { const { status, statusText, data } = response; - const fetchErrorResponse: FetchError = { status, statusText, data, config: options }; + const fetchErrorResponse: FetchError = { + status, + statusText, + data, + config: options, + traceId: response.headers.get(GRAFANA_TRACEID_HEADER) ?? undefined, + }; return throwError(fetchErrorResponse); }), retryWhen((attempts: Observable) => diff --git a/public/app/core/specs/backend_srv.test.ts b/public/app/core/specs/backend_srv.test.ts index a0fe3067587..ffa46ae60db 100644 --- a/public/app/core/specs/backend_srv.test.ts +++ b/public/app/core/specs/backend_srv.test.ts @@ -1,5 +1,5 @@ import 'whatwg-fetch'; // fetch polyfill needed for PhantomJs rendering -import { Observable, of } from 'rxjs'; +import { Observable, of, lastValueFrom } from 'rxjs'; import { delay } from 'rxjs/operators'; import { AppEvents, DataQueryErrorType, EventBusExtended } from '@grafana/data'; @@ -21,6 +21,7 @@ const getTestContext = (overides?: object) => { redirected: false, type: 'basic', url: 'http://localhost:3000/api/some-mock', + headers: new Map(), }; const props = { ...defaults, ...overides }; const textMock = jest.fn().mockResolvedValue(JSON.stringify(props.data)); @@ -29,6 +30,7 @@ const getTestContext = (overides?: object) => { ok: props.ok, status: props.status, statusText: props.statusText, + headers: props.headers, text: textMock, redirected: false, type: 'basic', @@ -355,6 +357,63 @@ describe('backendSrv', () => { }); }); }); + + describe('traceId handling', () => { + const opts = { url: '/something', method: 'GET' }; + it('should handle a success-response without traceId', async () => { + const ctx = getTestContext({ status: 200, statusText: 'OK', headers: new Headers() }); + const res = await lastValueFrom(ctx.backendSrv.fetch(opts)); + expect(res.traceId).toBeUndefined(); + }); + + it('should handle a success-response with traceId', async () => { + const ctx = getTestContext({ + status: 200, + statusText: 'OK', + headers: new Headers({ + 'grafana-trace-id': 'traceId1', + }), + }); + const res = await lastValueFrom(ctx.backendSrv.fetch(opts)); + expect(res.traceId).toBe('traceId1'); + }); + + it('should handle an error-response without traceId', () => { + const ctx = getTestContext({ + ok: false, + status: 500, + statusText: 'INTERNAL SERVER ERROR', + headers: new Headers(), + }); + return lastValueFrom(ctx.backendSrv.fetch(opts)).then( + (data) => { + throw new Error('must not get here'); + }, + (error) => { + expect(error.traceId).toBeUndefined(); + } + ); + }); + + it('should handle an error-response with traceId', () => { + const ctx = getTestContext({ + ok: false, + status: 500, + statusText: 'INTERNAL SERVER ERROR', + headers: new Headers({ + 'grafana-trace-id': 'traceId1', + }), + }); + return lastValueFrom(ctx.backendSrv.fetch(opts)).then( + (data) => { + throw new Error('must not get here'); + }, + (error) => { + expect(error.traceId).toBe('traceId1'); + } + ); + }); + }); }); describe('datasourceRequest', () => { @@ -369,6 +428,7 @@ describe('backendSrv', () => { ok: true, status: 200, statusText: 'Ok', + headers: new Map(), text: () => Promise.resolve(JSON.stringify(slowData)), redirected: false, type: 'basic', @@ -382,6 +442,7 @@ describe('backendSrv', () => { ok: true, status: 200, statusText: 'Ok', + headers: new Map(), text: () => Promise.resolve(JSON.stringify(fastData)), redirected: false, type: 'basic', diff --git a/public/app/features/inspector/InspectErrorTab.tsx b/public/app/features/inspector/InspectErrorTab.tsx index 9c9b4b71aa1..a93d1338b62 100644 --- a/public/app/features/inspector/InspectErrorTab.tsx +++ b/public/app/features/inspector/InspectErrorTab.tsx @@ -1,6 +1,7 @@ import React from 'react'; import { DataQueryError } from '@grafana/data'; +import { config } from '@grafana/runtime'; import { Alert, JSONFormatter } from '@grafana/ui'; interface InspectErrorTabProps { @@ -36,6 +37,12 @@ function renderError(error: DataQueryError) { <> {error.status && <>Status: {error.status}. Message: } {msg} + {config.featureToggles.showTraceId && error.traceId != null && ( + <> +
+ (Trace ID: {error.traceId}) + + )} ); } else { diff --git a/public/app/features/inspector/InspectStatsTab.tsx b/public/app/features/inspector/InspectStatsTab.tsx index 541a3bd0da5..3977cffcb2b 100644 --- a/public/app/features/inspector/InspectStatsTab.tsx +++ b/public/app/features/inspector/InspectStatsTab.tsx @@ -2,9 +2,11 @@ import React from 'react'; import { PanelData, QueryResultMetaStat, TimeZone } from '@grafana/data'; import { selectors } from '@grafana/e2e-selectors'; +import { config } from '@grafana/runtime'; import { t } from 'app/core/internationalization'; import { InspectStatsTable } from './InspectStatsTable'; +import { InspectStatsTraceIdsTable } from './InspectStatsTraceIdsTable'; interface InspectStatsTabProps { data: PanelData; @@ -59,11 +61,15 @@ export const InspectStatsTab = ({ data, timeZone }: InspectStatsTabProps) => { const statsTableName = t('dashboard.inspect-stats.table-title', 'Stats'); const dataStatsTableName = t('dashboard.inspect-stats.data-title', 'Data source stats'); + const traceIdsStatsTableName = t('dashboard.inspect-stats.data-traceids', 'Trace IDs'); return (
+ {config.featureToggles.showTraceId && ( + + )}
); }; diff --git a/public/app/features/inspector/InspectStatsTraceIdsTable.tsx b/public/app/features/inspector/InspectStatsTraceIdsTable.tsx new file mode 100644 index 00000000000..ab958f9bf66 --- /dev/null +++ b/public/app/features/inspector/InspectStatsTraceIdsTable.tsx @@ -0,0 +1,47 @@ +import { css } from '@emotion/css'; +import React from 'react'; + +import { GrafanaTheme2 } from '@grafana/data'; +import { stylesFactory, useTheme2 } from '@grafana/ui'; + +type Props = { + name: string; + traceIds: string[]; +}; + +export const InspectStatsTraceIdsTable = ({ name, traceIds }: Props) => { + const theme = useTheme2(); + const styles = getStyles(theme); + + if (traceIds.length === 0) { + return null; + } + + return ( +
+
{name}
+ + + {traceIds.map((traceId, index) => { + return ( + + + + ); + })} + +
{traceId}
+
+ ); +}; + +const getStyles = stylesFactory((theme: GrafanaTheme2) => { + return { + wrapper: css` + padding-bottom: ${theme.spacing(2)}; + `, + cell: css` + text-align: right; + `, + }; +}); diff --git a/public/app/features/query/components/QueryErrorAlert.tsx b/public/app/features/query/components/QueryErrorAlert.tsx index 3a31cdadadf..efe60c58312 100644 --- a/public/app/features/query/components/QueryErrorAlert.tsx +++ b/public/app/features/query/components/QueryErrorAlert.tsx @@ -2,6 +2,7 @@ import { css } from '@emotion/css'; import React from 'react'; import { DataQueryError, GrafanaTheme2 } from '@grafana/data'; +import { config } from '@grafana/runtime'; import { Icon, useStyles2 } from '@grafana/ui'; export interface Props { @@ -18,7 +19,14 @@ export function QueryErrorAlert({ error }: Props) {
-
{message}
+
+ {message} + {config.featureToggles.showTraceId && error.traceId != null && ( + <> +
(Trace ID: {error.traceId}) + + )} +
); } diff --git a/public/app/features/query/state/runRequest.test.ts b/public/app/features/query/state/runRequest.test.ts index d52bdfdee42..03334ea7dbf 100644 --- a/public/app/features/query/state/runRequest.test.ts +++ b/public/app/features/query/state/runRequest.test.ts @@ -223,6 +223,49 @@ describe('runRequest', () => { }); }); + runRequestScenario('When the response contains traceIds', (ctx) => { + ctx.setup(() => { + ctx.start(); + ctx.emitPacket({ + data: [{ name: 'data-a', refId: 'A' } as DataFrame], + }); + ctx.emitPacket({ + data: [{ name: 'data-b', refId: 'B' } as DataFrame], + }); + ctx.emitPacket({ + data: [{ name: 'data-c', refId: 'C' } as DataFrame], + traceIds: ['t1', 't2'], + }); + ctx.emitPacket({ + data: [{ name: 'data-d', refId: 'D' } as DataFrame], + }); + ctx.emitPacket({ + data: [{ name: 'data-e', refId: 'E' } as DataFrame], + traceIds: ['t3', 't4'], + }); + ctx.emitPacket({ + data: [{ name: 'data-e', refId: 'E' } as DataFrame], + traceIds: ['t4', 't4'], + }); + }); + it('should collect traceIds correctly', () => { + const { results } = ctx; + expect(results).toHaveLength(6); + expect(results[0].traceIds).toBeUndefined(); + + // this is the result of adding no-traces data to no-traces state + expect(results[1].traceIds).toBeUndefined(); + // this is the result of adding with-traces data to no-traces state + expect(results[2].traceIds).toStrictEqual(['t1', 't2']); + // this is the result of adding no-traces data to with-traces state + expect(results[3].traceIds).toStrictEqual(['t1', 't2']); + // this is the result of adding with-traces data to with-traces state + expect(results[4].traceIds).toStrictEqual(['t1', 't2', 't3', 't4']); + // this is the result of adding with-traces data to with-traces state with duplicate traceIds + expect(results[5].traceIds).toStrictEqual(['t1', 't2', 't3', 't4']); + }); + }); + runRequestScenario('After response with state Streaming', (ctx) => { ctx.setup(() => { ctx.start(); diff --git a/public/app/features/query/state/runRequest.ts b/public/app/features/query/state/runRequest.ts index d7480fdaef9..1c4ace55276 100644 --- a/public/app/features/query/state/runRequest.ts +++ b/public/app/features/query/state/runRequest.ts @@ -88,6 +88,13 @@ export function processResponsePacket(packet: DataQueryResponse, state: RunningQ timeRange, }; + // we use a Set to deduplicate the traceIds + const traceIdSet = new Set([...(state.panelData.traceIds ?? []), ...(packet.traceIds ?? [])]); + + if (traceIdSet.size > 0) { + panelData.traceIds = Array.from(traceIdSet); + } + return { packets, panelData }; } diff --git a/public/app/plugins/datasource/graphite/datasource_integration.test.ts b/public/app/plugins/datasource/graphite/datasource_integration.test.ts index 5c853d496a5..fe9b3ab4859 100644 --- a/public/app/plugins/datasource/graphite/datasource_integration.test.ts +++ b/public/app/plugins/datasource/graphite/datasource_integration.test.ts @@ -116,12 +116,12 @@ function mockBackendSrv(data: string) { redirected: false, type: 'basic', url: 'http://localhost:3000/api/some-mock', - headers: { + headers: new Headers({ method: 'GET', url: '/functions', // to work around Graphite returning invalid JSON responseType: 'text', - }, + }), }; return of(mockedResponse); }); diff --git a/public/app/plugins/datasource/loki/responseUtils.ts b/public/app/plugins/datasource/loki/responseUtils.ts index f961eb95acf..306c3b387f8 100644 --- a/public/app/plugins/datasource/loki/responseUtils.ts +++ b/public/app/plugins/datasource/loki/responseUtils.ts @@ -154,7 +154,15 @@ export function combineResponses(currentResult: DataQueryResponse | null, newRes // some grafana parts do not behave well. // we just choose the old error, if it exists, // otherwise the new error, if it exists. - currentResult.error = currentResult.error ?? newResult.error; + const mergedError = currentResult.error ?? newResult.error; + if (mergedError != null) { + currentResult.error = mergedError; + } + + const mergedTraceIds = [...(currentResult.traceIds ?? []), ...(newResult.traceIds ?? [])]; + if (mergedTraceIds.length > 0) { + currentResult.traceIds = mergedTraceIds; + } return currentResult; }